From bc44fbb61a66c739e9c904356ca705500288da07 Mon Sep 17 00:00:00 2001 From: andrea Date: Mon, 12 Aug 2024 16:02:03 +0200 Subject: [PATCH 1/7] fix: handle missing data for prices/rates --- yfdata/parsing.py | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/yfdata/parsing.py b/yfdata/parsing.py index 9fbe1ed..27e755c 100644 --- a/yfdata/parsing.py +++ b/yfdata/parsing.py @@ -29,25 +29,38 @@ def parse_prices_or_rates(body: dict, ticker_or_pair: str) -> DataFrame: else: raise ValueError("Instrument type not supported.") - ts = data["timestamp"] + if "timestamp" in data and "indicators" in data: + + ts = data["timestamp"] + quotes = data["indicators"]["quote"][0] + + df = DataFrame( + data={ + "ts": ts, + "o": quotes["open"], + "h": quotes["high"], + "l": quotes["low"], + "c": quotes["close"], + "v": quotes["volume"], + } + ) - quotes = data["indicators"]["quote"][0] - - df = DataFrame( - data={ - "ts": ts, - "o": quotes["open"], - "h": quotes["high"], - "l": quotes["low"], - "c": quotes["close"], - "v": quotes["volume"], - } - ) + else: + df = DataFrame( + data={ + "ts": [], + "o": [], + "h": [], + "l": [], + "c": [], + "v": [], + } + ) df[code_name] = ticker_or_pair.lower() df["ts"] = pd.to_datetime(df["ts"], unit="s") - # .dt.tz_convert('Europe/Paris').dt.tz_convert('UTC') + # Reorder the columns. df = df[[code_name, "ts", "o", "h", "l", "c", "v"]] return df From b7e457361d216e1eab03cb4a6b8fc47b1dfb834b Mon Sep 17 00:00:00 2001 From: andrea Date: Mon, 12 Aug 2024 16:02:32 +0200 Subject: [PATCH 2/7] chg: removed github badge --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index ecf12eb..19f2978 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # yf-data ![application-build](https://github.com/caps6/yf-data/actions/workflows/python-build.yml/badge.svg) -![GitHub Downloads](https://img.shields.io/github/downloads/caps6/yf-data/total) ![PyPI - Downloads](https://img.shields.io/pypi/dm/yfdata) A simple-but-working python module that returns data from Yahoo Finance. From 86351e91b28d27d5c18a56713f2c3d0a619f5cb3 Mon Sep 17 00:00:00 2001 From: andrea Date: Mon, 12 Aug 2024 16:02:48 +0200 Subject: [PATCH 3/7] =?UTF-8?q?Bump=20version:=200.1.2=20=E2=86=92=200.1.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.md | 2 +- pyproject.toml | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index fb41de1..0cfc5c9 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.2 +current_version = 0.1.3 commit = True tag = True diff --git a/README.md b/README.md index 19f2978..eb81086 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ A simple-but-working python module that returns data from Yahoo Finance. -Current Version: 0.1.2 +Current Version: 0.1.3 ### Features diff --git a/pyproject.toml b/pyproject.toml index f0956ba..1ee2e5c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "yfdata" -version = "0.1.2" +version = "0.1.3" description = "Client for Yahoo Finance data" authors = ["andrea "] license = "MIT" diff --git a/setup.py b/setup.py index 5c1a6dd..f8b8baa 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ setup( name="yf-data", - version="0.1.2", + version="0.1.3", keywords="yahoo finance data", description=DESCRIPTION, packages=packages, From 888273f12ad01e9a36b4f1e0a6323fb85285beaf Mon Sep 17 00:00:00 2001 From: andrea Date: Mon, 12 Aug 2024 16:51:51 +0200 Subject: [PATCH 4/7] fix: add check for dict when parsing data --- yfdata/parsing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yfdata/parsing.py b/yfdata/parsing.py index 27e755c..26e6fac 100644 --- a/yfdata/parsing.py +++ b/yfdata/parsing.py @@ -29,7 +29,7 @@ def parse_prices_or_rates(body: dict, ticker_or_pair: str) -> DataFrame: else: raise ValueError("Instrument type not supported.") - if "timestamp" in data and "indicators" in data: + if isinstance(data, dict) and "timestamp" in data and "indicators" in data: ts = data["timestamp"] quotes = data["indicators"]["quote"][0] @@ -136,7 +136,7 @@ def parse_dividends(body: dict, ticker: str) -> DataFrame: dates = [] dividends = [] - if "events" in data: + if isinstance(data, dict) and "events" in data: dict_dividends = data["events"]["dividends"] From 2f6f67ec390c3b8de7ae1bf2da77610ea9b4533e Mon Sep 17 00:00:00 2001 From: andrea Date: Mon, 12 Aug 2024 16:52:22 +0200 Subject: [PATCH 5/7] =?UTF-8?q?Bump=20version:=200.1.3=20=E2=86=92=200.1.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.md | 2 +- pyproject.toml | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0cfc5c9..c7f9fbd 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.3 +current_version = 0.1.4 commit = True tag = True diff --git a/README.md b/README.md index eb81086..f4e5fa4 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ A simple-but-working python module that returns data from Yahoo Finance. -Current Version: 0.1.3 +Current Version: 0.1.4 ### Features diff --git a/pyproject.toml b/pyproject.toml index 1ee2e5c..5bd3c15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "yfdata" -version = "0.1.3" +version = "0.1.4" description = "Client for Yahoo Finance data" authors = ["andrea "] license = "MIT" diff --git a/setup.py b/setup.py index f8b8baa..cb2f7fd 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ setup( name="yf-data", - version="0.1.3", + version="0.1.4", keywords="yahoo finance data", description=DESCRIPTION, packages=packages, From a29f7b405cd65d2d1f3d3067068bb93e845877db Mon Sep 17 00:00:00 2001 From: andrea Date: Tue, 13 Aug 2024 15:43:08 +0200 Subject: [PATCH 6/7] fix: additional controls for missing data --- yfdata/parsing.py | 92 +++++++++++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 39 deletions(-) diff --git a/yfdata/parsing.py b/yfdata/parsing.py index 26e6fac..bcfcd37 100644 --- a/yfdata/parsing.py +++ b/yfdata/parsing.py @@ -19,33 +19,43 @@ def parse_prices_or_rates(body: dict, ticker_or_pair: str) -> DataFrame: """ - data = body["chart"]["result"][0] + df = None - metadata = data["meta"] - if metadata["instrumentType"] == "EQUITY" or metadata["instrumentType"] == "ETF": - code_name = "ticker" - elif metadata["instrumentType"] == "CURRENCY": - code_name = "pair" - else: - raise ValueError("Instrument type not supported.") + result = body["chart"]["result"] - if isinstance(data, dict) and "timestamp" in data and "indicators" in data: + if isinstance(result, list) and len(result) > 0: - ts = data["timestamp"] - quotes = data["indicators"]["quote"][0] + data = body["chart"]["result"][0] - df = DataFrame( - data={ - "ts": ts, - "o": quotes["open"], - "h": quotes["high"], - "l": quotes["low"], - "c": quotes["close"], - "v": quotes["volume"], - } - ) + metadata = data["meta"] + if ( + metadata["instrumentType"] == "EQUITY" + or metadata["instrumentType"] == "ETF" + ): + code_name = "ticker" + elif metadata["instrumentType"] == "CURRENCY": + code_name = "pair" + else: + raise ValueError("Instrument type not supported.") + + if isinstance(data, dict) and "timestamp" in data and "indicators" in data: + + ts = data["timestamp"] + quotes = data["indicators"]["quote"][0] - else: + df = DataFrame( + data={ + "ts": ts, + "o": quotes["open"], + "h": quotes["high"], + "l": quotes["low"], + "c": quotes["close"], + "v": quotes["volume"], + } + ) + + if df is None: + code_name = "instrument" df = DataFrame( data={ "ts": [], @@ -100,15 +110,15 @@ def parse_financials(body: dict, ticker: str, freq: str, mapping: dict) -> DataF dates.append(item["asOfDate"]) values.append(float(item["reportedValue"]["raw"])) - df = DataFrame( - data={ - "date": dates, - "value": values, - } - ) + df = DataFrame( + data={ + "date": dates, + "value": values, + } + ) - df["metric"] = mapping[yahoo_metric_name] - dfs.append(df) + df["metric"] = mapping[yahoo_metric_name] + dfs.append(df) df = pd.concat(dfs, ignore_index=True) df["ticker"] = ticker.lower() @@ -131,22 +141,26 @@ def parse_dividends(body: dict, ticker: str) -> DataFrame: """ - data = body["chart"]["result"][0] - dates = [] dividends = [] - if isinstance(data, dict) and "events" in data: + result = body["chart"]["result"] + + if isinstance(result, list) and len(result) > 0: + + data = result[0] + + if isinstance(data, dict) and "events" in data: - dict_dividends = data["events"]["dividends"] + dict_dividends = data["events"]["dividends"] - for uts in dict_dividends: + for uts in dict_dividends: - if "amount" in dict_dividends[uts]: + if "amount" in dict_dividends[uts]: - ts = utils.timestamp2datetime(int(uts)) - dates.append(ts) - dividends.append(dict_dividends[uts]["amount"]) + ts = utils.timestamp2datetime(int(uts)) + dates.append(ts) + dividends.append(dict_dividends[uts]["amount"]) df = DataFrame( data={ From 9abf0eee96811d2b544456d2e345240eb418fc43 Mon Sep 17 00:00:00 2001 From: andrea Date: Tue, 13 Aug 2024 15:44:26 +0200 Subject: [PATCH 7/7] =?UTF-8?q?Bump=20version:=200.1.4=20=E2=86=92=200.1.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- README.md | 2 +- pyproject.toml | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c7f9fbd..b6166a6 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.4 +current_version = 0.1.5 commit = True tag = True diff --git a/README.md b/README.md index f4e5fa4..d8f2023 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ A simple-but-working python module that returns data from Yahoo Finance. -Current Version: 0.1.4 +Current Version: 0.1.5 ### Features diff --git a/pyproject.toml b/pyproject.toml index 5bd3c15..8bf4de7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "yfdata" -version = "0.1.4" +version = "0.1.5" description = "Client for Yahoo Finance data" authors = ["andrea "] license = "MIT" diff --git a/setup.py b/setup.py index cb2f7fd..9f56b67 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ setup( name="yf-data", - version="0.1.4", + version="0.1.5", keywords="yahoo finance data", description=DESCRIPTION, packages=packages,