Bump version to 0.2.4

dev -> main

CHANGELOG.rst

@@ -1,6 +1,18 @@
 Change Log
 ===========
 
+0.2.4
+-----
+- Fix Yahoo data decryption #1297
+- New feature: 'Ticker.get_shares_full()' #1301
+- Improve caching of financials data #1284
+- Restore download() original alignment behaviour #1283
+- Fix the database lock error in multithread download #1276
+
+0.2.3
+-----
+- Make financials API '_' use consistent
+
 0.2.2
 -----
 - Restore 'financials' attribute (map to 'income_stmt')

README.md

@@ -42,6 +42,13 @@ Yahoo! finance API is intended for personal use only.**
 
 ---
 
+## What's new in version 0.2
+
+- Optimised web scraping
+- All 3 financials tables now match website so expect keys to change. If you really want old tables, use [`Ticker.get_[income_stmt|balance_sheet|cashflow](legacy=True, ...)`](https://github.com/ranaroussi/yfinance/blob/85783da515761a145411d742c2a8a3c1517264b0/yfinance/base.py#L968)
+- price data improvements: fix bug NaN rows with dividend; new repair feature for missing or 100x prices `download(repair=True)`; new attribute `Ticker.history_metadata`
+[See release notes for full list of changes](https://github.com/ranaroussi/yfinance/releases/tag/0.2.1)
+
 ## Quick Start
 
 ### The Ticker module
@@ -77,6 +84,7 @@ msft.capital_gains
 
 # show share count
 msft.shares
+msft.get_shares_full()
 
 # show financials:
 # - income statement
@@ -206,8 +214,7 @@ data = yf.download(  # or pdr.get_data_yahoo(...
         interval = "5d",
 
         # Whether to ignore timezone when aligning ticker data from 
-        # different timezones. Default is True. False may be useful for 
-        # minute/hourly data.
+        # different timezones. Default is False.
         ignore_tz = False,
 
         # group by ticker (to access via data['SPY'])

meta.yaml

@@ -1,5 +1,5 @@
 {% set name = "yfinance" %}
-{% set version = "0.2.2" %}
+{% set version = "0.2.4" %}
 
 package:
   name: "{{ name|lower }}"

tests/ticker.py

@@ -65,6 +65,7 @@ class TestTicker(unittest.TestCase):
         dat.splits
         dat.actions
         dat.shares
+        dat.get_shares_full()
         dat.info
         dat.calendar
         dat.recommendations
@@ -100,6 +101,7 @@ class TestTicker(unittest.TestCase):
         dat.splits
         dat.actions
         dat.shares
+        dat.get_shares_full()
         dat.info
         dat.calendar
         dat.recommendations
@@ -528,6 +530,65 @@ class TestTickerMiscFinancials(unittest.TestCase):
         data_cached = self.ticker_old_fmt.get_cashflow(legacy=True, freq="quarterly")
         self.assertIs(data, data_cached, "data not cached")
 
+    def test_income_alt_names(self):
+        i1 = self.ticker.income_stmt
+        i2 = self.ticker.incomestmt
+        self.assertTrue(i1.equals(i2))
+        i3 = self.ticker.financials
+        self.assertTrue(i1.equals(i3))
+
+        i1 = self.ticker.get_income_stmt()
+        i2 = self.ticker.get_incomestmt()
+        self.assertTrue(i1.equals(i2))
+        i3 = self.ticker.get_financials()
+        self.assertTrue(i1.equals(i3))
+
+        i1 = self.ticker.quarterly_income_stmt
+        i2 = self.ticker.quarterly_incomestmt
+        self.assertTrue(i1.equals(i2))
+        i3 = self.ticker.quarterly_financials
+        self.assertTrue(i1.equals(i3))
+
+        i1 = self.ticker.get_income_stmt(freq="quarterly")
+        i2 = self.ticker.get_incomestmt(freq="quarterly")
+        self.assertTrue(i1.equals(i2))
+        i3 = self.ticker.get_financials(freq="quarterly")
+        self.assertTrue(i1.equals(i3))
+
+    def test_balance_sheet_alt_names(self):
+        i1 = self.ticker.balance_sheet
+        i2 = self.ticker.balancesheet
+        self.assertTrue(i1.equals(i2))
+
+        i1 = self.ticker.get_balance_sheet()
+        i2 = self.ticker.get_balancesheet()
+        self.assertTrue(i1.equals(i2))
+
+        i1 = self.ticker.quarterly_balance_sheet
+        i2 = self.ticker.quarterly_balancesheet
+        self.assertTrue(i1.equals(i2))
+
+        i1 = self.ticker.get_balance_sheet(freq="quarterly")
+        i2 = self.ticker.get_balancesheet(freq="quarterly")
+        self.assertTrue(i1.equals(i2))
+
+    def test_cash_flow_alt_names(self):
+        i1 = self.ticker.cash_flow
+        i2 = self.ticker.cashflow
+        self.assertTrue(i1.equals(i2))
+
+        i1 = self.ticker.get_cash_flow()
+        i2 = self.ticker.get_cashflow()
+        self.assertTrue(i1.equals(i2))
+
+        i1 = self.ticker.quarterly_cash_flow
+        i2 = self.ticker.quarterly_cashflow
+        self.assertTrue(i1.equals(i2))
+
+        i1 = self.ticker.get_cash_flow(freq="quarterly")
+        i2 = self.ticker.get_cashflow(freq="quarterly")
+        self.assertTrue(i1.equals(i2))
+
     def test_sustainability(self):
         data = self.ticker.sustainability
         self.assertIsInstance(data, pd.DataFrame, "data has wrong type")
@@ -594,6 +655,11 @@ class TestTickerMiscFinancials(unittest.TestCase):
         self.assertIsInstance(data, pd.DataFrame, "data has wrong type")
         self.assertFalse(data.empty, "data is empty")
 
+    def test_shares_full(self):
+        data = self.ticker.get_shares_full()
+        self.assertIsInstance(data, pd.Series, "data has wrong type")
+        self.assertFalse(data.empty, "data is empty")
+
     def test_info(self):
         data = self.ticker.info
         self.assertIsInstance(data, dict, "data has wrong type")

yfinance/base.py

@@ -40,6 +40,7 @@ from .scrapers.analysis import Analysis
 from .scrapers.fundamentals import Fundamentals
 from .scrapers.holders import Holders
 from .scrapers.quote import Quote
+import json as _json
 
 _BASE_URL_ = 'https://query2.finance.yahoo.com'
 _SCRAPE_URL_ = 'https://finance.yahoo.com/quote'
@@ -998,6 +999,12 @@ class TickerBase:
             return data.to_dict()
         return data
 
+    def get_incomestmt(self, proxy=None, as_dict=False, pretty=False, freq="yearly", legacy=False):
+        return self.get_income_stmt(proxy, as_dict, pretty, freq, legacy)
+
+    def get_financials(self, proxy=None, as_dict=False, pretty=False, freq="yearly", legacy=False):
+        return self.get_income_stmt(proxy, as_dict, pretty, freq, legacy)
+
     def get_balance_sheet(self, proxy=None, as_dict=False, pretty=False, freq="yearly", legacy=False):
         """
         :Parameters:
@@ -1031,7 +1038,10 @@ class TickerBase:
             return data.to_dict()
         return data
 
-    def get_cashflow(self, proxy=None, as_dict=False, pretty=False, freq="yearly", legacy=False):
+    def get_balancesheet(self, proxy=None, as_dict=False, pretty=False, freq="yearly", legacy=False):
+        return self.get_balance_sheet(proxy, as_dict, pretty, freq, legacy)
+
+    def get_cash_flow(self, proxy=None, as_dict=False, pretty=False, freq="yearly", legacy=False):
         """
         :Parameters:
             as_dict: bool
@@ -1064,6 +1074,9 @@ class TickerBase:
             return data.to_dict()
         return data
 
+    def get_cashflow(self, proxy=None, as_dict=False, pretty=False, freq="yearly", legacy=False):
+        return self.get_cash_flow(proxy, as_dict, pretty, freq, legacy)
+
     def get_dividends(self, proxy=None):
         if self._history is None:
             self.history(period="max", proxy=proxy)
@@ -1106,6 +1119,59 @@ class TickerBase:
             return data.to_dict()
         return data
 
+    def get_shares_full(self, start=None, end=None, proxy=None):
+        # Process dates
+        tz = self._get_ticker_tz(debug_mode=False, proxy=None, timeout=10)
+        dt_now = _pd.Timestamp.utcnow().tz_convert(tz)
+        if start is not None:
+            start_ts = utils._parse_user_dt(start, tz)
+            start = _pd.Timestamp.fromtimestamp(start_ts).tz_localize("UTC").tz_convert(tz)
+            start_d = start.date()
+        if end is not None:
+            end_ts = utils._parse_user_dt(end, tz)
+            end = _pd.Timestamp.fromtimestamp(end_ts).tz_localize("UTC").tz_convert(tz)
+            end_d = end.date()
+        if end is None:
+            end = dt_now
+        if start is None:
+            start = end - _pd.Timedelta(days=548)  # 18 months
+        if start >= end:
+            print("ERROR: start date must be before end")
+            return None
+        start = start.floor("D")
+        end = end.ceil("D")
+        
+        # Fetch
+        ts_url_base = "https://query2.finance.yahoo.com/ws/fundamentals-timeseries/v1/finance/timeseries/{0}?symbol={0}".format(self.ticker)
+        shares_url = ts_url_base + "&period1={}&period2={}".format(int(start.timestamp()), int(end.timestamp()))
+        try:
+            json_str = self._data.cache_get(shares_url).text
+            json_data = _json.loads(json_str)
+        except:
+            print(f"{self.ticker}: Yahoo web request for share count failed")
+            return None
+        try:
+            fail = json_data["finance"]["error"]["code"] == "Bad Request"
+        except:
+            fail = False
+        if fail:
+            print(f"{self.ticker}: Yahoo web request for share count failed")
+            return None
+
+        shares_data = json_data["timeseries"]["result"]
+        if not "shares_out" in shares_data[0]:
+            print(f"{self.ticker}: Yahoo did not return share count in date range {start} -> {end}")
+            return None
+        try:
+            df = _pd.Series(shares_data[0]["shares_out"], index=_pd.to_datetime(shares_data[0]["timestamp"], unit="s"))
+        except Exception as e:
+            print(f"{self.ticker}: Failed to parse shares count data: "+str(e))
+            return None
+
+        df.index = df.index.tz_localize(tz)
+        df = df.sort_index()
+        return df
+
     def get_isin(self, proxy=None) -> Optional[str]:
         # *** experimental ***
         if self._isin is not None:
@@ -1242,8 +1308,8 @@ class TickerBase:
         dates[cn] = _pd.to_datetime(dates[cn], format="%b %d, %Y, %I %p")
         # - instead of attempting decoding of ambiguous timezone abbreviation, just use 'info':
         self._quote.proxy = proxy
-        dates[cn] = dates[cn].dt.tz_localize(
-            tz=self._quote.info["exchangeTimezoneName"])
+        tz = self._get_ticker_tz(debug_mode=False, proxy=proxy, timeout=30)
+        dates[cn] = dates[cn].dt.tz_localize(tz)
 
         dates = dates.set_index("Earnings Date")
 

yfinance/data.py

@@ -46,13 +46,33 @@ def lru_cache_freezeargs(func):
     return wrapped
 
 
-def decrypt_cryptojs_aes(data):
+def decrypt_cryptojs_aes_stores(data):
     encrypted_stores = data['context']['dispatcher']['stores']
-    _cs = data["_cs"]
-    _cr = data["_cr"]
 
-    _cr = b"".join(int.to_bytes(i, length=4, byteorder="big", signed=True) for i in json.loads(_cr)["words"])
-    password = hashlib.pbkdf2_hmac("sha1", _cs.encode("utf8"), _cr, 1, dklen=32).hex()
+    if "_cs" in data and "_cr" in data:
+        _cs = data["_cs"]
+        _cr = data["_cr"]
+        _cr = b"".join(int.to_bytes(i, length=4, byteorder="big", signed=True) for i in json.loads(_cr)["words"])
+        password = hashlib.pbkdf2_hmac("sha1", _cs.encode("utf8"), _cr, 1, dklen=32).hex()
+    else:
+        # Currently assume one extra key in dict, which is password. Print error if 
+        # more extra keys detected.
+        new_keys = [k for k in data.keys() if k not in ["context", "plugins"]]
+        l = len(new_keys)
+        if l == 0:
+            return None
+        elif l == 1 and isinstance(data[new_keys[0]], str):
+            password_key = new_keys[0]
+        else:
+            msg = "Yahoo has again changed data format, yfinance now unsure which key(s) is for decryption:"
+            k = new_keys[0]
+            k_str = k if len(k) < 32 else k[:32-3]+"..."
+            msg += f" '{k_str}'->{type(data[k])}"
+            for i in range(1, len(new_keys)):
+                msg += f" , '{k_str}'->{type(data[k])}"
+            raise Exception(msg)
+        password_key = new_keys[0]
+        password = data[password_key]
 
     encrypted_stores = b64decode(encrypted_stores)
     assert encrypted_stores[0:8] == b"Salted__"
@@ -98,7 +118,10 @@ def decrypt_cryptojs_aes(data):
         key, iv = key_iv[:keySize], key_iv[keySize:final_length]
         return key, iv
 
-    key, iv = EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5")
+    try:
+        key, iv = EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5")
+    except:
+        raise Exception("yfinance failed to decrypt Yahoo data response")
 
     if usePycryptodome:
         cipher = AES.new(key, AES.MODE_CBC, iv=iv)
@@ -176,15 +199,16 @@ class TickerData:
 
         data = json.loads(json_str)
 
-        if "_cs" in data and "_cr" in data:
-            data = decrypt_cryptojs_aes(data)
-
-        if "context" in data and "dispatcher" in data["context"]:
-            # Keep old code, just in case
-            data = data['context']['dispatcher']['stores']
+        stores = decrypt_cryptojs_aes_stores(data)
+        if stores is None:
+            # Maybe Yahoo returned old format, not encrypted
+            if "context" in data and "dispatcher" in data["context"]:
+                stores = data['context']['dispatcher']['stores']
+        if stores is None:
+            raise Exception(f"{self.ticker}: Failed to extract data stores from web request")
 
         # return data
-        new_data = json.dumps(data).replace('{}', 'null')
+        new_data = json.dumps(stores).replace('{}', 'null')
         new_data = re.sub(
             r'{[\'|\"]raw[\'|\"]:(.*?),(.*?)}', r'\1', new_data)
 

yfinance/multi.py

@@ -29,7 +29,7 @@ from . import Ticker, utils
 from . import shared
 
 
-def download(tickers, start=None, end=None, actions=False, threads=True, ignore_tz=True,
+def download(tickers, start=None, end=None, actions=False, threads=True, ignore_tz=False,
              group_by='column', auto_adjust=False, back_adjust=False, repair=False, keepna=False,
              progress=True, period="max", show_errors=True, interval="1d", prepost=False,
              proxy=None, rounding=False, timeout=10):
@@ -68,7 +68,7 @@ def download(tickers, start=None, end=None, actions=False, threads=True, ignore_
             How many threads to use for mass downloading. Default is True
         ignore_tz: bool
             When combining from different timezones, ignore that part of datetime.
-            Default is True
+            Default is False
         proxy: str
             Optional. Proxy server URL scheme. Default is None
         rounding: bool

yfinance/scrapers/fundamentals.py

@@ -195,7 +195,7 @@ class Financials:
         url = ts_url_base + "&type=" + ",".join([timescale + k for k in keys])
         # Yahoo returns maximum 4 years or 5 quarters, regardless of start_dt:
         start_dt = datetime.datetime(2016, 12, 31)
-        end = (datetime.datetime.now() + datetime.timedelta(days=366))
+        end = pd.Timestamp.utcnow().ceil("D")
         url += "&period1={}&period2={}".format(int(start_dt.timestamp()), int(end.timestamp()))
 
         # Step 3: fetch and reshape data

yfinance/scrapers/quote.py

@@ -194,9 +194,11 @@ class Quote:
             for k in keys:
                 url += "&type=" + k
             # Request 6 months of data
-            url += "&period1={}".format(
-                int((datetime.datetime.now() - datetime.timedelta(days=365 // 2)).timestamp()))
-            url += "&period2={}".format(int((datetime.datetime.now() + datetime.timedelta(days=1)).timestamp()))
+            start = pd.Timestamp.utcnow().floor("D") - datetime.timedelta(days=365 // 2)
+            start = int(start.timestamp())
+            end = pd.Timestamp.utcnow().ceil("D")
+            end = int(end.timestamp())
+            url += f"&period1={start}&period2={end}"
 
             json_str = self._data.cache_get(url=url, proxy=proxy).text
             json_data = json.loads(json_str)

yfinance/ticker.py

@@ -153,14 +153,6 @@ class Ticker(TickerBase):
     def quarterly_earnings(self) -> _pd.DataFrame:
         return self.get_earnings(freq='quarterly')
 
-    @property
-    def financials(self) -> _pd.DataFrame:
-        return self.income_stmt
-
-    @property
-    def quarterly_financials(self) -> _pd.DataFrame:
-        return self.quarterly_income_stmt
-
     @property
     def income_stmt(self) -> _pd.DataFrame:
         return self.get_income_stmt(pretty=True)
@@ -169,6 +161,22 @@ class Ticker(TickerBase):
     def quarterly_income_stmt(self) -> _pd.DataFrame:
         return self.get_income_stmt(pretty=True, freq='quarterly')
 
+    @property
+    def incomestmt(self) -> _pd.DataFrame:
+        return self.income_stmt
+
+    @property
+    def quarterly_incomestmt(self) -> _pd.DataFrame:
+        return self.quarterly_income_stmt
+
+    @property
+    def financials(self) -> _pd.DataFrame:
+        return self.income_stmt
+
+    @property
+    def quarterly_financials(self) -> _pd.DataFrame:
+        return self.quarterly_income_stmt
+
     @property
     def balance_sheet(self) -> _pd.DataFrame:
         return self.get_balance_sheet(pretty=True)
@@ -185,13 +193,21 @@ class Ticker(TickerBase):
     def quarterly_balancesheet(self) -> _pd.DataFrame:
         return self.quarterly_balance_sheet
 
+    @property
+    def cash_flow(self) -> _pd.DataFrame:
+        return self.get_cash_flow(pretty=True, freq="yearly")
+
+    @property
+    def quarterly_cash_flow(self) -> _pd.DataFrame:
+        return self.get_cash_flow(pretty=True, freq='quarterly')
+
     @property
     def cashflow(self) -> _pd.DataFrame:
-        return self.get_cashflow(pretty=True, freq="yearly")
+        return self.cash_flow
 
     @property
     def quarterly_cashflow(self) -> _pd.DataFrame:
-        return self.get_cashflow(pretty=True, freq='quarterly')
+        return self.quarterly_cash_flow
 
     @property
     def recommendations_summary(self):

yfinance/utils.py

@@ -607,7 +607,7 @@ def safe_merge_dfs(df_main, df_sub, interval):
         if interval.endswith('m') or interval.endswith('h') or interval == "1d":
             # Update: is possible with daily data when dividend very recent
             f_missing = ~df_sub.index.isin(df.index)
-            df_sub_missing = df_sub[f_missing]
+            df_sub_missing = df_sub[f_missing].copy()
             keys = {"Adj Open", "Open", "Adj High", "High", "Adj Low", "Low", "Adj Close",
                     "Close"}.intersection(df.columns)
             df_sub_missing[list(keys)] = _np.nan
@@ -743,8 +743,10 @@ class _TzCache:
     """Simple sqlite file cache of ticker->timezone"""
 
     def __init__(self):
-        self._tz_db = None
         self._setup_cache_folder()
+        # Must init db here, where is thread-safe
+        self._tz_db = _KVStore(_os.path.join(self._db_dir, "tkr-tz.db"))
+        self._migrate_cache_tkr_tz()
 
     def _setup_cache_folder(self):
         if not _os.path.isdir(self._db_dir):
@@ -776,11 +778,6 @@ class _TzCache:
 
     @property
     def tz_db(self):
-        # lazy init
-        if self._tz_db is None:
-            self._tz_db = _KVStore(_os.path.join(self._db_dir, "tkr-tz.db"))
-            self._migrate_cache_tkr_tz()
-
         return self._tz_db
 
     def _migrate_cache_tkr_tz(self):

yfinance/version.py

@@ -1 +1 @@
-version = "0.2.2"
+version = "0.2.4"