Bump version to 0.2.6

Fix 'Ticker.basic_info' lazy-loading

.github/ISSUE_TEMPLATE/bug_report.md

@@ -23,20 +23,20 @@ and comparing against [PIP](https://pypi.org/project/yfinance/#history).
 
 ### Does Yahoo actually have the data?
 
-Visit `finance.yahoo.com` and confim they have your data. Maybe your ticker was delisted.
+Are spelling ticker *exactly* same as Yahoo?
 
-Then check that you are spelling ticker *exactly* same as Yahoo.
+Visit `finance.yahoo.com` and confim they have your data. Maybe your ticker was delisted.
 
 ### Are you spamming Yahoo?
 
-Yahoo Finance free service has limit on query rate (roughly 100/s). Them delaying or blocking your spam is not a bug.
+Yahoo Finance free service has limit on query rate dependent on request - roughly 500/minute for prices, 10/minute for info. Them delaying or blocking your spam is not a bug.
 
 ### Still think it's a bug?
 
 Delete this default message and submit your bug report here, providing the following as best you can:
 
+- Simple code that reproduces your problem
+- Error message, with traceback if shown
 - Info about your system:
   - yfinance version
   - operating system
-- Simple code that reproduces your problem
-- The error message

CHANGELOG.rst

@@ -1,6 +1,23 @@
 Change Log
 ===========
 
+0.2.6
+-----
+- Fix Ticker.basic_info lazy-loading #1342
+
+0.2.5
+-----
+- Fix Yahoo data decryption again #1336
+- New: Ticker.basic_info - faster Ticker.info #1317
+
+0.2.4
+-----
+- Fix Yahoo data decryption #1297
+- New feature: 'Ticker.get_shares_full()' #1301
+- Improve caching of financials data #1284
+- Restore download() original alignment behaviour #1283
+- Fix the database lock error in multithread download #1276
+
 0.2.3
 -----
 - Make financials API '_' use consistent

README.md

@@ -60,7 +60,9 @@ import yfinance as yf
 
 msft = yf.Ticker("MSFT")
 
-# get stock info
+# fast access to subset of stock info
+msft.basic_info
+# slow access to all stock info
 msft.info
 
 # get historical market data
@@ -84,6 +86,7 @@ msft.capital_gains
 
 # show share count
 msft.shares
+msft.get_shares_full()
 
 # show financials:
 # - income statement
@@ -213,8 +216,7 @@ data = yf.download(  # or pdr.get_data_yahoo(...
         interval = "5d",
 
         # Whether to ignore timezone when aligning ticker data from 
-        # different timezones. Default is True. False may be useful for 
-        # minute/hourly data.
+        # different timezones. Default is False.
         ignore_tz = False,
 
         # group by ticker (to access via data['SPY'])

meta.yaml

@@ -1,5 +1,5 @@
 {% set name = "yfinance" %}
-{% set version = "0.2.3" %}
+{% set version = "0.2.6" %}
 
 package:
   name: "{{ name|lower }}"

tests/ticker.py

@@ -9,6 +9,7 @@ Specific test class:
 
 """
 import pandas as pd
+import numpy as np
 
 from .context import yfinance as yf
 
@@ -65,6 +66,7 @@ class TestTicker(unittest.TestCase):
         dat.splits
         dat.actions
         dat.shares
+        dat.get_shares_full()
         dat.info
         dat.calendar
         dat.recommendations
@@ -100,6 +102,7 @@ class TestTicker(unittest.TestCase):
         dat.splits
         dat.actions
         dat.shares
+        dat.get_shares_full()
         dat.info
         dat.calendar
         dat.recommendations
@@ -653,14 +656,121 @@ class TestTickerMiscFinancials(unittest.TestCase):
         self.assertIsInstance(data, pd.DataFrame, "data has wrong type")
         self.assertFalse(data.empty, "data is empty")
 
+    def test_shares_full(self):
+        data = self.ticker.get_shares_full()
+        self.assertIsInstance(data, pd.Series, "data has wrong type")
+        self.assertFalse(data.empty, "data is empty")
+
+    def test_bad_freq_value_raises_exception(self):
+        self.assertRaises(ValueError, lambda: self.ticker.get_cashflow(freq="badarg"))
+
+
+class TestTickerInfo(unittest.TestCase):
+    session = None
+
+    @classmethod
+    def setUpClass(cls):
+        cls.session = requests_cache.CachedSession(backend='memory')
+
+    @classmethod
+    def tearDownClass(cls):
+        if cls.session is not None:
+            cls.session.close()
+
+    def setUp(self):
+        tkrs = ["ESLT.TA", "BP.L", "GOOGL"]
+        self.tickers = [yf.Ticker(tkr, session=self.session) for tkr in tkrs]
+
+    def tearDown(self):
+        self.ticker = None
+
     def test_info(self):
         data = self.ticker.info
         self.assertIsInstance(data, dict, "data has wrong type")
         self.assertIn("symbol", data.keys(), "Did not find expected key in info dict")
         self.assertEqual("GOOGL", data["symbol"], "Wrong symbol value in info dict")
 
-    def test_bad_freq_value_raises_exception(self):
-        self.assertRaises(ValueError, lambda: self.ticker.get_cashflow(freq="badarg"))
+    def test_basic_info(self):
+        yf.scrapers.quote.PRUNE_INFO = False
+
+        # basic_info_keys = self.ticker.basic_info.keys()
+        basic_info_keys = set()
+        for ticker in self.tickers:
+            basic_info_keys.update(set(ticker.basic_info.keys()))
+        basic_info_keys = sorted(list(basic_info_keys))
+
+        key_rename_map = {}
+        key_rename_map["last_price"] = ["currentPrice", "regularMarketPrice"]
+        key_rename_map["open"] = ["open", "regularMarketOpen"]
+        key_rename_map["day_high"] = ["dayHigh", "regularMarketDayHigh"]
+        key_rename_map["day_low"] = ["dayLow", "regularMarketDayLow"]
+        key_rename_map["previous_close"] = ["previousClose", "regularMarketPreviousClose"]
+
+        # preMarketPrice
+
+        key_rename_map["fifty_day_average"] = "fiftyDayAverage"
+        key_rename_map["two_hundred_day_average"] = "twoHundredDayAverage"
+        key_rename_map["year_change"] = "52WeekChange"
+        key_rename_map["year_high"] = "fiftyTwoWeekHigh"
+        key_rename_map["year_low"] = "fiftyTwoWeekLow"
+
+        key_rename_map["last_volume"] = ["volume", "regularMarketVolume"]
+        key_rename_map["ten_day_average_volume"] = ["averageVolume10days", "averageDailyVolume10Day"]
+        key_rename_map["three_month_average_volume"] = "averageVolume"
+
+        key_rename_map["market_cap"] = "marketCap"
+        key_rename_map["shares"] = "floatShares"
+        key_rename_map["timezone"] = "exchangeTimezoneName"
+
+        approximate_keys = {"fifty_day_average", "ten_day_average_volume"}
+        approximate_keys.update({"market_cap"})
+
+        # bad_keys = []
+        bad_keys = {"shares"}
+
+        # Loose tolerance for averages, no idea why don't match info[]. Is info wrong?
+        custom_tolerances = {}
+        # custom_tolerances["ten_day_average_volume"] = 1e-3
+        custom_tolerances["ten_day_average_volume"] = 1e-1
+        # custom_tolerances["three_month_average_volume"] = 1e-2
+        custom_tolerances["three_month_average_volume"] = 5e-1
+        custom_tolerances["fifty_day_average"] = 1e-2
+        custom_tolerances["two_hundred_day_average"] = 1e-2
+
+        for k in basic_info_keys:
+            if k in key_rename_map:
+                k2 = key_rename_map[k]
+            else:
+                k2 = k
+
+            if not isinstance(k2, list):
+                k2 = [k2]
+
+            for m in k2:
+                for ticker in self.tickers:
+                    if not m in ticker.info:
+                        print(sorted(list(ticker.info.keys())))
+                        raise Exception("Need to add/fix mapping for basic_info key", k)
+
+                    if k in bad_keys:
+                        # Doesn't match, investigate why
+                        continue
+
+                    if k in custom_tolerances:
+                        rtol = custom_tolerances[k]
+                    else:
+                        rtol = 5e-3
+                        # rtol = 1e-4
+
+                    print(f"Testing key {m} -> {k} ticker={ticker.ticker}")
+                    # if k in approximate_keys:
+                    v1 = ticker.basic_info[k]
+                    v2 = ticker.info[m]
+                    if isinstance(v1, float) or isinstance(v2, int):
+                        self.assertTrue(np.isclose(v1, v2, rtol=rtol), f"{k}: {v1} != {v2}")
+                    else:
+                        self.assertEqual(v1, v2, f"{k}: {v1} != {v2}")
+
 
 
 def suite():

yfinance/base.py

@@ -40,12 +40,336 @@ from .scrapers.analysis import Analysis
 from .scrapers.fundamentals import Fundamentals
 from .scrapers.holders import Holders
 from .scrapers.quote import Quote
+import json as _json
 
 _BASE_URL_ = 'https://query2.finance.yahoo.com'
 _SCRAPE_URL_ = 'https://finance.yahoo.com/quote'
 _ROOT_URL_ = 'https://finance.yahoo.com'
 
 
+class BasicInfo:
+    # Contain small subset of info[] items that can be fetched faster elsewhere.
+    # Imitates a dict.
+    def __init__(self, tickerBaseObject):
+        self._tkr = tickerBaseObject
+
+        self._prices_1y = None
+        self._md = None
+
+        self._currency = None
+        self._exchange = None
+        self._timezone = None
+
+        self._shares = None
+        self._mcap = None
+
+        self._open = None
+        self._day_high = None
+        self._day_low = None
+        self._last_price = None
+        self._last_volume = None
+
+        self._prev_close = None
+
+        self._50d_day_average = None
+        self._200d_day_average = None
+        self._year_high = None
+        self._year_low = None
+        self._year_change = None
+
+        self._10d_avg_vol = None
+        self._3mo_avg_vol = None
+
+    # dict imitation:
+    def keys(self):
+        # attrs = utils.attributes(self)
+        # return attrs.keys()
+        # utils.attributes is calling each method, bad!
+        # Have to hardcode
+        keys = ["currency", "exchange", "timezone"]
+        keys += ["shares", "market_cap"]
+        keys += ["last_price", "previous_close", "open", "day_high", "day_low"]
+        keys += ["last_volume"]
+        keys += ["fifty_day_average", "two_hundred_day_average", "ten_day_average_volume", "three_month_average_volume"]
+        keys += ["year_high", "year_low", "year_change"]
+        return keys
+    def items(self):
+        return [(k,self[k]) for k in self.keys()]
+    def __getitem__(self, k):
+        if not isinstance(k, str):
+            raise KeyError(f"key must be a string")
+        if not k in self.keys():
+            raise KeyError(f"'{k}' not valid key. Examine 'BasicInfo.keys()'")
+        return getattr(self, k)
+    def __contains__(self, k):
+        return k in self.keys()
+    def __iter__(self):
+        return iter(self.keys())
+
+    def __str__(self):
+        return "lazy-loading dict with keys = " + str(self.keys())
+    def __repr__(self):
+        return self.__str__()
+
+    def _get_1y_prices(self, fullDaysOnly=False):
+        if self._prices_1y is None:
+            self._prices_1y = self._tkr.history(period="380d", auto_adjust=False)
+            self._md = self._tkr.get_history_metadata()
+            try:
+                ctp = self._md["currentTradingPeriod"]
+                self._today_open = pd.to_datetime(ctp["regular"]["start"], unit='s', utc=True).tz_convert(self.timezone)
+                self._today_close = pd.to_datetime(ctp["regular"]["end"], unit='s', utc=True).tz_convert(self.timezone)
+                self._today_midnight = self._today_close.ceil("D")
+            except:
+                self._today_open = None
+                self._today_close = None
+                self._today_midnight = None
+                raise
+
+        if self._prices_1y.empty:
+            return self.self._prices_1y
+
+        dt1 = self._prices_1y.index[-1]
+        if fullDaysOnly and self._exchange_open_now():
+            # Exclude today
+            dt1 -= utils._interval_to_timedelta("1h")
+        dt0 = dt1 - utils._interval_to_timedelta("1y") + utils._interval_to_timedelta("1d")
+        return self._prices_1y.loc[dt0:dt1]
+
+    def _get_exchange_metadata(self):
+        if self._md is not None:
+            return self._md
+
+        self._get_1y_prices()
+        self._md = self._tkr.get_history_metadata()
+        return self._md
+
+    def _exchange_open_now(self):
+        t = pd.Timestamp.utcnow()
+        self._get_exchange_metadata()
+
+        # if self._today_open is None and self._today_close is None:
+        #     r = False
+        # else:
+        #     r = self._today_open <= t and t < self._today_close
+
+        # if self._today_midnight is None:
+        #     r = False
+        # elif self._today_midnight.date() > t.tz_convert(self.timezone).date():
+        #     r = False
+        # else:
+        #     r = t < self._today_midnight
+
+        last_day_cutoff = self._get_1y_prices().index[-1] + _datetime.timedelta(days=1)
+        last_day_cutoff += _datetime.timedelta(minutes=20)
+        r = t < last_day_cutoff
+
+        # print("_exchange_open_now() returning", r)
+        return r
+
+    @property
+    def currency(self):
+        if self._currency is not None:
+            return self._currency
+
+        if self._tkr._history_metadata is None:
+            self._get_1y_prices()
+        md = self._tkr.get_history_metadata()
+        self._currency = md["currency"]
+        return self._currency
+
+    def _currency_is_cents(self):
+        return self.currency in ["GBp", "ILA"]
+
+    @property
+    def exchange(self):
+        if self._exchange is not None:
+            return self._exchange
+
+        self._exchange = self._get_exchange_metadata()["exchangeName"]
+        return self._exchange
+
+    @property
+    def timezone(self):
+        if self._timezone is not None:
+            return self._timezone
+
+        self._timezone = self._get_exchange_metadata()["exchangeTimezoneName"]
+        return self._timezone
+
+    @property
+    def shares(self):
+        if self._shares is not None:
+            return self._shares
+
+        shares = self._tkr.get_shares_full(start=pd.Timestamp.utcnow().date()-pd.Timedelta(days=548))
+        if shares is None:
+            # Requesting 18 months failed, so fallback to shares which should include last year
+            shares = self._tkr.get_shares()
+        if shares is None:
+            raise Exception(f"{self._tkr.ticker}: Cannot retrieve share count")
+        if isinstance(shares, pd.DataFrame):
+            shares = shares[shares.columns[0]]
+        self._shares = shares.iloc[-1]
+        return self._shares
+
+    @property
+    def last_price(self):
+        if self._last_price is not None:
+            return self._last_price
+        # self._last_price = self._get_exchange_metadata()["regularMarketPrice"]
+        prices = self._get_1y_prices()
+        self._last_price = _np.nan if prices.empty else prices["Close"].iloc[-1]
+        return self._last_price
+
+    @property
+    def previous_close(self):
+        if self._prev_close is not None:
+            return self._prev_close
+        prices = self._get_1y_prices()
+        self._prev_close = _np.nan if prices.empty else prices["Close"].iloc[-2]
+        return self._prev_close
+
+    @property
+    def open(self):
+        if self._open is not None:
+            return self._open
+        prices = self._get_1y_prices()
+        self._open = _np.nan if prices.empty else prices["Open"].iloc[-1]
+        return self._open
+
+    @property
+    def day_high(self):
+        if self._day_high is not None:
+            return self._day_high
+        prices = self._get_1y_prices()
+        self._day_high = _np.nan if prices.empty else prices["High"].iloc[-1]
+        return self._day_high
+
+    @property
+    def day_low(self):
+        if self._day_low is not None:
+            return self._day_low
+        prices = self._get_1y_prices()
+        self._day_low = _np.nan if prices.empty else prices["Low"].iloc[-1]
+        return self._day_low
+
+    @property
+    def last_volume(self):
+        if self._last_volume is not None:
+            return self._last_volume
+        prices = self._get_1y_prices()
+        self._last_volume = 0 if prices.empty else prices["Volume"].iloc[-1]
+        return self._last_volume
+
+    @property
+    def fifty_day_average(self):
+        if self._50d_day_average is not None:
+            return self._50d_day_average
+
+        prices = self._get_1y_prices(fullDaysOnly=True)
+        if prices.empty:
+            self._50d_day_average = _np.nan
+        else:
+            n = prices.shape[0]
+            a = n-50
+            b = n
+            if a < 0:
+                a = 0
+            self._50d_day_average = prices["Close"].iloc[a:b].mean()
+
+        return self._50d_day_average
+
+    @property
+    def two_hundred_day_average(self):
+        if self._200d_day_average is not None:
+            return self._200d_day_average
+
+        prices = self._get_1y_prices(fullDaysOnly=True)
+        if prices.empty:
+            self._200d_day_average = _np.nan
+        else:
+            n = prices.shape[0]
+            a = n-200
+            b = n
+            if a < 0:
+                a = 0
+
+            self._200d_day_average = prices["Close"].iloc[a:b].mean()
+
+        return self._200d_day_average
+
+    @property
+    def ten_day_average_volume(self):
+        if self._10d_avg_vol is not None:
+            return self._10d_avg_vol
+
+        prices = self._get_1y_prices(fullDaysOnly=True)
+        if prices.empty:
+            self._10d_avg_vol = 0
+        else:
+            n = prices.shape[0]
+            a = n-10
+            b = n
+            if a < 0:
+                a = 0
+            self._10d_avg_vol = prices["Volume"].iloc[a:b].mean()
+
+        return self._10d_avg_vol
+
+    @property
+    def three_month_average_volume(self):
+        if self._3mo_avg_vol is not None:
+            return self._3mo_avg_vol
+
+        prices = self._get_1y_prices(fullDaysOnly=True)
+        if prices.empty:
+            self._3mo_avg_vol = 0
+        else:
+            dt1 = prices.index[-1]
+            dt0 = dt1 - utils._interval_to_timedelta("3mo") + utils._interval_to_timedelta("1d")
+            self._3mo_avg_vol = prices.loc[dt0:dt1, "Volume"].mean()
+
+        return self._3mo_avg_vol
+
+    @property
+    def year_high(self):
+        if self._year_high is not None:
+            return self._year_high
+
+        prices = self._get_1y_prices(fullDaysOnly=True)
+        self._year_high = prices["High"].max()
+        return self._year_high
+
+    @property
+    def year_low(self):
+        if self._year_low is not None:
+            return self._year_low
+
+        prices = self._get_1y_prices(fullDaysOnly=True)
+        self._year_low = prices["Low"].min()
+        return self._year_low
+
+    @property
+    def year_change(self):
+        if self._year_change is not None:
+            return self._year_change
+
+        prices = self._get_1y_prices(fullDaysOnly=True)
+        self._year_change = (prices["Close"].iloc[-1] - prices["Close"].iloc[0]) / prices["Close"].iloc[0]
+        return self._year_change
+
+    @property
+    def market_cap(self):
+        if self._mcap is not None:
+            return self._mcap
+
+        self._mcap = self.shares * self.last_price
+        if self._currency_is_cents():
+            self._mcap *= 0.01
+        return self._mcap
+
+
 class TickerBase:
     def __init__(self, ticker, session=None):
         self.ticker = ticker.upper()
@@ -76,6 +400,8 @@ class TickerBase:
         self._quote = Quote(self._data)
         self._fundamentals = Fundamentals(self._data)
 
+        self._basic_info = BasicInfo(self)
+
     def stats(self, proxy=None):
         ticker_url = "{}/{}".format(self._scrape_url, self.ticker)
 
@@ -894,6 +1220,10 @@ class TickerBase:
         data = self._quote.info
         return data
 
+    @property
+    def basic_info(self):
+        return self._basic_info
+
     def get_sustainability(self, proxy=None, as_dict=False):
         self._quote.proxy = proxy
         data = self._quote.sustainability
@@ -1118,6 +1448,59 @@ class TickerBase:
             return data.to_dict()
         return data
 
+    def get_shares_full(self, start=None, end=None, proxy=None):
+        # Process dates
+        tz = self._get_ticker_tz(debug_mode=False, proxy=None, timeout=10)
+        dt_now = _pd.Timestamp.utcnow().tz_convert(tz)
+        if start is not None:
+            start_ts = utils._parse_user_dt(start, tz)
+            start = _pd.Timestamp.fromtimestamp(start_ts).tz_localize("UTC").tz_convert(tz)
+            start_d = start.date()
+        if end is not None:
+            end_ts = utils._parse_user_dt(end, tz)
+            end = _pd.Timestamp.fromtimestamp(end_ts).tz_localize("UTC").tz_convert(tz)
+            end_d = end.date()
+        if end is None:
+            end = dt_now
+        if start is None:
+            start = end - _pd.Timedelta(days=548)  # 18 months
+        if start >= end:
+            print("ERROR: start date must be before end")
+            return None
+        start = start.floor("D")
+        end = end.ceil("D")
+        
+        # Fetch
+        ts_url_base = "https://query2.finance.yahoo.com/ws/fundamentals-timeseries/v1/finance/timeseries/{0}?symbol={0}".format(self.ticker)
+        shares_url = ts_url_base + "&period1={}&period2={}".format(int(start.timestamp()), int(end.timestamp()))
+        try:
+            json_str = self._data.cache_get(shares_url).text
+            json_data = _json.loads(json_str)
+        except:
+            print(f"{self.ticker}: Yahoo web request for share count failed")
+            return None
+        try:
+            fail = json_data["finance"]["error"]["code"] == "Bad Request"
+        except:
+            fail = False
+        if fail:
+            print(f"{self.ticker}: Yahoo web request for share count failed")
+            return None
+
+        shares_data = json_data["timeseries"]["result"]
+        if not "shares_out" in shares_data[0]:
+            print(f"{self.ticker}: Yahoo did not return share count in date range {start} -> {end}")
+            return None
+        try:
+            df = _pd.Series(shares_data[0]["shares_out"], index=_pd.to_datetime(shares_data[0]["timestamp"], unit="s"))
+        except Exception as e:
+            print(f"{self.ticker}: Failed to parse shares count data: "+str(e))
+            return None
+
+        df.index = df.index.tz_localize(tz)
+        df = df.sort_index()
+        return df
+
     def get_isin(self, proxy=None) -> Optional[str]:
         # *** experimental ***
         if self._isin is not None:
@@ -1254,8 +1637,8 @@ class TickerBase:
         dates[cn] = _pd.to_datetime(dates[cn], format="%b %d, %Y, %I %p")
         # - instead of attempting decoding of ambiguous timezone abbreviation, just use 'info':
         self._quote.proxy = proxy
-        dates[cn] = dates[cn].dt.tz_localize(
-            tz=self._quote.info["exchangeTimezoneName"])
+        tz = self._get_ticker_tz(debug_mode=False, proxy=proxy, timeout=30)
+        dates[cn] = dates[cn].dt.tz_localize(tz)
 
         dates = dates.set_index("Earnings Date")
 

yfinance/data.py

@@ -46,20 +46,74 @@ def lru_cache_freezeargs(func):
     return wrapped
 
 
-def decrypt_cryptojs_aes(data):
+def decrypt_cryptojs_aes_stores(data):
     encrypted_stores = data['context']['dispatcher']['stores']
-    _cs = data["_cs"]
-    _cr = data["_cr"]
 
-    _cr = b"".join(int.to_bytes(i, length=4, byteorder="big", signed=True) for i in json.loads(_cr)["words"])
-    password = hashlib.pbkdf2_hmac("sha1", _cs.encode("utf8"), _cr, 1, dklen=32).hex()
+    password = None
+    candidate_passwords = []
+    if "_cs" in data and "_cr" in data:
+        _cs = data["_cs"]
+        _cr = data["_cr"]
+        _cr = b"".join(int.to_bytes(i, length=4, byteorder="big", signed=True) for i in json.loads(_cr)["words"])
+        password = hashlib.pbkdf2_hmac("sha1", _cs.encode("utf8"), _cr, 1, dklen=32).hex()
+    else:
+        # Currently assume one extra key in dict, which is password. Print error if 
+        # more extra keys detected.
+        new_keys = [k for k in data.keys() if k not in ["context", "plugins"]]
+        new_keys_values = set([data[k] for k in new_keys])
+
+        # Maybe multiple keys have same value - keep one of each
+        new_keys2 = []
+        new_keys2_values = set()
+        for k in new_keys:
+            v = data[k]
+            if not v in new_keys2_values:
+                new_keys2.append(k)
+                new_keys2_values.add(v)
+
+        l = len(new_keys)
+        if l == 0:
+            return None
+        elif l == 1 and isinstance(data[new_keys[0]], str):
+            password_key = new_keys[0]
+        # else:
+        #     msg = "Yahoo has again changed data format, yfinance now unsure which key(s) is for decryption:"
+        #     new_keys_pretty = {}
+        #     l = min(10, len(new_keys))
+        #     for i in range(0, l):
+        #         k = new_keys[i]
+        #         k_str = k if len(k) < 32 else k[:32-3]+"..."
+        #         v = data[k]
+        #         v_type = type(v)
+        #         v_str = str(v)
+        #         if len(v_str) > 256:
+        #             v_str = v_str[:256]+"..."
+        #         new_keys_pretty[k_str] = f"{v_str}' ({v_type})"
+        #     for k in new_keys_pretty:
+        #         msg += '\n' + f"'{k}' -> '{new_keys_pretty[k]}'"
+        #     if len(new_keys) > l:
+        #         d = len(new_keys) - l
+        #         msg += '\n' + "..."
+        #         msg += '\n' + f"{d} more options!"
+        #     raise Exception(msg)
+        # password_key = new_keys[0]
+        # password = data[password_key]
+
+        # The above attempt to smartly pick out decryption key has stopped working.
+        # Fortunately the keys Yahoo use are currently hardcoded in their JSON:
+        candidate_passwords += ["ad4d90b3c9f2e1d156ef98eadfa0ff93e4042f6960e54aa2a13f06f528e6b50ba4265a26a1fd5b9cd3db0d268a9c34e1d080592424309429a58bce4adc893c87", \
+            "e9a8ab8e5620b712ebc2fb4f33d5c8b9c80c0d07e8c371911c785cf674789f1747d76a909510158a7b7419e86857f2d7abbd777813ff64840e4cbc514d12bcae", 
+            "6ae2523aeafa283dad746556540145bf603f44edbf37ad404d3766a8420bb5eb1d3738f52a227b88283cca9cae44060d5f0bba84b6a495082589f5fe7acbdc9e",
+            "3365117c2a368ffa5df7313a4a84988f73926a86358e8eea9497c5ff799ce27d104b68e5f2fbffa6f8f92c1fef41765a7066fa6bcf050810a9c4c7872fd3ebf0"]
+
+        # candidate_passwords += [data[k] for k in new_keys]  # don't do these, none work
 
     encrypted_stores = b64decode(encrypted_stores)
     assert encrypted_stores[0:8] == b"Salted__"
     salt = encrypted_stores[8:16]
     encrypted_stores = encrypted_stores[16:]
 
-    def EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") -> tuple:
+    def _EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") -> tuple:
         """OpenSSL EVP Key Derivation Function
         Args:
             password (Union[str, bytes, bytearray]): Password to generate key from.
@@ -98,19 +152,42 @@ def decrypt_cryptojs_aes(data):
         key, iv = key_iv[:keySize], key_iv[keySize:final_length]
         return key, iv
 
-    key, iv = EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5")
+    def _decrypt(encrypted_stores, password, key, iv):
+        if usePycryptodome:
+            cipher = AES.new(key, AES.MODE_CBC, iv=iv)
+            plaintext = cipher.decrypt(encrypted_stores)
+            plaintext = unpad(plaintext, 16, style="pkcs7")
+        else:
+            cipher = Cipher(algorithms.AES(key), modes.CBC(iv))
+            decryptor = cipher.decryptor()
+            plaintext = decryptor.update(encrypted_stores) + decryptor.finalize()
+            unpadder = padding.PKCS7(128).unpadder()
+            plaintext = unpadder.update(plaintext) + unpadder.finalize()
+            plaintext = plaintext.decode("utf-8")
+        return plaintext
 
-    if usePycryptodome:
-        cipher = AES.new(key, AES.MODE_CBC, iv=iv)
-        plaintext = cipher.decrypt(encrypted_stores)
-        plaintext = unpad(plaintext, 16, style="pkcs7")
+    if not password is None:
+        try:
+            key, iv = _EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5")
+        except:
+            raise Exception("yfinance failed to decrypt Yahoo data response")
+        plaintext = _decrypt(encrypted_stores, password, key, iv)
     else:
-        cipher = Cipher(algorithms.AES(key), modes.CBC(iv))
-        decryptor = cipher.decryptor()
-        plaintext = decryptor.update(encrypted_stores) + decryptor.finalize()
-        unpadder = padding.PKCS7(128).unpadder()
-        plaintext = unpadder.update(plaintext) + unpadder.finalize()
-        plaintext = plaintext.decode("utf-8")
+        success = False
+        for i in range(len(candidate_passwords)):
+            # print(f"Trying candiate pw {i+1}/{len(candidate_passwords)}")
+            password = candidate_passwords[i]
+            try:
+                key, iv = _EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5")
+
+                plaintext = _decrypt(encrypted_stores, password, key, iv)
+
+                success = True
+                break
+            except:
+                pass
+        if not success:
+            raise Exception("yfinance failed to decrypt Yahoo data response with hardcoded keys, contact developers")
 
     decoded_stores = json.loads(plaintext)
     return decoded_stores
@@ -176,15 +253,16 @@ class TickerData:
 
         data = json.loads(json_str)
 
-        if "_cs" in data and "_cr" in data:
-            data = decrypt_cryptojs_aes(data)
-
-        if "context" in data and "dispatcher" in data["context"]:
-            # Keep old code, just in case
-            data = data['context']['dispatcher']['stores']
+        stores = decrypt_cryptojs_aes_stores(data)
+        if stores is None:
+            # Maybe Yahoo returned old format, not encrypted
+            if "context" in data and "dispatcher" in data["context"]:
+                stores = data['context']['dispatcher']['stores']
+        if stores is None:
+            raise Exception(f"{self.ticker}: Failed to extract data stores from web request")
 
         # return data
-        new_data = json.dumps(data).replace('{}', 'null')
+        new_data = json.dumps(stores).replace('{}', 'null')
         new_data = re.sub(
             r'{[\'|\"]raw[\'|\"]:(.*?),(.*?)}', r'\1', new_data)
 

yfinance/multi.py

@@ -29,7 +29,7 @@ from . import Ticker, utils
 from . import shared
 
 
-def download(tickers, start=None, end=None, actions=False, threads=True, ignore_tz=True,
+def download(tickers, start=None, end=None, actions=False, threads=True, ignore_tz=False,
              group_by='column', auto_adjust=False, back_adjust=False, repair=False, keepna=False,
              progress=True, period="max", show_errors=True, interval="1d", prepost=False,
              proxy=None, rounding=False, timeout=10):
@@ -68,7 +68,7 @@ def download(tickers, start=None, end=None, actions=False, threads=True, ignore_
             How many threads to use for mass downloading. Default is True
         ignore_tz: bool
             When combining from different timezones, ignore that part of datetime.
-            Default is True
+            Default is False
         proxy: str
             Optional. Proxy server URL scheme. Default is None
         rounding: bool

yfinance/scrapers/fundamentals.py

@@ -195,7 +195,7 @@ class Financials:
         url = ts_url_base + "&type=" + ",".join([timescale + k for k in keys])
         # Yahoo returns maximum 4 years or 5 quarters, regardless of start_dt:
         start_dt = datetime.datetime(2016, 12, 31)
-        end = (datetime.datetime.now() + datetime.timedelta(days=366))
+        end = pd.Timestamp.utcnow().ceil("D")
         url += "&period1={}&period2={}".format(int(start_dt.timestamp()), int(end.timestamp()))
 
         # Step 3: fetch and reshape data

yfinance/scrapers/quote.py

@@ -7,6 +7,75 @@ from yfinance import utils
 from yfinance.data import TickerData
 
 
+info_retired_keys_price = {"currentPrice", "dayHigh", "dayLow", "open", "previousClose", "volume"}
+info_retired_keys_price.update({"regularMarket"+s for s in ["DayHigh", "DayLow", "Open", "PreviousClose", "Price", "Volume"]})
+info_retired_keys_price.update({"fiftyTwoWeekLow", "fiftyTwoWeekHigh", "fiftyTwoWeekChange", "fiftyDayAverage", "twoHundredDayAverage"})
+info_retired_keys_price.update({"averageDailyVolume10Day", "averageVolume10days", "averageVolume"})
+info_retired_keys_exchange = {"currency", "exchange", "exchangeTimezoneName", "exchangeTimezoneShortName"}
+info_retired_keys_marketCap = {"marketCap"}
+info_retired_keys_symbol = {"symbol"}
+info_retired_keys = info_retired_keys_price | info_retired_keys_exchange | info_retired_keys_marketCap | info_retired_keys_symbol
+#
+info_retired_keys = []
+
+
+PRUNE_INFO = True
+# PRUNE_INFO = False
+
+
+from collections.abc import MutableMapping
+class InfoDictWrapper(MutableMapping):
+    """ Simple wrapper around info dict, intercepting 'gets' to 
+    print how-to-migrate messages for specific keys. Requires
+    override dict API"""
+
+    def __init__(self, info):
+        self.info = info
+
+    def keys(self):
+        return self.info.keys()
+
+    def __str__(self):
+        return self.info.__str__()
+
+    def __repr__(self):
+        return self.info.__repr__()
+
+    def __contains__(self, k):
+        return k in self.info.keys()
+
+    def __getitem__(self, k):
+        if k in info_retired_keys_price:
+            print(f"Price data removed from info. Use Ticker.basic_info or history() instead")
+            return None
+        elif k in info_retired_keys_exchange:
+            print(f"Exchange data removed from info. Use Ticker.basic_info or Ticker.get_history_metadata() instead")
+            return None
+        elif k in info_retired_keys_marketCap:
+            print(f"Market cap removed from info. Use Ticker.basic_info instead")
+            return None
+        elif k in info_retired_keys_symbol:
+            print(f"Symbol removed from info. You know this already")
+            return None
+        return self.info[self._keytransform(k)]
+
+    def __setitem__(self, k, value):
+        self.info[self._keytransform(k)] = value
+
+    def __delitem__(self, k):
+        del self.info[self._keytransform(k)]
+
+    def __iter__(self):
+        return iter(self.info)
+    
+    def __len__(self):
+        return len(self.info)
+
+    def _keytransform(self, k):
+        return k
+
+
+
 class Quote:
 
     def __init__(self, data: TickerData, proxy=None):
@@ -130,6 +199,15 @@ class Quote:
         except Exception:
             pass
 
+        # Delete redundant info[] keys, because values can be accessed faster
+        # elsewhere - e.g. price keys. Hope is reduces Yahoo spam effect.
+        if PRUNE_INFO:
+            for k in info_retired_keys:
+                if k in self._info:
+                    del self._info[k]
+            # InfoDictWrapper will explain how to access above data elsewhere
+            self._info = InfoDictWrapper(self._info)
+
         # events
         try:
             cal = pd.DataFrame(quote_summary_store['calendarEvents']['earnings'])
@@ -194,9 +272,11 @@ class Quote:
             for k in keys:
                 url += "&type=" + k
             # Request 6 months of data
-            url += "&period1={}".format(
-                int((datetime.datetime.now() - datetime.timedelta(days=365 // 2)).timestamp()))
-            url += "&period2={}".format(int((datetime.datetime.now() + datetime.timedelta(days=1)).timestamp()))
+            start = pd.Timestamp.utcnow().floor("D") - datetime.timedelta(days=365 // 2)
+            start = int(start.timestamp())
+            end = pd.Timestamp.utcnow().ceil("D")
+            end = int(end.timestamp())
+            url += f"&period1={start}&period2={end}"
 
             json_str = self._data.cache_get(url=url, proxy=proxy).text
             json_data = json.loads(json_str)

yfinance/ticker.py

@@ -133,6 +133,10 @@ class Ticker(TickerBase):
     def shares(self) -> _pd.DataFrame :
         return self.get_shares()
 
+    @property
+    def market_cap(self) -> float:
+        return self.calc_market_cap()
+
     @property
     def info(self) -> dict:
         return self.get_info()

yfinance/tickers.py

@@ -34,12 +34,8 @@ class Tickers:
         tickers = tickers if isinstance(
             tickers, list) else tickers.replace(',', ' ').split()
         self.symbols = [ticker.upper() for ticker in tickers]
-        ticker_objects = {}
+        self.tickers = {ticker:Ticker(ticker, session=session) for ticker in self.symbols}
 
-        for ticker in self.symbols:
-            ticker_objects[ticker] = Ticker(ticker, session=session)
-
-        self.tickers = ticker_objects
         # self.tickers = _namedtuple(
         #     "Tickers", ticker_objects.keys(), rename=True
         # )(*ticker_objects.values())

yfinance/utils.py

@@ -49,6 +49,18 @@ user_agent_headers = {
     'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
 
 
+# From https://stackoverflow.com/a/59128615
+from types import FunctionType
+from inspect import getmembers
+def attributes(obj):
+    disallowed_names = {
+      name for name, value in getmembers(type(obj)) 
+        if isinstance(value, FunctionType)}
+    return {
+      name: getattr(obj, name) for name in dir(obj) 
+        if name[0] != '_' and name not in disallowed_names and hasattr(obj, name)}
+
+
 def is_isin(string):
     return bool(_re.match("^([A-Z]{2})([A-Z0-9]{9})([0-9]{1})$", string))
 
@@ -307,7 +319,11 @@ def _parse_user_dt(dt, exchange_tz):
 
 def _interval_to_timedelta(interval):
     if interval == "1mo":
-        return _dateutil.relativedelta(months=1)
+        return _dateutil.relativedelta.relativedelta(months=1)
+    elif interval == "3mo":
+        return _dateutil.relativedelta.relativedelta(months=3)
+    elif interval == "1y":
+        return _dateutil.relativedelta.relativedelta(years=1)
     elif interval == "1wk":
         return _pd.Timedelta(days=7, unit='d')
     else: 
@@ -607,7 +623,7 @@ def safe_merge_dfs(df_main, df_sub, interval):
         if interval.endswith('m') or interval.endswith('h') or interval == "1d":
             # Update: is possible with daily data when dividend very recent
             f_missing = ~df_sub.index.isin(df.index)
-            df_sub_missing = df_sub[f_missing]
+            df_sub_missing = df_sub[f_missing].copy()
             keys = {"Adj Open", "Open", "Adj High", "High", "Adj Low", "Low", "Adj Close",
                     "Close"}.intersection(df.columns)
             df_sub_missing[list(keys)] = _np.nan
@@ -743,8 +759,10 @@ class _TzCache:
     """Simple sqlite file cache of ticker->timezone"""
 
     def __init__(self):
-        self._tz_db = None
         self._setup_cache_folder()
+        # Must init db here, where is thread-safe
+        self._tz_db = _KVStore(_os.path.join(self._db_dir, "tkr-tz.db"))
+        self._migrate_cache_tkr_tz()
 
     def _setup_cache_folder(self):
         if not _os.path.isdir(self._db_dir):
@@ -776,11 +794,6 @@ class _TzCache:
 
     @property
     def tz_db(self):
-        # lazy init
-        if self._tz_db is None:
-            self._tz_db = _KVStore(_os.path.join(self._db_dir, "tkr-tz.db"))
-            self._migrate_cache_tkr_tz()
-
         return self._tz_db
 
     def _migrate_cache_tkr_tz(self):

yfinance/version.py

@@ -1 +1 @@
-version = "0.2.3"
+version = "0.2.6"