Bump version to 0.1.95

Fix quotes html parsing when missing root.App.main

CHANGELOG.rst

@@ -1,6 +1,26 @@
 Change Log
 ===========
 
+0.1.95
+------
+- Fix info[] bug #1257
+
+0.1.94
+------
+- Fix delisted ticker info[]
+
+0.1.93
+------
+- Fix Ticker.shares
+
+0.1.92
+------
+- Decrypt new Yahoo encryption #1255
+
+0.1.90
+------
+- Restore lxml req, increase min ver #1237
+
 0.1.89
 ------
 - Remove unused incompatible dependency #1222

README.md

@@ -277,7 +277,9 @@ To install `yfinance` using `conda`, see
 -   [Pandas](https://github.com/pydata/pandas) \>= 1.3.0
 -   [Numpy](http://www.numpy.org) \>= 1.16.5
 -   [requests](http://docs.python-requests.org/en/master/) \>= 2.26
+-   [lxml](https://pypi.org/project/lxml/) \>= 4.9.1
 -   [appdirs](https://pypi.org/project/appdirs) \>= 1.4.4
+-   [cryptography](https://pypi.org/project/cryptography) \>=3.3.2
 
 ### Optional (if you want to use `pandas_datareader`)
 

meta.yaml

@@ -1,5 +1,5 @@
 {% set name = "yfinance" %}
-{% set version = "0.1.89" %}
+{% set version = "0.1.95" %}
 
 package:
   name: "{{ name|lower }}"
@@ -20,7 +20,9 @@ requirements:
     - numpy >=1.16.5
     - requests >=2.26
     - multitasking >=0.0.7
+    - lxml >=4.9.1
     - appdirs >= 1.4.4
+    - cryptography >= 3.3.2
     - pip
     - python
 
@@ -29,7 +31,9 @@ requirements:
     - numpy >=1.16.5
     - requests >=2.26
     - multitasking >=0.0.7
+    - lxml >=4.9.1
     - appdirs >= 1.4.4
+    - cryptography >= 3.3.2
     - python
 
 test:

requirements.txt

@@ -2,4 +2,6 @@ pandas>=1.3.0
 numpy>=1.16.5
 requests>=2.26
 multitasking>=0.0.7
+lxml>=4.9.1
 appdirs>=1.4.4
+cryptography>=3.3.2

setup.py

@@ -63,7 +63,8 @@ setup(
     packages=find_packages(exclude=['contrib', 'docs', 'tests', 'examples']),
     install_requires=['pandas>=1.3.0', 'numpy>=1.16.5',
                       'requests>=2.26', 'multitasking>=0.0.7',
-                      'appdirs>=1.4.4'],
+                      'lxml>=4.9.1', 'appdirs>=1.4.4',
+                      'cryptography>=3.3.2'],
     entry_points={
         'console_scripts': [
             'sample=sample:main',

test_yfinance.py

@@ -20,8 +20,15 @@ import datetime
 session = None
 import requests_cache ; session = requests_cache.CachedSession("yfinance.cache", expire_after=24*60*60)
 
-symbols = ['MSFT', 'IWO', 'VFINX', '^GSPC', 'BTC-USD']
-tickers = [yf.Ticker(symbol, session=session) for symbol in symbols]
+# Good symbols = all attributes should work
+good_symbols = ['MSFT', 'IWO', 'VFINX', '^GSPC', 'BTC-USD']
+good_tickers = [yf.Ticker(symbol, session=session) for symbol in good_symbols]
+# Dodgy symbols = Yahoo data incomplete, so exclude from some tests
+dodgy_symbols = ["G7W.DU"]
+dodgy_tickers = [yf.Ticker(symbol, session=session) for symbol in dodgy_symbols]
+symbols = good_symbols + dodgy_symbols
+tickers = good_tickers + dodgy_tickers
+# Delisted = no data expected but yfinance shouldn't raise exception
 delisted_symbols = ["BRK.B", "SDLP"]
 delisted_tickers = [yf.Ticker(symbol, session=session) for symbol in delisted_symbols]
 
@@ -118,8 +125,7 @@ class TestTicker(unittest.TestCase):
             ticker.earnings_dates
 
     def test_holders(self):
-        for ticker in tickers:
-            assert(ticker.info is not None and ticker.info != {})
+        for ticker in good_tickers:
             assert(ticker.major_holders is not None)
             assert(ticker.institutional_holders is not None)
 

yfinance/utils.py

@@ -31,6 +31,18 @@ import sys as _sys
 import os as _os
 import appdirs as _ad
 
+from base64 import b64decode
+import hashlib
+usePycryptodome = False  # slightly faster
+# usePycryptodome = True
+if usePycryptodome:
+    # NOTE: if decide to use 'pycryptodome', set min version to 3.6.6
+    from Crypto.Cipher import AES
+    from Crypto.Util.Padding import unpad
+else:
+    from cryptography.hazmat.primitives import padding
+    from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+
 from threading import Lock
 mutex = Lock()
 
@@ -109,24 +121,112 @@ def get_html(url, proxy=None, session=None):
     return html
 
 
+
+def decrypt_cryptojs_stores(data):
+    """
+    Yahoo has started encrypting data stores, this method decrypts it.
+    :param data: Python dict of the json data
+    :return: The decrypted string data in data['context']['dispatcher']['stores']
+    """
+
+    _cs = data["_cs"]
+    # Assumes _cr has format like: '{"words":[-449732894,601032952,157396918,2056341829],"sigBytes":16}';
+    _cr = _json.loads(data["_cr"])
+    _cr = b"".join(int.to_bytes(i, length=4, byteorder="big", signed=True) for i in _cr["words"])
+
+    password = hashlib.pbkdf2_hmac("sha1", _cs.encode("utf8"), _cr, 1, dklen=32).hex()
+
+    encrypted_stores = data['context']['dispatcher']['stores']
+    encrypted_stores = b64decode(encrypted_stores)
+    assert encrypted_stores[0:8] == b"Salted__"
+    salt = encrypted_stores[8:16]
+    encrypted_stores = encrypted_stores[16:]
+
+    key, iv = _EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5")
+
+    if usePycryptodome:
+        cipher = AES.new(key, AES.MODE_CBC, iv=iv)
+        plaintext = cipher.decrypt(encrypted_stores)
+        plaintext = unpad(plaintext, 16, style="pkcs7")
+    else:
+        cipher = Cipher(algorithms.AES(key), modes.CBC(iv))
+        decryptor = cipher.decryptor()
+        plaintext = decryptor.update(encrypted_stores) + decryptor.finalize()
+        unpadder = padding.PKCS7(128).unpadder()
+        plaintext = unpadder.update(plaintext) + unpadder.finalize()
+        plaintext = plaintext.decode("utf-8")
+
+    return plaintext
+
+def _EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") -> tuple:
+    """OpenSSL EVP Key Derivation Function
+    Args:
+        password (Union[str, bytes, bytearray]): Password to generate key from.
+        salt (Union[bytes, bytearray]): Salt to use.
+        keySize (int, optional): Output key length in bytes. Defaults to 32.
+        ivSize (int, optional): Output Initialization Vector (IV) length in bytes. Defaults to 16.
+        iterations (int, optional): Number of iterations to perform. Defaults to 1.
+        hashAlgorithm (str, optional): Hash algorithm to use for the KDF. Defaults to 'md5'.
+    Returns:
+        key, iv: Derived key and Initialization Vector (IV) bytes.
+
+    Taken from: https://gist.github.com/rafiibrahim8/0cd0f8c46896cafef6486cb1a50a16d3
+    OpenSSL original code: https://github.com/openssl/openssl/blob/master/crypto/evp/evp_key.c#L78
+    """
+
+    assert iterations > 0, "Iterations can not be less than 1."
+
+    if isinstance(password, str):
+        password = password.encode("utf-8")
+
+    final_length = keySize + ivSize
+    key_iv = b""
+    block = None
+
+    while len(key_iv) < final_length:
+        hasher = hashlib.new(hashAlgorithm)
+        if block:
+            hasher.update(block)
+        hasher.update(password)
+        hasher.update(salt)
+        block = hasher.digest()
+        for _ in range(1, iterations):
+            block = hashlib.new(hashAlgorithm, block).digest()
+        key_iv += block
+
+    key, iv = key_iv[:keySize], key_iv[keySize:final_length]
+    return key, iv
+
+
 def get_json(url, proxy=None, session=None):
     session = session or _requests
     html = session.get(url=url, proxies=proxy, headers=user_agent_headers).text
 
-    if "QuoteSummaryStore" not in html:
-        html = session.get(url=url, proxies=proxy).text
-        if "QuoteSummaryStore" not in html:
-            return {}
+    if not "root.App.main =" in html:
+        return {}
 
     json_str = html.split('root.App.main =')[1].split(
         '(this)')[0].split(';\n}')[0].strip()
-    data = _json.loads(json_str)[
-        'context']['dispatcher']['stores']['QuoteSummaryStore']
+    data = _json.loads(json_str)
+
+    if "_cs" in data and "_cr" in data:
+        data_stores = _json.loads(decrypt_cryptojs_stores(data))
+    else:
+        if "context" in data and "dispatcher" in data["context"]:
+            # Keep old code, just in case
+            data_stores = data['context']['dispatcher']['stores']
+        else:
+            data_stores = data
+
+    if not 'QuoteSummaryStore' in data_stores:
+        # Problem in data. Either delisted, or Yahoo spam triggered
+        return {}
+
+    data = data_stores['QuoteSummaryStore']
     # add data about Shares Outstanding for companies' tickers if they are available
     try:
-        data['annualBasicAverageShares'] = _json.loads(
-            json_str)['context']['dispatcher']['stores'][
-                'QuoteTimeSeriesStore']['timeSeries']['annualBasicAverageShares']
+        data['annualBasicAverageShares'] = \
+            data_stores['QuoteTimeSeriesStore']['timeSeries']['annualBasicAverageShares']
     except Exception:
         pass
 

yfinance/version.py

@@ -1 +1 @@
-version = "0.1.89"
+version = "0.1.95"