Merge branch 'dev' into feature/expose-exceptions-v2

fix: Correct column names

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -18,7 +18,15 @@ body:
 
         Are you spelling symbol *exactly* same as Yahoo?
 
-        Then visit `finance.yahoo.com` and confirm they have the data you want. Maybe your symbol was delisted, or your expectations of `yfinance` are wrong.
+        Then visit `finance.yahoo.com` and confirm they have the data you want. Maybe your symbol was delisted.
+
+        ### Data is wrong
+
+        **yfinance is not affiliated with Yahoo**. If Yahoo Finance website data is bad, tell Yahoo: https://help.yahoo.com/kb/finance-for-web/report-problems-feedback-ideas-yahoo-finance-sln28397.html.
+
+        But if yfinance is creating error during processing, then finish this form.
+
+        For price errors, try price repair: https://ranaroussi.github.io/yfinance/advanced/price_repair.html
 
         ### Are you spamming Yahoo?
 
@@ -63,7 +71,7 @@ body:
     id: bad-data-proof
     attributes:
       label: "Bad data proof"
-      description: "If `yfinance` returning bad data, show proof of good data here. Best proof is screenshot of finance.yahoo.com"
+      description: "If yfinance returning bad data, show proof of good data on Yahoo Finance website here."
     validations:
       required: false
 

.github/workflows/auto_close_default_issues.yml

@@ -0,0 +1,36 @@
+name: Auto-close issues using default template
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  check-template:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check if issue uses custom template
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const issue = context.payload.issue;
+            const body = issue.body || '';
+            
+            // Check for specific fields from your custom form
+            // Adjust these patterns based on your form structure
+            const hasCustomFields = body.includes('### Describe bug') || 
+                                   body.includes('### Simple code that reproduces');
+            
+            if (!hasCustomFields) {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: issue.number,
+                body: 'This issue appears to use the default template. Stop that. Use our custom bug report form.'
+              });
+              
+              await github.rest.issues.update({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: issue.number,
+                state: 'closed'
+              });
+            }

.github/workflows/deploy_doc.yml

@@ -3,7 +3,8 @@ name: Build and Deploy Sphinx Docs
 on:
   push:
     branches:
-      - dev-documented
+      - main
+      # - dev-documented
   workflow_dispatch:
 
 jobs:

.github/workflows/ruff.yml

@@ -9,5 +9,7 @@ jobs:
   ruff:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: chartboost/ruff-action@v1
+      - uses: actions/checkout@v4
+      - uses: astral-sh/ruff-action@v3
+        with:
+          args: check . --exclude yfinance/pricing_pb2.py

.gitignore

@@ -9,7 +9,6 @@ yfinance.egg-info
 build/
 *.html
 *.css
-*.png
 test.ipynb
 
 # Environments
@@ -24,4 +23,4 @@ ENV/
 /doc/_build/
 /doc/source/reference/api
 !yfinance.css
-!/doc/source/development/assets/branches.png
+!/doc/source/development/assets/branches.png

CHANGELOG.rst

@@ -1,6 +1,53 @@
 Change Log
 ===========
 
+0.2.65
+------
+Financials: ensure dtype float #2563
+Prices: fix handling arguments start/end/period #2561 #2562
+Price repair: when changing FX, update metadata #2558
+
+0.2.64
+------
+Prices:
+- handle dividends with FX, convert if repair=True  #2549
+- fix 'period' arg when start or end set  #2550
+earnings_dates: handle 'Event Type' properly  #2555
+
+0.2.63
+------
+Fix download(ISIN)  # 2531
+
+0.2.62
+------
+Fix prices 'period=max' sometimes failing  # 2509
+ISIN cache  #2516
+Proxy:
+- fix false 'proxy deprecated' messages
+- fix ISIN + proxy  #2514
+- replace print_once with warnings  #2523
+Error handling:
+- detect rate-limit during crumb fetch  #2491
+- replace requests.HTTPError with curl_cffi
+
+0.2.61
+------
+Fix ALL type hints in websocket #2493
+
+0.2.60
+------
+Fix cookie reuse, and handle DNS blocking fc.yahoo.com #2483
+Fixes for websocket:
+- relax protobuf version #2485
+- increase websockets version #2485
+- fix type hints #2488
+Fix predefined screen offset #2440
+
+0.2.59
+------
+Fix the fix for rate-limit #2452
+Feature: live price data websocket #2201
+
 0.2.58
 ------
 Fix false rate-limit problem #2430

CONTRIBUTING.md

@@ -1,90 +1,23 @@
 # Contributing
 
-## Changes
-
-The list of changes can be found in the [Changelog](https://github.com/ranaroussi/yfinance/blob/main/CHANGELOG.rst)
-
-## Running a branch
-
-### With PIP
-
-```bash
-pip install git+https://github.com/ranaroussi/yfinance.git@BRANCH
-```
-
-E.g.:
-
-```bash
-pip install git+https://github.com/ranaroussi/yfinance.git@feature/something
-```
-
-### With Git
-
-1: Download from GitHub:
-
-```bash
-git clone https://github.com/ranaroussi/yfinance.git
-```
-
-Or if a specific branch:
-
-```bash
-git clone -b <BRANCH NAME> https://github.com/ranaroussi/yfinance.git
-```
-
-> [!NOTE]
-> Only do the next part if you are installing globally
->
-> If you are installing for 1 specific project, then you can skip this step
-> and just `git clone` in the project directory
-
-2: Add download location to Python search path
-
-Two different ways, choose one:
-
-`. Add path to PYTHONPATH environment variable
-1. Add to top of Python file: 
-```python
-import sys
-sys.path.insert(0, "path/to/downloaded/yfinance")
-```
-
-3: Verify
-
-```python
-import yfinance
-print(yfinance)
-```
-
-Output should be:
-
-`<module 'yfinance' from 'path/to/downloaded/yfinance/yfinance/__init__.py'>`
-
-If output looks like this then you did step 2 wrong
-
-`<module 'yfinance' from '.../lib/python3.10/site-packages/yfinance/__init__.py'>`
+yfinance relies on the community to investigate bugs and contribute code.
 
+This is a quick short guide, full guide at https://ranaroussi.github.io/yfinance/development/index.html
 
 ## Branches
 
-To support rapid development without breaking stable versions, this project uses a two-layer branch model: ![image](https://private-user-images.githubusercontent.com/96923577/269063055-5afe5e2b-a43c-4a64-a736-a9e57fb5fe70.png?jwt=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJnaXRodWIuY29tIiwiYXVkIjoicmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbSIsImtleSI6ImtleTUiLCJleHAiOjE3NDYxNjkxODAsIm5iZiI6MTc0NjE2ODg4MCwicGF0aCI6Ii85NjkyMzU3Ny8yNjkwNjMwNTUtNWFmZTVlMmItYTQzYy00YTY0LWE3MzYtYTllNTdmYjVmZTcwLnBuZz9YLUFtei1BbGdvcml0aG09QVdTNC1ITUFDLVNIQTI1NiZYLUFtei1DcmVkZW50aWFsPUFLSUFWQ09EWUxTQTUzUFFLNFpBJTJGMjAyNTA1MDIlMkZ1cy1lYXN0LTElMkZzMyUyRmF3czRfcmVxdWVzdCZYLUFtei1EYXRlPTIwMjUwNTAyVDA2NTQ0MFomWC1BbXotRXhwaXJlcz0zMDAmWC1BbXotU2lnbmF0dXJlPTZkYWJlZDQyNWMwNTBmY2M5NDY4YmQ2OWM4ZmYzYzg2NThiNGEzMzkzN2NlMjNiZWYwMGM0Mzg5NzVlOWQ4YzImWC1BbXotU2lnbmVkSGVhZGVycz1ob3N0In0.ZK4rk7xa0w1U4KCkvv4hUKdWjuxeleuMYvZ3se6V33I) ([inspiration](https://miro.medium.com/max/700/1*2YagIpX6LuauC3ASpwHekg.png))
+YFinance uses a two-layer branch model:
 
-* **dev** : new features & some bug-fixes merged here, tested together, conflicts fixed, etc. Once stable, can merge to ...
+* **dev**: new features & most bug-fixes merged here, tested together, conflicts fixed, etc.
+* **main**: stable branch where PIP releases are created.
 
-* **main**: stable branch. Where PIP releases are created
-
-
-Most of the time you want to target the **dev** branch, but default is **main** so remember to switch to **dev**.
-
-**Exception:** can straight merge into **main** if:
-
-* yfinance is massively broken
-
-* or part of yfinance is broken and the fix is simple and clearly doesn't affect anything else
-
-* or not editing part of the code (e.g. editing this file)
+## Running a branch
 
+```bash
+pip install git+ranaroussi/yfinance.git@dev  # <- dev branch
+```
 
+https://ranaroussi.github.io/yfinance/development/running.html
 
 ### I'm a GitHub newbie, how do I contribute code?
 
@@ -92,67 +25,36 @@ Most of the time you want to target the **dev** branch, but default is **main**
 
 2. Implement your change in your fork, ideally in a specific branch
 
-3. Create a Pull Request, from your fork to this project. If addressing an Issue, link to it
+3. Create a [Pull Request](https://github.com/ranaroussi/yfinance/pulls), from your fork to this project. If addressing an Issue, link to it
 
-
-### [How to download & run a GitHub version of yfinance](https://github.com/ranaroussi/yfinance/discussions/1080)
+https://ranaroussi.github.io/yfinance/development/code.html
 
 ## Documentation website
 
-The new docs website [ranaroussi.github.io/yfinance/index.html](https://ranaroussi.github.io/yfinance/index.html) is generated automatically from code. When you fetch a branch or PR, probably it also has changes to docs - to generate and view this:
+The new docs website is generated automatically from code. https://ranaroussi.github.io/yfinance/index.html
 
-```bash
-pip install -r requirements.txt
-pip install Sphinx==8.0.2 pydata-sphinx-theme==0.15.4 Jinja2==3.1.4 sphinx-copybutton==0.5.2
-sphinx-build -b html doc/source doc/_build/html
-python -m http.server -d ./doc/_build/html
-# open "localhost:8000" in browser
-```
+Remember to updates docs when you change code, and check docs locally.
+
+https://ranaroussi.github.io/yfinance/development/documentation.html
+
+## Git tricks
+
+Help keep the Git commit history and [network graph](https://github.com/ranaroussi/yfinance/network) compact:
+
+* got a long descriptive commit message? `git commit -m "short sentence summary" -m "full commit message"`
+
+* combine multiple commits into 1 with `git squash`
+
+* `git rebase` is your friend: change base branch, or "merge in" updates
+
+https://ranaroussi.github.io/yfinance/development/code.html#git-stuff
 
 ## Unit tests
 
 Tests have been written using the built-in Python module `unittest`. Examples:
 
-* Run all price tests: `python -m unittest tests.test_prices`
-
-* Run sub-set of price tests: `python -m unittest tests.test_prices.TestPriceRepair`
-
-* Run a specific test: `python -m unittest tests.test_prices.TestPriceRepair.test_ticker_missing`
-
 * Run all tests: `python -m unittest discover -s tests`
 
-> [!NOTE]
-> The tests are currently failing already
->
-> Standard result:
->
-> **Failures:** 11
->
-> **Errors:** 93
->
-> **Skipped:** 1
+https://ranaroussi.github.io/yfinance/development/testing.html
 
-## Git stuff
-### commits
-
-To keep the Git commit history and [network graph](https://github.com/ranaroussi/yfinance/network) compact please follow these two rules:
-
-* For long commit messages use this: `git commit -m "short sentence summary" -m "full commit message"`
-
-* `squash` tiny/negligible commits back with meaningful commits, or to combine successive related commits. [Guide](https://docs.gitlab.com/ee/topics/git/git_rebase.html#interactive-rebase) but basically it's:
-
-
-```bash
-git rebase -i HEAD~2
-git push --force-with-lease origin <branch-name>
-```
-
-### rebase
- 
-You might be asked to move your branch from `main` to `dev`. Make sure you have pulled **all** relevant branches then run:
-
-```bash
-git checkout <your branch>
-git rebase --onto dev main <branch-name>
-git push --force-with-lease origin <branch-name>
-```
+> See the [Developer Guide](https://ranaroussi.github.io/yfinance/development/contributing.html#GIT-STUFF) for more information.

README.md

@@ -40,6 +40,7 @@
 - `Tickers`: multiple tickers' data
 - `download`: download market data for multiple tickers
 - `Market`: get information about a market
+- `WebSocket` and `AsyncWebSocket`: live streaming data
 - `Search`: quotes and news from search
 - `Sector` and `Industry`: sector and industry information
 - `EquityQuery` and `Screener`: build query to screen market
@@ -52,6 +53,8 @@ Install `yfinance` from PYPI using `pip`:
 $ pip install yfinance
 ```
 
+### [yfinance relies on the community to investigate bugs and contribute code. Here's how you can help.](CONTRIBUTING.md)
+
 ---
 
 ![Star History Chart](https://api.star-history.com/svg?repos=ranaroussi/yfinance)

doc/source/_templates/autosummary/class.rst

@@ -5,6 +5,7 @@
 .. currentmodule:: {{ module }}
 
 .. autoclass:: {{ objname }}
+   :exclude-members: {% for item in attributes %}{{ item }}{% if not loop.last %}, {% endif %}{% endfor %}
 
    {% block attributes %}
    {% if attributes %}

doc/source/advanced/caching.rst

@@ -1,47 +1,6 @@
 Caching
 =======
 
-Smarter Scraping
-----------------
-
-Install the `nospam` package to cache API calls and reduce spam to Yahoo:
-
-.. code-block:: bash
-
-   pip install yfinance[nospam]
-
-To use a custom `requests` session, pass a `session=` argument to
-the Ticker constructor. This allows for caching calls to the API as well as a custom way to modify requests via  the `User-agent` header.
-
-.. code-block:: python
-
-   import requests_cache
-   session = requests_cache.CachedSession('yfinance.cache')
-   session.headers['User-agent'] = 'my-program/1.0'
-   ticker = yf.Ticker('MSFT', session=session)
-   
-   # The scraped response will be stored in the cache
-   ticker.actions
-
-
-Combine `requests_cache` with rate-limiting to avoid triggering Yahoo's rate-limiter/blocker that can corrupt data.
-
-.. code-block:: python
-
-   from requests import Session
-   from requests_cache import CacheMixin, SQLiteCache
-   from requests_ratelimiter import LimiterMixin, MemoryQueueBucket
-   from pyrate_limiter import Duration, RequestRate, Limiter
-   class CachedLimiterSession(CacheMixin, LimiterMixin, Session):
-      pass
-
-   session = CachedLimiterSession(
-      limiter=Limiter(RequestRate(2, Duration.SECOND*5)),  # max 2 requests per 5 seconds
-      bucket_class=MemoryQueueBucket,
-      backend=SQLiteCache("yfinance.cache"),
-   )
-
-
 Persistent Cache
 ----------------
 

doc/source/advanced/index.rst

@@ -8,4 +8,5 @@ Advanced
    logging
    config
    caching
-   multi_level_columns
+   multi_level_columns
+   price_repair

doc/source/advanced/price_repair.rst

@@ -0,0 +1,277 @@
+************
+Price Repair
+************
+
+The new argument ``repair=True`` in ``history()`` and ``download()`` will attempt to fix a variety of price errors caused by Yahoo. Only US market data appears perfect, I guess Yahoo doesn't care much about rest of world?
+
+The returned table will have a new column ``Repaired?`` that specifies if row was repaired.
+
+Price repair
+============
+
+Missing dividend adjustment
+---------------------------
+
+If dividend in data but preceding ``Adj Close`` = ``Close``, then manually apply dividend-adjustment to ``Adj Close``.
+Note: ``Repaired?`` is NOT set to ``True`` because fix only changes ``Adj Close``
+
+.. figure:: /_static/images/repair-prices-missing-div-adjust.png
+   :alt: 8TRA.DE: repair missing dividend adjustment
+   :width: 80%
+   :align: left
+
+   8TRA.DE
+
+.. container:: clearer
+
+   ..
+
+Missing split adjustment
+------------------------
+
+If stock split in data but preceding price data is not adjusted, then manually apply stock split.
+Requires date range include 1 day after stock split for calibration - sometimes Yahoo fails to adjust prices on stock split day.
+
+.. figure:: /_static/images/repair-prices-missing-split-adjust.png
+   :alt: MOB.ST: repair missing split adjustment
+   :width: 80%
+   :align: left
+
+   MOB.ST
+
+.. container:: clearer
+
+   ..
+
+Missing data
+------------
+
+If price data is clearly missing or corrupt, then reconstructed using smaller interval e.g. ``1h`` to fix ``1d`` data.
+
+.. figure:: /_static/images/repair-prices-missing-row.png
+   :alt: 1COV.DE: repair missing row
+   :width: 80%
+   :align: left
+
+   1COV.DE missing row
+
+.. container:: clearer
+
+   ..
+
+.. figure:: /_static/images/repair-prices-missing-volume-intraday.png
+   :alt: 1COV.DE: repair missing Volume, but intraday price changed
+   :width: 80%
+   :align: left
+
+   1COV.DE missing Volume, but intraday price changed
+
+.. container:: clearer
+
+   ..
+
+.. figure:: /_static/images/repair-prices-missing-volume-daily.png
+   :alt: 0316.HK: repair missing Volume, but daily price changed
+   :width: 80%
+   :align: left
+
+   0316.HK missing Volume, but daily price changed
+
+.. container:: clearer
+
+   ..
+
+100x errors
+-----------
+
+Sometimes Yahoo mixes up currencies e.g. $/cents or £/pence. So some prices are 100x wrong.
+Sometimes they are spread randomly through data - these detected with ``scipy`` module.
+Other times they are in a block, because Yahoo decided one day to permanently switch currency.
+
+.. figure:: /_static/images/repair-prices-100x.png
+   :alt: AET.L: repair 100x
+   :width: 80%
+   :align: left
+
+   AET.L
+
+Price reconstruction - algorithm notes
+--------------------------------------
+
+Spam minimised by grouping fetches. Tries to be aware of data limits e.g. ``1h`` cannot be fetched beyond 2 years.
+
+If Yahoo eventually does fix the bad data that required reconstruction, you will see it's slightly different to reconstructed prices and volume often significantly different. Best I can do, and beats missing data.
+
+Dividend repair (new)
+=====================
+
+Fix errors in dividends:
+
+1. adjustment missing or 100x too small/big for the dividend
+2. duplicate dividend (within 7 days)
+3. dividend 100x too big/small for the ex-dividend price drop
+4. ex-div date wrong (price drop is few days/weeks after)
+
+Most errors I've seen are on London stock exchange (£/pence mixup), but no exchange is safe.
+
+IMPORTANT - false positives
+---------------------------
+
+Because fixing (3) relies on price action, there is a chance of a "false positive" (FP) - thinking an error exists when data is good.
+FP rate increases with longer intervals, so only 1d intervals are repaired. If you request repair on multiday intervals (weekly etc), then: 1d is fetched from Yahoo, repaired, then resampled - **this has nice side-effect of solving Yahoo's flawed way of div-adjusting multiday intervals.**
+
+FP rate on 1d is tiny. They tend to happen with tiny dividends e.g. 0.5%, mistaking normal price volatility for an ex-div drop 100x bigger than the dividend, causing repair of the "too small" dividend (repair logic already tries to account for normal volatility by subtracting median). Either accept the risk, or fetch 6-12 months of prices with at least 2 dividends - then can analyse the dividends together to identify false positives.
+
+Adjustment missing
+------------------
+
+1398.HK
+
+.. code-block:: text
+
+   # ORIGINAL:
+                              Close  Adj Close  Dividends
+   2024-07-08 00:00:00+08:00   4.33       4.33   0.335715
+   2024-07-04 00:00:00+08:00   4.83       4.83   0.000000
+
+.. code-block:: text
+
+   # REPAIRED:
+                              Close  Adj Close  Dividends
+   2024-07-08 00:00:00+08:00   4.33   4.330000   0.335715
+   2024-07-04 00:00:00+08:00   4.83   4.494285   0.000000
+
+Adjustment too small
+--------------------
+
+3IN.L
+
+.. code-block:: text
+
+   # ORIGINAL:
+                              Close  Adj Close  Dividends
+   2024-06-13 00:00:00+01:00  3.185   3.185000    0.05950
+   2024-06-12 00:00:00+01:00  3.270   3.269405    0.00000
+
+.. code-block:: text
+
+   # REPAIRED:
+                              Close  Adj Close  Dividends
+   2024-06-13 00:00:00+01:00  3.185   3.185000    0.05950
+   2024-06-12 00:00:00+01:00  3.270   3.210500    0.00000
+
+Duplicate (within 7 days)
+-------------------------
+
+ALC.SW
+
+.. code-block:: text
+
+   # ORIGINAL:
+                                  Close  Adj Close  Dividends
+   2023-05-10 00:00:00+02:00  70.580002  70.352142       0.21
+   2023-05-09 00:00:00+02:00  65.739998  65.318443       0.21
+   2023-05-08 00:00:00+02:00  66.379997  65.745682       0.00
+
+.. code-block:: text
+
+   # REPAIRED:
+                                  Close  Adj Close  Dividends
+   2023-05-10 00:00:00+02:00  70.580002  70.352142       0.00
+   2023-05-09 00:00:00+02:00  65.739998  65.527764       0.21
+   2023-05-08 00:00:00+02:00  66.379997  65.956371       0.00
+
+Dividend too big
+----------------
+
+HLCL.L
+
+.. code-block:: text
+
+   # ORIGINAL:
+                              Close  Adj Close  Dividends
+   2024-06-27 00:00:00+01:00  2.360     2.3600       1.78
+   2024-06-26 00:00:00+01:00  2.375     2.3572       0.00
+
+   # REPAIRED:
+                              Close  Adj Close  Dividends
+   2024-06-27 00:00:00+01:00  2.360     2.3600     0.0178
+   2024-06-26 00:00:00+01:00  2.375     2.3572     0.0000
+
+Dividend & adjust too big
+-------------------------
+
+LTI.L
+
+.. code-block:: text
+
+   # ORIGINAL:
+                              Close  Adj Close     Adj  Dividends
+   2024-08-08 00:00:00+01:00  768.0      768.0  1.0000     5150.0
+   2024-08-07 00:00:00+01:00  819.0    -4331.0 -5.2882        0.0
+                              Close  Adj Close     Adj  Dividends
+   2024-08-08 00:00:00+01:00  768.0      768.0  1.0000       51.5
+   2024-08-07 00:00:00+01:00  819.0      767.5  0.9371        0.0
+
+Dividend too small
+------------------
+
+BVT.L
+
+.. code-block:: text
+
+   # ORIGINAL:
+                               Close  Adj Close     Adj  Dividends
+   2022-02-03 00:00:00+00:00  0.7534   0.675197  0.8962    0.00001
+   2022-02-01 00:00:00+00:00  0.7844   0.702970  0.8962    0.00000
+
+.. code-block:: text
+
+   # REPAIRED:
+                               Close  Adj Close     Adj  Dividends
+   2022-02-03 00:00:00+00:00  0.7534   0.675197  0.8962      0.001
+   2022-02-01 00:00:00+00:00  0.7844   0.702075  0.8950      0.000
+
+Adjusted 2x on day before
+-------------------------
+
+clue: Close < Low
+
+2020.OL
+
+.. code-block:: text
+
+   # ORIGINAL:
+                                     Low       Close   Adj Close  Dividends
+   2023-12-21 00:00:00+01:00  120.199997  121.099998  118.868782       0.18
+   2023-12-20 00:00:00+01:00  122.000000  121.900002  119.477371       0.00
+
+.. code-block:: text
+
+   # REPAIRED:
+                                     Low       Close   Adj Close  Dividends
+   2023-12-21 00:00:00+01:00  120.199997  121.099998  118.868782       0.18
+   2023-12-20 00:00:00+01:00  122.000000  122.080002  119.654045       0.00
+
+ex-div date wrong
+-----------------
+
+TETY.ST
+
+.. code-block:: text
+
+   # ORIGINAL:
+                                  Close  Adj Close  Dividends
+   2022-06-22 00:00:00+02:00  66.699997  60.085415        0.0
+   2022-06-21 00:00:00+02:00  71.599998  64.499489        0.0
+   2022-06-20 00:00:00+02:00  71.800003  64.679657        5.0
+   2022-06-17 00:00:00+02:00  71.000000  59.454838        0.0
+
+.. code-block:: text
+
+   # REPAIRED:
+                                  Close  Adj Close  Dividends
+   2022-06-22 00:00:00+02:00  66.699997  60.085415        5.0
+   2022-06-21 00:00:00+02:00  71.599998  60.007881        0.0
+   2022-06-20 00:00:00+02:00  71.800003  60.175503        0.0
+   2022-06-17 00:00:00+02:00  71.000000  59.505021        0.0

doc/source/development/code.rst

@@ -0,0 +1,90 @@
+****
+Code
+****
+
+To support rapid development without breaking stable versions, this project uses a two-layer branch model:
+
+.. image:: assets/branches.png
+   :alt: Branching Model
+
+`Inspiration <https://miro.medium.com/max/700/1*2YagIpX6LuauC3ASpwHekg.png>`_
+
+- **dev**: New features and some bug fixes are merged here. This branch allows collective testing, conflict resolution, and further stabilization before merging into the stable branch.
+- **main**: Stable branch where PIP releases are created.
+
+By default, branches target **main**, but most contributions should target **dev**. 
+
+**Exceptions**:
+Direct merges to **main** are allowed if:
+
+- `yfinance` is massively broken
+- Part of `yfinance` is broken, and the fix is simple and isolated
+- Not updating the code (e.g. docs)
+
+Creating your branch
+--------------------
+
+1. Fork the repository on GitHub. If already forked, remember to ``Sync fork``
+
+2. Clone your forked repository:
+
+   .. code-block:: bash
+
+      git clone https://github.com/{user}/{repo}.git
+
+3. Create a new branch for your feature or bug fix, from appropriate base branch:
+
+   .. code-block:: bash
+
+      git checkout {base e.g. dev}
+      git pull
+      git checkout -b {your branch}
+
+4. Make your changes, commit them, and push your branch to GitHub. To keep the commit history and `network graph <https://github.com/ranaroussi/yfinance/network>`_ compact, give your commits a very short summary then description:
+
+   .. code-block:: bash
+
+      git commit -m "short sentence summary" -m "full commit message"
+      # Long message can be multiple lines (tip: copy-paste)
+
+6. `Open a pull request on Github <https://github.com/ranaroussi/yfinance/pulls>`_.
+
+Running a branch
+----------------
+
+Please see `this page </development/running>`_.
+
+Git stuff
+---------
+
+- You might be asked to move your branch from ``main`` to ``dev``. This is a ``git rebase``. Remember to update **all** branches involved.
+
+  .. code-block:: bash
+
+     # update all branches:
+     git checkout main
+     git pull
+     git checkout dev
+     git pull
+     # rebase from main to dev:
+     git checkout {your branch}
+     git pull
+     git rebase --onto dev main {your branch}
+     git push --force-with-lease origin {your branch}
+
+- ``git rebase`` can also be used to update your branch with new commits from base, but without adding a commit to your branch history like git merge does. This keeps history clean and avoids future merge problems.
+
+  .. code-block:: bash
+
+     git checkout {base branch e.g. dev}
+     git pull
+     git checkout {your branch}
+     git rebase {base}
+     git push --force-with-lease origin {your branch}
+
+- ``git squash`` tiny or negligible commits with meaningful ones, or to combine successive related commits. `git squash guide <https://docs.gitlab.com/ee/topics/git/git_rebase.html#interactive-rebase>`_
+
+  .. code-block:: bash
+
+     git rebase -i HEAD~2
+     git push --force-with-lease origin {your branch}

doc/source/development/contributing.rst

@@ -1,109 +0,0 @@
-********************************
-Contributiong to yfinance
-********************************
-
-`yfinance` relies on the community to investigate bugs and contribute code. Here’s how you can help:
-
-Contributing
-------------
-
-1. Fork the repository on GitHub.
-2. Clone your forked repository:
-
-   .. code-block:: bash
-
-      git clone https://github.com/your-username/yfinance.git
-
-3. Create a new branch for your feature or bug fix:
-
-   .. code-block:: bash
-
-      git checkout -b feature-branch-name
-
-4. Make your changes, commit them, and push your branch to GitHub. To keep the commit history and `network graph <https://github.com/ranaroussi/yfinance/network>`_ compact:
-
-   Use short summaries for commits
-
-   .. code-block:: shell
-
-      git commit -m "short summary" -m "full commit message"
-
-   **Squash** tiny or negligible commits with meaningful ones.
-
-   .. code-block:: shell
-
-      git rebase -i HEAD~2
-      git push --force-with-lease origin <branch-name>
-
-5. Open a pull request on the `yfinance` GitHub page.
-
-For more information, see the `Developer Guide <https://github.com/ranaroussi/yfinance/discussions/1084>`_.
-
-Branches
----------
-
-To support rapid development without breaking stable versions, this project uses a two-layer branch model:
-
-.. image:: assets/branches.png
-   :alt: Branching Model
-
-`Inspiration <https://miro.medium.com/max/700/1*2YagIpX6LuauC3ASpwHekg.png>`_
-
-- **dev**: New features and some bug fixes are merged here. This branch allows collective testing, conflict resolution, and further stabilization before merging into the stable branch.
-- **main**: Stable branch where PIP releases are created.
-
-By default, branches target **main**, but most contributions should target **dev**. 
-
-**Exceptions**:
-Direct merges to **main** are allowed if:
-
-- `yfinance` is massively broken
-- Part of `yfinance` is broken, and the fix is simple and isolated
-
-Unit Tests
-----------
-
-Tests are written using Python’s `unittest` module. Here are some ways to run tests:
-
-- **Run all price tests**:
-
-  .. code-block:: shell
-
-     python -m unittest tests.test_prices
-
-- **Run a subset of price tests**:
-
-  .. code-block:: shell
-
-     python -m unittest tests.test_prices.TestPriceRepair
-
-- **Run a specific test**:
-
-  .. code-block:: shell
-
-     python -m unittest tests.test_prices.TestPriceRepair.test_ticker_missing
-
-- **Run all tests**:
-
-  .. code-block:: shell
-
-     python -m unittest discover -s tests
-
-Rebasing
---------------
-
-If asked to move your branch from **main** to **dev**:
-
-1. Ensure all relevant branches are pulled.
-2. Run:
-
-   .. code-block:: shell
-
-      git checkout <your-branch>
-      git rebase --onto dev main <branch-name>
-      git push --force-with-lease origin <branch-name>
-
-Running the GitHub Version of yfinance
---------------------------------------
-
-To download and run a GitHub version of `yfinance`, refer to `GitHub discussion <https://github.com/ranaroussi/yfinance/discussions/1080>`_

doc/source/development/documentation.rst

@@ -1,15 +1,15 @@
-*************************************
-Contribution to the documentation
-*************************************
+*************
+Documentation
+*************
 
 .. contents:: Documentation:
    :local:
 
 About documentation
-------------------------
+-------------------
 * yfinance documentation is written in reStructuredText (rst) and built using Sphinx.
-* The documentation file is in `doc/source/..`.
-* Most of the notes under API References read from class and methods docstrings. These documentations, found in `doc/source/reference/api` is autogenerated by Sphinx and not included in git.
+* The documentation file is in ``doc/source/..``.
+* Most of the notes under API References read from class and methods docstrings. These documentations, found in ``doc/source/reference/api`` is autogenerated by Sphinx and not included in git.
 
 Building documentation locally
 -------------------------------
@@ -17,30 +17,38 @@ To build the documentation locally, follow these steps:
 
 1. **Install Required Dependencies**:
 
-   * Make sure `Sphinx` and any other dependencies are installed. If a `requirements.txt` file is available, you can install dependencies by running:
-   
-   .. code-block:: console
+   * Make sure ``Sphinx`` and any other dependencies are installed. If a ``requirements.txt`` file is available, you can install dependencies by running:
+
+   .. code-block:: bash
 
       pip install -r requirements.txt
-   
+      pip install Sphinx==8.0.2 pydata-sphinx-theme==0.15.4 Jinja2==3.1.4 sphinx-copybutton==0.5.2
+  
 
 2. **Build with Sphinx**:
     
    * After dependencies are installed, use the sphinx-build command to generate HTML documentation. 
-   * Go to `doc/` directory Run:
+   * Go to ``doc/`` directory Run:
 
-   .. code-block:: console
+   .. code-block:: bash
 
-      make clean && make html 
+      sphinx-build -b html doc/source doc/_build/html
 
 3. **View Documentation Locally**:
 
-   * Open `doc/build/html/index.html` in the browser to view the generated documentation.
+   .. code-block:: bash
 
-Building documentation on main 
--------------------------------
-The documentation updates are built on merge to `main` branch. This is done via GitHub Actions workflow based on `/yfinance/.github/workflows/deploy_doc.yml`.
+      python -m http.server -d ./doc/_build/html
 
-1. Reivew the changes locally and push to `dev`.
+   Then open "localhost:8000" in browser
 
-2. When `dev` gets merged to `main`, GitHub Actions workflow is automated to build documentation.
+
+Publishing documentation
+------------------------
+
+Merge into ``main`` branch triggers auto-generating documentation by action ``.github/workflows/deploy_doc.yml``.
+This publishes the generated HTML into branch ``documentation``.
+
+1. Review the changes locally and push to ``dev``.
+
+2. When ``dev`` gets merged to ``main``, GitHub Actions workflow is automated to build documentation.

doc/source/development/index.rst

@@ -2,9 +2,12 @@
 Development
 ===========
 
+yfinance relies on the community to investigate bugs and contribute code. Here's how you can help:
+
 .. toctree::
    :maxdepth: 1
 
-   contributing
+   code
+   running
    documentation
-   reporting_bug
+   testing

doc/source/development/reporting_bug.rst

@@ -1,5 +0,0 @@
-********************************
-Reporting a Bug
-********************************
-
-Open a new issue on our `GitHub <https://github.com/ranaroussi/yfinance/issues>`_.

doc/source/development/running.rst

@@ -0,0 +1,62 @@
+Running a branch
+================
+
+With PIP
+--------
+
+.. code-block:: bash
+
+   pip install git+https://github.com/{user}/{repo}.git@{branch}
+
+E.g.:
+
+.. code-block:: bash
+
+   pip install git+https://github.com/ranaroussi/yfinance.git@feature/name
+
+With Git
+--------
+
+1: Download from GitHub:
+
+.. code-block:: bash
+    git clone https://github.com/{user}/{repo}.git
+    pip install -r ./yfinance/requirements.txt
+
+Or if a specific branch:
+
+.. code-block:: bash
+    git clone -b {branch} https://github.com/{user}/{repo}.git
+    pip install -r ./yfinance/requirements.txt
+
+.. NOTE::
+    Only do the next part if you are installing globally
+
+    If you are installing for 1 specific project, then you can skip this step
+    and just `git clone` in the project directory
+
+2. Add download location to Python search path
+
+Two different ways, choose one:
+
+1) Add path to ``PYTHONPATH`` environment variable
+
+2) Add to top of Python file: 
+.. code-block:: python
+    import sys
+    sys.path.insert(0, "path/to/downloaded/yfinance")
+
+
+3: Verify
+
+.. code-block:: python
+    import yfinance
+    print(yfinance)
+
+Output should be:
+
+`<module 'yfinance' from 'path/to/downloaded/yfinance/yfinance/__init__.py'>`
+
+If output looks like this then you did step 2 wrong
+
+`<module 'yfinance' from '.../lib/python3.10/site-packages/yfinance/__init__.py'>`

doc/source/development/testing.rst

@@ -0,0 +1,50 @@
+Unit Tests
+----------
+
+Tests are written using Python's `unittest` module. Here are some ways to run tests:
+
+- **Run all price tests**:
+
+  .. code-block:: bash
+
+     python -m unittest tests.test_prices
+
+- **Run a subset of price tests**:
+
+  .. code-block:: bash
+
+     python -m unittest tests.test_prices.TestPriceRepair
+
+- **Run a specific test**:
+
+  .. code-block:: bash
+
+     python -m unittest tests.test_prices_repair.TestPriceRepair.test_ticker_missing
+
+- **General command**:
+
+  ..code-block:: bash
+
+     python -m unittest tests.{file}.{class}.{method}
+
+- **Run all tests**:
+
+  .. code-block:: bash
+
+     python -m unittest discover -s tests
+
+.. note::
+
+    The tests are currently failing already
+
+    Standard result:
+
+    **Failures:** 11
+
+    **Errors:** 93
+
+    **Skipped:** 1
+
+.. seealso::
+
+    See the ` ``unittest`` module <https://docs.python.org/3/library/unittest.html>`_ for more information.

doc/source/reference/examples/live_async.py

@@ -0,0 +1,23 @@
+import asyncio
+import yfinance as yf
+
+# define your message callback
+def message_handler(message):
+    print("Received message:", message)
+
+async def main():
+    # =======================
+    # With Context Manager
+    # =======================
+    async with yf.AsyncWebSocket() as ws:
+        await ws.subscribe(["AAPL", "BTC-USD"])
+        await ws.listen()
+
+    # =======================
+    # Without Context Manager
+    # =======================
+    ws = yf.AsyncWebSocket()
+    await ws.subscribe(["AAPL", "BTC-USD"])
+    await ws.listen()
+
+asyncio.run(main())

doc/source/reference/examples/live_sync.py

@@ -0,0 +1,19 @@
+import yfinance as yf
+
+# define your message callback
+def message_handler(message):
+    print("Received message:", message)
+
+# =======================
+# With Context Manager
+# =======================
+with yf.WebSocket() as ws:
+    ws.subscribe(["AAPL", "BTC-USD"])
+    ws.listen(message_handler)
+
+# =======================
+# Without Context Manager
+# =======================
+ws = yf.WebSocket()
+ws.subscribe(["AAPL", "BTC-USD"])
+ws.listen(message_handler)

doc/source/reference/examples/ticker.py

@@ -20,3 +20,6 @@ dat.info
 
 # analysis
 dat.analyst_price_targets
+
+# websocket
+dat.live()

doc/source/reference/examples/tickers.py

@@ -5,4 +5,7 @@ tickers = yf.Tickers('msft aapl goog')
 # access each ticker using (example)
 tickers.tickers['MSFT'].info
 tickers.tickers['AAPL'].history(period="1mo")
-tickers.tickers['GOOG'].actions
+tickers.tickers['GOOG'].actions
+
+# websocket
+tickers.live()

doc/source/reference/index.rst

@@ -19,6 +19,8 @@ The following are the publicly available classes, and functions exposed by the `
 - :attr:`download <yfinance.download>`: Function to download market data for multiple tickers.
 - :attr:`Search <yfinance.Search>`: Class for accessing search results.
 - :attr:`Lookup <yfinance.Lookup>`: Class for looking up tickers.
+- :class:`WebSocket <yfinance.WebSocket>`: Class for synchronously streaming live market data.
+- :class:`AsyncWebSocket <yfinance.AsyncWebSocket>`: Class for asynchronously streaming live market data.
 - :attr:`Sector <yfinance.Sector>`: Domain class for accessing sector information.
 - :attr:`Industry <yfinance.Industry>`: Domain class for accessing industry information.
 - :attr:`Market <yfinance.Market>`: Class for accessing market status & summary.
@@ -41,6 +43,7 @@ The following are the publicly available classes, and functions exposed by the `
    yfinance.market
    yfinance.search
    yfinance.lookup
+   yfinance.websocket
    yfinance.sector_industry
    yfinance.screener
    yfinance.functions

doc/source/reference/yfinance.stock.rst

@@ -20,6 +20,9 @@ Ticker stock methods
    :meth:`yfinance.scrapers.history.PriceHistory.history`
       Documentation for history
 
+   :doc:`../advanced/price_repair`
+      Documentation for price repair
+
 .. autosummary::
    :toctree: api/
    :recursive:

doc/source/reference/yfinance.websocket.rst

@@ -0,0 +1,46 @@
+=====================
+WebSocket
+=====================
+
+.. currentmodule:: yfinance
+
+The `WebSocket` module allows you to stream live price data from Yahoo Finance using both synchronous and asynchronous clients.
+
+Classes
+------------
+
+.. autosummary::
+   :toctree: api/
+
+   WebSocket
+   AsyncWebSocket
+
+Synchronous WebSocket
+----------------------
+
+The `WebSocket` class provides a synchronous interface for subscribing to price updates.
+
+Sample Code:
+
+.. literalinclude:: examples/live_sync.py
+   :language: python
+
+Asynchronous WebSocket
+-----------------------
+
+The `AsyncWebSocket` class provides an asynchronous interface for subscribing to price updates.
+
+Sample Code:
+
+.. literalinclude:: examples/live_async.py
+   :language: python
+
+.. note::
+    If you're running asynchronous code in a Jupyter notebook, you may encounter issues with event loops. To resolve this, you need to import and apply `nest_asyncio` to allow nested event loops.
+
+    Add the following code before running asynchronous operations:
+
+    .. code-block:: python
+
+        import nest_asyncio
+        nest_asyncio.apply()

meta.yaml

@@ -1,5 +1,5 @@
 {% set name = "yfinance" %}
-{% set version = "0.2.58" %}
+{% set version = "0.2.65" %}
 
 package:
   name: "{{ name|lower }}"

requirements.txt

@@ -10,4 +10,6 @@ peewee>=3.16.2
 requests_cache>=1.0
 requests_ratelimiter>=0.3.1
 scipy>=1.6.3
-curl_cffi>=0.7
+curl_cffi>=0.7
+protobuf>=3.19.0
+websockets>=13.0

setup.py

@@ -63,11 +63,17 @@ setup(
                       'requests>=2.31', 'multitasking>=0.0.7',
                       'platformdirs>=2.0.0', 'pytz>=2022.5',
                       'frozendict>=2.3.4', 'peewee>=3.16.2',
-                      'beautifulsoup4>=4.11.1', 'curl_cffi>=0.7'],
+                      'beautifulsoup4>=4.11.1', 'curl_cffi>=0.7',
+                      'protobuf>=3.19.0', 'websockets>=13.0'],
     extras_require={
         'nospam': ['requests_cache>=1.0', 'requests_ratelimiter>=0.3.1'],
         'repair': ['scipy>=1.6.3'],
     },
+    # Include protobuf files for websocket support
+    package_data={
+        'yfinance': ['pricing.proto', 'pricing_pb2.py'],
+    },
+    include_package_data=True,
     # Note: Pandas.read_html() needs html5lib & beautifulsoup4
     entry_points={
         'console_scripts': [

tests/context.py

@@ -5,8 +5,8 @@ import datetime as _dt
 import sys
 import os
 import yfinance
-from requests_ratelimiter import LimiterSession
-from pyrate_limiter import Duration, RequestRate, Limiter
+# from requests_ratelimiter import LimiterSession
+# from pyrate_limiter import Duration, RequestRate, Limiter
 
 _parent_dp = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 _src_dp = _parent_dp
@@ -25,12 +25,15 @@ if os.path.isdir(testing_cache_dirpath):
         import shutil
         shutil.rmtree(testing_cache_dirpath)
 
-# Setup a session to only rate-limit
-history_rate = RequestRate(1, Duration.SECOND)
-limiter = Limiter(history_rate)
-session_gbl = LimiterSession(limiter=limiter)
+# Since switching to curl_cffi, the requests_ratelimiter|cache won't work.
+session_gbl = None
 
-# Use this instead if you also want caching:
+# # Setup a session to only rate-limit
+# history_rate = RequestRate(1, Duration.SECOND)
+# limiter = Limiter(history_rate)
+# session_gbl = LimiterSession(limiter=limiter)
+
+# # Use this instead if you also want caching:
 # from requests_cache import CacheMixin, SQLiteCache
 # from requests_ratelimiter import LimiterMixin
 # from requests import Session

tests/test_live.py

@@ -0,0 +1,30 @@
+import unittest
+from unittest.mock import Mock
+
+from yfinance.live import BaseWebSocket
+
+
+class TestWebSocket(unittest.TestCase):
+    def test_decode_message_valid(self):
+        message = ("CgdCVEMtVVNEFYoMuUcYwLCVgIplIgNVU0QqA0NDQzApOAFFPWrEP0iAgOrxvANVx/25R12csrRHZYD8skR9/"
+                   "7i0R7ABgIDq8bwD2AEE4AGAgOrxvAPoAYCA6vG8A/IBA0JUQ4ECAAAAwPrjckGJAgAA2P5ZT3tC")
+
+        ws = BaseWebSocket(Mock())
+        decoded = ws._decode_message(message)
+
+        expected = {'id': 'BTC-USD', 'price': 94745.08, 'time': '1736509140000', 'currency': 'USD', 'exchange': 'CCC',
+                    'quote_type': 41, 'market_hours': 1, 'change_percent': 1.5344921, 'day_volume': '59712028672',
+                    'day_high': 95227.555, 'day_low': 92517.22, 'change': 1431.8906, 'open_price': 92529.99,
+                    'last_size': '59712028672', 'price_hint': '2', 'vol_24hr': '59712028672',
+                    'vol_all_currencies': '59712028672', 'from_currency': 'BTC', 'circulating_supply': 19808172.0,
+                    'market_cap': 1876726640000.0}
+
+        self.assertEqual(expected, decoded)
+
+    def test_decode_message_invalid(self):
+        websocket = BaseWebSocket(Mock())
+        base64_message = "invalid_base64_string"
+        decoded = websocket._decode_message(base64_message)
+        assert "error" in decoded
+        assert "raw_base64" in decoded
+        self.assertEqual(base64_message, decoded["raw_base64"])

tests/test_prices.py

@@ -224,7 +224,7 @@ class TestPriceHistory(unittest.TestCase):
         start_d = _dt.date(2022, 1, 1)
         end_d = _dt.date(2023, 1, 1)
 
-        tkr_div_dates = {'BHP.AX': [_dt.date(2022, 9, 1), _dt.date(2022, 2, 24)],  # Yahoo claims 23-Feb but wrong because DST
+        tkr_div_dates = {'BHP.AX': [_dt.date(2022, 9, 1), _dt.date(2022, 2, 24)],
                          'IMP.JO': [_dt.date(2022, 9, 21), _dt.date(2022, 3, 16)],
                          'BP.L': [_dt.date(2022, 11, 10), _dt.date(2022, 8, 11), _dt.date(2022, 5, 12),
                                   _dt.date(2022, 2, 17)],

tests/test_ticker.py

@@ -18,9 +18,9 @@ from yfinance.exceptions import YFPricesMissingError, YFInvalidPeriodError, YFNo
 
 
 import unittest
-import requests_cache
+# import requests_cache
 from typing import Union, Any, get_args, _GenericAlias
-from urllib.parse import urlparse, parse_qs, urlencode, urlunparse
+# from urllib.parse import urlparse, parse_qs, urlencode, urlunparse
 
 ticker_attributes = (
     ("major_holders", pd.DataFrame),
@@ -234,6 +234,23 @@ class TestTicker(unittest.TestCase):
         for attribute_name, attribute_type in ticker_attributes:
             assert_attribute_type(self, dat, attribute_name, attribute_type)
 
+    def test_ticker_with_symbol_mic(self):
+        equities = [
+            ("OR", "XPAR"),      # L'Oréal on Euronext Paris
+            ("AAPL", "XNYS"),    # Apple on NYSE
+            ("GOOGL", "XNAS"),   # Alphabet on NASDAQ
+            ("BMW", "XETR"),     # BMW on XETRA
+        ]
+        for eq in  equities:
+            # No exception = pass
+            yf.Ticker(eq)
+            yf.Ticker((eq[0], eq[1].lower()))
+
+    def test_ticker_with_symbol_mic_invalid(self):
+        with self.assertRaises(ValueError) as cm:
+            yf.Ticker(('ABC', 'XXXX'))
+        self.assertIn("Unknown MIC code: 'XXXX'", str(cm.exception))
+
 
 class TestTickerHistory(unittest.TestCase):
     session = None
@@ -284,36 +301,37 @@ class TestTickerHistory(unittest.TestCase):
                         else:
                             self.assertIsInstance(data.columns, pd.MultiIndex)
 
-    def test_no_expensive_calls_introduced(self):
-        """
-        Make sure calling history to get price data has not introduced more calls to yahoo than absolutely necessary.
-        As doing other type of scraping calls than "query2.finance.yahoo.com/v8/finance/chart" to yahoo website
-        will quickly trigger spam-block when doing bulk download of history data.
-        """
-        symbol = "GOOGL"
-        period = "1y"
-        with requests_cache.CachedSession(backend="memory") as session:
-            ticker = yf.Ticker(symbol, session=session)
-            ticker.history(period=period)
-            actual_urls_called = [r.url for r in session.cache.filter()]
+    # Hopefully one day we find an equivalent "requests_cache" that works with "curl_cffi"
+    # def test_no_expensive_calls_introduced(self):
+    #     """
+    #     Make sure calling history to get price data has not introduced more calls to yahoo than absolutely necessary.
+    #     As doing other type of scraping calls than "query2.finance.yahoo.com/v8/finance/chart" to yahoo website
+    #     will quickly trigger spam-block when doing bulk download of history data.
+    #     """
+    #     symbol = "GOOGL"
+    #     period = "1y"
+    #     with requests_cache.CachedSession(backend="memory") as session:
+    #         ticker = yf.Ticker(symbol, session=session)
+    #         ticker.history(period=period)
+    #         actual_urls_called = [r.url for r in session.cache.filter()]
 
-        # Remove 'crumb' argument
-        for i in range(len(actual_urls_called)):
-            u = actual_urls_called[i]
-            parsed_url = urlparse(u)
-            query_params = parse_qs(parsed_url.query)
-            query_params.pop('crumb', None)
-            query_params.pop('cookie', None)
-            u = urlunparse(parsed_url._replace(query=urlencode(query_params, doseq=True)))
-            actual_urls_called[i] = u
-        actual_urls_called = tuple(actual_urls_called)
+    #     # Remove 'crumb' argument
+    #     for i in range(len(actual_urls_called)):
+    #         u = actual_urls_called[i]
+    #         parsed_url = urlparse(u)
+    #         query_params = parse_qs(parsed_url.query)
+    #         query_params.pop('crumb', None)
+    #         query_params.pop('cookie', None)
+    #         u = urlunparse(parsed_url._replace(query=urlencode(query_params, doseq=True)))
+    #         actual_urls_called[i] = u
+    #     actual_urls_called = tuple(actual_urls_called)
 
-        expected_urls = [
-            f"https://query2.finance.yahoo.com/v8/finance/chart/{symbol}?interval=1d&range=1d",  # ticker's tz
-            f"https://query2.finance.yahoo.com/v8/finance/chart/{symbol}?events=div%2Csplits%2CcapitalGains&includePrePost=False&interval=1d&range={period}"
-        ]
-        for url in actual_urls_called:
-            self.assertTrue(url in expected_urls, f"Unexpected URL called: {url}")
+    #     expected_urls = [
+    #         f"https://query2.finance.yahoo.com/v8/finance/chart/{symbol}?interval=1d&range=1d",  # ticker's tz
+    #         f"https://query2.finance.yahoo.com/v8/finance/chart/{symbol}?events=div%2Csplits%2CcapitalGains&includePrePost=False&interval=1d&range={period}"
+    #     ]
+    #     for url in actual_urls_called:
+    #         self.assertTrue(url in expected_urls, f"Unexpected URL called: {url}")
 
     def test_dividends(self):
         data = self.ticker.dividends
@@ -889,8 +907,6 @@ class TestTickerAnalysts(unittest.TestCase):
         data = self.ticker.upgrades_downgrades
         self.assertIsInstance(data, pd.DataFrame, "data has wrong type")
         self.assertFalse(data.empty, "data is empty")
-        self.assertTrue(len(data.columns) == 4, "data has wrong number of columns")
-        self.assertCountEqual(data.columns.values.tolist(), ['Firm', 'ToGrade', 'FromGrade', 'Action'], "data has wrong column names")
         self.assertIsInstance(data.index, pd.DatetimeIndex, "data has wrong index type")
 
         data_cached = self.ticker.upgrades_downgrades
@@ -984,7 +1000,7 @@ class TestTickerInfo(unittest.TestCase):
         self.symbols.append("QCSTIX")  # good for testing, doesn't trade
         self.symbols += ["BTC-USD", "IWO", "VFINX", "^GSPC"]
         self.symbols += ["SOKE.IS", "ADS.DE"]  # detected bugs
-        self.symbols += ["EXTO", "NEPT" ] # Issues 2343 and 2363
+        self.symbols += ["EXTO" ] # Issues 2343
         self.tickers = [yf.Ticker(s, session=self.session) for s in self.symbols]
 
     def tearDown(self):
@@ -1000,7 +1016,6 @@ class TestTickerInfo(unittest.TestCase):
         self.assertIsInstance(data, dict, "data has wrong type")
         expected_keys = ['industry', 'currentPrice', 'exchange', 'floatShares', 'companyOfficers', 'bid']
         for k in expected_keys:
-            print(k)
             self.assertIn("symbol", data.keys(), f"Did not find expected key '{k}' in info dict")
         self.assertEqual(self.symbols[0], data["symbol"], "Wrong symbol value in info dict")
 
@@ -1037,15 +1052,6 @@ class TestTickerInfo(unittest.TestCase):
         self.assertCountEqual(['quoteType', 'symbol', 'underlyingSymbol', 'uuid', 'maxAge', 'trailingPegRatio'], data.keys())
         self.assertIn("trailingPegRatio", data.keys(), "Did not find expected key 'trailingPegRatio' in info dict")
 
-        # Test issue 2363 (Empty QuoteResponse)
-        data = self.tickers[11].info
-        expected_keys = ['maxAge', 'priceHint', 'previousClose', 'open', 'dayLow', 'dayHigh', 'regularMarketPreviousClose',
-                         'regularMarketOpen', 'regularMarketDayLow', 'regularMarketDayHigh', 'volume', 'regularMarketVolume',
-                         'bid', 'ask', 'bidSize', 'askSize', 'fiftyTwoWeekLow', 'fiftyTwoWeekHigh', 'currency', 'tradeable',
-                         'exchange', 'quoteType', 'symbol', 'underlyingSymbol', 'shortName', 'timeZoneFullName', 'timeZoneShortName',
-                         'uuid', 'gmtOffSetMilliseconds', 'trailingPegRatio']
-        self.assertCountEqual(expected_keys, data.keys())
-
     # def test_fast_info_matches_info(self):
     #     fast_info_keys = set()
     #     for ticker in self.tickers:

tests/test_utils.py

@@ -15,7 +15,7 @@ import pandas as pd
 
 import unittest
 
-from yfinance.utils import is_valid_period_format, _dts_in_same_interval
+from yfinance.utils import is_valid_period_format, _dts_in_same_interval, _parse_user_dt
 
 
 class TestPandas(unittest.TestCase):
@@ -169,6 +169,19 @@ class TestDateIntervalCheck(unittest.TestCase):
         dt3 = pd.Timestamp("2024-10-15 10:31:00")
         self.assertFalse(_dts_in_same_interval(dt1, dt3, "1min"))
 
+    def test_parse_user_dt(self):
+        exchange_tz = "US/Eastern"
+        dtstr = "2024-01-04"
+        epoch = 1704344400  # output of `pd.Timestamp("2024-01-04", tz="US/Eastern").timestamp()`
+        expected = pd.Timestamp(dtstr, tz=exchange_tz)
+
+        self.assertEqual(_parse_user_dt(epoch, exchange_tz), expected)
+        self.assertEqual(_parse_user_dt(dtstr, exchange_tz), expected)
+        self.assertEqual(_parse_user_dt(datetime(year=2024, month=1, day=4).date(), exchange_tz), expected)
+        self.assertEqual(_parse_user_dt(datetime(year=2024, month=1, day=4), exchange_tz), expected)
+        with self.assertRaises(ValueError):
+            self.assertEqual(_parse_user_dt(float(epoch), exchange_tz), expected)
+
 if __name__ == "__main__":
     unittest.main()
 

yfinance/__init__.py

@@ -25,12 +25,14 @@ from .lookup import Lookup
 from .ticker import Ticker
 from .tickers import Tickers
 from .multi import download
+from .live import WebSocket, AsyncWebSocket
 from .utils import enable_debug_mode
 from .cache import set_tz_cache_location
 from .domain.sector import Sector
 from .domain.industry import Industry
 from .domain.market import Market
 from .data import YfData
+from .config import YfConfig
 
 from .screener.query import EquityQuery, FundQuery
 from .screener.screener import screen, PREDEFINED_SCREENER_QUERIES
@@ -41,13 +43,15 @@ __author__ = "Ran Aroussi"
 import warnings
 warnings.filterwarnings('default', category=DeprecationWarning, module='^yfinance')
 
-__all__ = ['download', 'Market', 'Search', 'Lookup', 'Ticker', 'Tickers', 'enable_debug_mode', 'set_tz_cache_location', 'Sector', 'Industry']
+__all__ = ['download', 'Market', 'Search', 'Lookup', 'Ticker', 'Tickers', 'enable_debug_mode', 'set_tz_cache_location', 'Sector', 'Industry', 'WebSocket', 'AsyncWebSocket']
 # screener stuff:
 __all__ += ['EquityQuery', 'FundQuery', 'screen', 'PREDEFINED_SCREENER_QUERIES']
 
 # Config stuff:
 _NOTSET=object()
-def set_config(proxy=_NOTSET):
+def set_config(proxy=_NOTSET, hide_exceptions=_NOTSET):
     if proxy is not _NOTSET:
         YfData(proxy=proxy)
+    if hide_exceptions is not _NOTSET:
+        YfConfig(hide_exceptions=hide_exceptions)
 __all__ += ["set_config"]

yfinance/base.py

@@ -32,8 +32,11 @@ from curl_cffi import requests
 
 
 from . import utils, cache
+from .const import _MIC_TO_YAHOO_SUFFIX
 from .data import YfData
-from .exceptions import YFEarningsDateMissing, YFRateLimitError
+from .config import YfConfig
+from .exceptions import YFDataException, YFEarningsDateMissing, YFRateLimitError
+from .live import WebSocket
 from .scrapers.analysis import Analysis
 from .scrapers.fundamentals import Fundamentals
 from .scrapers.holders import Holders
@@ -43,11 +46,41 @@ from .scrapers.funds import FundsData
 
 from .const import _BASE_URL_, _ROOT_URL_, _QUERY1_URL_, _SENTINEL_
 
+from io import StringIO
+from bs4 import BeautifulSoup
+
 
 _tz_info_fetch_ctr = 0
 
 class TickerBase:
     def __init__(self, ticker, session=None, proxy=_SENTINEL_):
+        """
+        Initialize a Yahoo Finance Ticker object.
+
+        Args:
+            ticker (str | tuple[str, str]):
+                Yahoo Finance symbol (e.g. "AAPL")
+                or a tuple of (symbol, MIC) e.g. ('OR','XPAR')
+                (MIC = market identifier code)
+
+            session (requests.Session, optional):
+                Custom requests session.
+        """        
+        if isinstance(ticker, tuple):
+            if len(ticker) != 2:
+                raise ValueError("Ticker tuple must be (symbol, mic_code)")
+            base_symbol, mic_code = ticker
+            # ticker = yahoo_ticker(base_symbol, mic_code)
+            if mic_code.startswith('.'):
+                mic_code = mic_code[1:]
+            if mic_code.upper() not in _MIC_TO_YAHOO_SUFFIX:
+                raise ValueError(f"Unknown MIC code: '{mic_code}'")
+            sfx = _MIC_TO_YAHOO_SUFFIX[mic_code.upper()]
+            if sfx != '':
+                ticker = f'{base_symbol}.{sfx}'
+            else:
+                ticker = base_symbol
+
         self.ticker = ticker.upper()
         self.session = session or requests.Session(impersonate="chrome")
         self._tz = None
@@ -65,17 +98,22 @@ class TickerBase:
         if self.ticker == "":
             raise ValueError("Empty ticker name")
 
+        self._data: YfData = YfData(session=session)
+        if proxy is not _SENTINEL_:
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
+            self._data._set_proxy(proxy)
+
         # accept isin as ticker
         if utils.is_isin(self.ticker):
             isin = self.ticker
-            self.ticker = utils.get_ticker_by_isin(self.ticker, None, session)
+            c = cache.get_isin_cache()
+            self.ticker = c.lookup(isin)
+            if not self.ticker:
+                self.ticker = utils.get_ticker_by_isin(isin)
             if self.ticker == "":
                 raise ValueError(f"Invalid ISIN number: {isin}")
-
-        self._data: YfData = YfData(session=session)
-        if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
-            self._data._set_proxy(proxy)
+            if self.ticker:
+                c.store(isin, self.ticker)
 
         # self._price_history = PriceHistory(self._data, self.ticker)
         self._price_history = None  # lazy-load
@@ -87,6 +125,9 @@ class TickerBase:
 
         self._fast_info = None
 
+        self._message_handler = None
+        self.ws = None
+
     @utils.log_indent_decorator
     def history(self, *args, **kwargs) -> pd.DataFrame:
         return self._lazy_load_price_history().history(*args, **kwargs)
@@ -148,6 +189,8 @@ class TickerBase:
             # Must propagate this
             raise
         except Exception as e:
+            if not YfConfig().hide_exceptions:
+                raise
             logger.error(f"Failed to get ticker '{self.ticker}' reason: {e}")
             return None
         else:
@@ -159,6 +202,8 @@ class TickerBase:
                 try:
                     return data["chart"]["result"][0]["meta"]["exchangeTimezoneName"]
                 except Exception as err:
+                    if not YfConfig().hide_exceptions:
+                        raise
                     logger.error(f"Could not get exchangeTimezoneName for ticker '{self.ticker}' reason: {err}")
                     logger.debug("Got response: ")
                     logger.debug("-------------")
@@ -172,7 +217,7 @@ class TickerBase:
         Columns: period  strongBuy  buy  hold  sell  strongSell
         """
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         data = self._quote.recommendations
@@ -182,7 +227,7 @@ class TickerBase:
 
     def get_recommendations_summary(self, proxy=_SENTINEL_, as_dict=False):
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         return self.get_recommendations(as_dict=as_dict)
@@ -194,7 +239,7 @@ class TickerBase:
         Columns: firm toGrade fromGrade action
         """
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         data = self._quote.upgrades_downgrades
@@ -204,21 +249,21 @@ class TickerBase:
 
     def get_calendar(self, proxy=_SENTINEL_) -> dict:
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         return self._quote.calendar
 
     def get_sec_filings(self, proxy=_SENTINEL_) -> dict:
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         return self._quote.sec_filings
 
     def get_major_holders(self, proxy=_SENTINEL_, as_dict=False):
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         data = self._holders.major
@@ -228,7 +273,7 @@ class TickerBase:
 
     def get_institutional_holders(self, proxy=_SENTINEL_, as_dict=False):
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         data = self._holders.institutional
@@ -239,7 +284,7 @@ class TickerBase:
 
     def get_mutualfund_holders(self, proxy=_SENTINEL_, as_dict=False):
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         data = self._holders.mutualfund
@@ -250,7 +295,7 @@ class TickerBase:
 
     def get_insider_purchases(self, proxy=_SENTINEL_, as_dict=False):
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         data = self._holders.insider_purchases
@@ -261,7 +306,7 @@ class TickerBase:
 
     def get_insider_transactions(self, proxy=_SENTINEL_, as_dict=False):
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         data = self._holders.insider_transactions
@@ -272,7 +317,7 @@ class TickerBase:
 
     def get_insider_roster_holders(self, proxy=_SENTINEL_, as_dict=False):
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         data = self._holders.insider_roster
@@ -283,7 +328,7 @@ class TickerBase:
 
     def get_info(self, proxy=_SENTINEL_) -> dict:
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         data = self._quote.info
@@ -291,21 +336,16 @@ class TickerBase:
 
     def get_fast_info(self, proxy=_SENTINEL_):
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         if self._fast_info is None:
             self._fast_info = FastInfo(self)
         return self._fast_info
 
-    @property
-    def basic_info(self):
-        warnings.warn("'Ticker.basic_info' is deprecated and will be removed in future, Switch to 'Ticker.fast_info'", DeprecationWarning)
-        return self.fast_info
-
     def get_sustainability(self, proxy=_SENTINEL_, as_dict=False):
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         data = self._quote.sustainability
@@ -315,7 +355,7 @@ class TickerBase:
 
     def get_analyst_price_targets(self, proxy=_SENTINEL_) -> dict:
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         """
@@ -326,7 +366,7 @@ class TickerBase:
 
     def get_earnings_estimate(self, proxy=_SENTINEL_, as_dict=False):
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         """
@@ -338,7 +378,7 @@ class TickerBase:
 
     def get_revenue_estimate(self, proxy=_SENTINEL_, as_dict=False):
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         """
@@ -350,7 +390,7 @@ class TickerBase:
 
     def get_earnings_history(self, proxy=_SENTINEL_, as_dict=False):
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         """
@@ -366,7 +406,7 @@ class TickerBase:
         Columns:    current  7daysAgo  30daysAgo  60daysAgo  90daysAgo
         """
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         data = self._analysis.eps_trend
@@ -378,7 +418,7 @@ class TickerBase:
         Columns:    upLast7days  upLast30days  downLast7days  downLast30days
         """
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         data = self._analysis.eps_revisions
@@ -390,7 +430,7 @@ class TickerBase:
         Columns:    stock  industry  sector  index
         """
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         data = self._analysis.growth_estimates
@@ -407,7 +447,7 @@ class TickerBase:
                 Default is "yearly"
         """
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         if self._fundamentals.earnings is None:
@@ -434,7 +474,7 @@ class TickerBase:
                 Default is "yearly"
         """
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         data = self._fundamentals.financials.get_income_time_series(freq=freq)
@@ -448,14 +488,14 @@ class TickerBase:
 
     def get_incomestmt(self, proxy=_SENTINEL_, as_dict=False, pretty=False, freq="yearly"):
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         return self.get_income_stmt(proxy, as_dict, pretty, freq)
 
     def get_financials(self, proxy=_SENTINEL_, as_dict=False, pretty=False, freq="yearly"):
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         return self.get_income_stmt(proxy, as_dict, pretty, freq)
@@ -474,7 +514,7 @@ class TickerBase:
                 Default is "yearly"
         """
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
 
@@ -489,7 +529,7 @@ class TickerBase:
 
     def get_balancesheet(self, proxy=_SENTINEL_, as_dict=False, pretty=False, freq="yearly"):
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         return self.get_balance_sheet(proxy, as_dict, pretty, freq)
@@ -508,7 +548,7 @@ class TickerBase:
                 Default is "yearly"
         """
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
 
@@ -523,37 +563,37 @@ class TickerBase:
 
     def get_cashflow(self, proxy=_SENTINEL_, as_dict=False, pretty=False, freq="yearly"):
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
         return self.get_cash_flow(proxy, as_dict, pretty, freq)
 
     def get_dividends(self, proxy=_SENTINEL_, period="max") -> pd.Series:
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
         return self._lazy_load_price_history().get_dividends(period=period)
 
     def get_capital_gains(self, proxy=_SENTINEL_, period="max") -> pd.Series:
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
         return self._lazy_load_price_history().get_capital_gains(period=period)
 
     def get_splits(self, proxy=_SENTINEL_, period="max") -> pd.Series:
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
         return self._lazy_load_price_history().get_splits(period=period)
 
     def get_actions(self, proxy=_SENTINEL_, period="max") -> pd.Series:
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
         return self._lazy_load_price_history().get_actions(period=period)
 
     def get_shares(self, proxy=_SENTINEL_, as_dict=False) -> Union[pd.DataFrame, dict]:
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         data = self._fundamentals.shares
@@ -566,18 +606,16 @@ class TickerBase:
         logger = utils.get_yf_logger()
 
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         # Process dates
         tz = self._get_ticker_tz(timeout=10)
         dt_now = pd.Timestamp.utcnow().tz_convert(tz)
         if start is not None:
-            start_ts = utils._parse_user_dt(start, tz)
-            start = pd.Timestamp.fromtimestamp(start_ts).tz_localize("UTC").tz_convert(tz)
+            start = utils._parse_user_dt(start, tz)
         if end is not None:
-            end_ts = utils._parse_user_dt(end, tz)
-            end = pd.Timestamp.fromtimestamp(end_ts).tz_localize("UTC").tz_convert(tz)
+            end = utils._parse_user_dt(end, tz)
         if end is None:
             end = dt_now
         if start is None:
@@ -595,6 +633,8 @@ class TickerBase:
             json_data = self._data.cache_get(url=shares_url)
             json_data = json_data.json()
         except (_json.JSONDecodeError, requests.exceptions.RequestException):
+            if not YfConfig().hide_exceptions:
+                raise
             logger.error(f"{self.ticker}: Yahoo web request for share count failed")
             return None
         try:
@@ -602,6 +642,8 @@ class TickerBase:
         except KeyError:
             fail = False
         if fail:
+            if not YfConfig().hide_exceptions:
+                raise requests.exceptions.HTTPError("Yahoo web request for share count returned 'Bad Request'")
             logger.error(f"{self.ticker}: Yahoo web request for share count failed")
             return None
 
@@ -611,6 +653,8 @@ class TickerBase:
         try:
             df = pd.Series(shares_data[0]["shares_out"], index=pd.to_datetime(shares_data[0]["timestamp"], unit="s"))
         except Exception as e:
+            if not YfConfig().hide_exceptions:
+                raise
             logger.error(f"{self.ticker}: Failed to parse shares count data: {e}")
             return None
 
@@ -620,7 +664,7 @@ class TickerBase:
 
     def get_isin(self, proxy=_SENTINEL_) -> Optional[str]:
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         # *** experimental ***
@@ -666,7 +710,7 @@ class TickerBase:
         logger = utils.get_yf_logger()
 
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         tab_queryrefs = {
@@ -689,12 +733,12 @@ class TickerBase:
 
         data = self._data.post(url, body=payload)
         if data is None or "Will be right back" in data.text:
-            raise RuntimeError("*** YAHOO! FINANCE IS CURRENTLY DOWN! ***\n"
-                               "Our engineers are working quickly to resolve "
-                               "the issue. Thank you for your patience.")
+            raise YFDataException("*** YAHOO! FINANCE IS CURRENTLY DOWN! ***")
         try:
             data = data.json()
         except _json.JSONDecodeError:
+            if not YfConfig().hide_exceptions:
+                raise
             logger.error(f"{self.ticker}: Failed to retrieve the news and received faulty response instead.")
             data = {}
 
@@ -703,10 +747,111 @@ class TickerBase:
         self._news = [article for article in news if not article.get('ad', [])]
         return self._news
 
+    def get_earnings_dates(self, limit = 12, offset = 0) -> Optional[pd.DataFrame]:
+        return self._get_earnings_dates_using_scrape(limit, offset)
+
     @utils.log_indent_decorator
-    def get_earnings_dates(self, limit=12, proxy=_SENTINEL_) -> Optional[pd.DataFrame]:
+    def _get_earnings_dates_using_scrape(self, limit = 12, offset = 0) -> Optional[pd.DataFrame]:
+        """
+        Uses YfData.cache_get() to scrape earnings data from YahooFinance.
+        (https://finance.yahoo.com/calendar/earnings?symbol=INTC)
+    
+        Args:
+            limit (int): Number of rows to extract (max=100)
+            offset (int): if 0, search from future EPS estimates. 
+                          if 1, search from the most recent EPS. 
+                          if x, search from x'th recent EPS. 
+    
+        Returns:
+            pd.DataFrame in the following format.
+    
+                       EPS Estimate Reported EPS Surprise(%)
+            Date
+            2025-10-30         2.97            -           -
+            2025-07-22         1.73         1.54      -10.88
+            2025-05-06         2.63          2.7        2.57
+            2025-02-06         2.09         2.42       16.06
+            2024-10-31         1.92         1.55      -19.36
+            ...                 ...          ...         ...
+            2014-07-31         0.61         0.65        7.38
+            2014-05-01         0.55         0.68       22.92
+            2014-02-13         0.55         0.58        6.36
+            2013-10-31         0.51         0.54        6.86
+            2013-08-01         0.46          0.5        7.86
+        """
+        #####################################################
+        # Define Constants
+        #####################################################
+        if limit > 0 and limit <= 25:
+            size = 25
+        elif limit > 25 and limit <= 50:
+            size = 50
+        elif limit > 50 and limit <= 100:
+            size = 100
+        else:
+            raise ValueError("Please use limit <= 100")
+    
+        # Define the URL
+        url = "https://finance.yahoo.com/calendar/earnings?symbol={}&offset={}&size={}".format(
+            self.ticker, offset, size
+        )
+        #####################################################
+        # Get data
+        #####################################################
+        response = self._data.cache_get(url)
+    
+        #####################################################
+        # Response -> pd.DataFrame
+        #####################################################
+        # Parse the HTML content using BeautifulSoup
+        soup = BeautifulSoup(response.text, "html.parser")
+        # This page should have only one <table>
+        table = soup.find("table")
+        # If the table is found
+        if table:
+            # Get the HTML string of the table
+            table_html = str(table)
+    
+            # Wrap the HTML string in a StringIO object
+            html_stringio = StringIO(table_html)
+    
+            # Pass the StringIO object to pd.read_html()
+            df = pd.read_html(html_stringio, na_values=['-'])[0]
+    
+            # Drop redundant columns
+            df = df.drop(["Symbol", "Company"], axis=1)
+
+            # Backwards compatibility
+            df.rename(columns={'Surprise (%)': 'Surprise(%)'}, inplace=True)
+
+            df = df.dropna(subset="Earnings Date")
+
+            # Parse earnings date
+            # - Pandas doesn't like EDT, EST
+            df['Earnings Date'] = df['Earnings Date'].str.replace('EDT', 'America/New_York')
+            df['Earnings Date'] = df['Earnings Date'].str.replace('EST', 'America/New_York')
+            # - separate timezone string (last word)
+            dt_parts = df['Earnings Date'].str.rsplit(' ', n=1, expand=True)
+            dts = dt_parts[0]
+            tzs = dt_parts[1]
+            df['Earnings Date'] = pd.to_datetime(dts, format='%B %d, %Y at %I %p')
+            df['Earnings Date'] = pd.Series([dt.tz_localize(tz) for dt, tz in zip(df['Earnings Date'], tzs)])
+            df = df.set_index("Earnings Date")
+
+        else:
+            err_msg = "No earnings dates found, symbol may be delisted"
+            logger = utils.get_yf_logger()
+            logger.error(f'{self.ticker}: {err_msg}')
+            return None
+        return df
+
+    @utils.log_indent_decorator
+    def _get_earnings_dates_using_screener(self, limit=12, proxy=_SENTINEL_) -> Optional[pd.DataFrame]:
         """
         Get earning dates (future and historic)
+
+        In Summer 2025, Yahoo stopped updating the data at this endpoint.
+        So reverting to scraping HTML.
         
         Args:
             limit (int): max amount of upcoming and recent earnings dates to return.
@@ -718,7 +863,7 @@ class TickerBase:
         logger = utils.get_yf_logger()
 
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         clamped_limit = min(limit, 100)  # YF caps at 100, don't go higher
@@ -731,17 +876,11 @@ class TickerBase:
         params = {"lang": "en-US", "region": "US"}
         body = {
             "size": clamped_limit,
-            "query": {
-                "operator": "and",
-                "operands": [
-                    {"operator": "eq", "operands": ["ticker", self.ticker]},
-                    {"operator": "eq", "operands": ["eventtype", "2"]}
-                ]
-            },
+            "query": { "operator": "eq", "operands": ["ticker", self.ticker] },
             "sortField": "startdatetime",
             "sortType": "DESC",
             "entityIdType": "earnings",
-            "includeFields": ["startdatetime", "timeZoneShortName", "epsestimate", "epsactual", "epssurprisepct"]
+            "includeFields": ["startdatetime", "timeZoneShortName", "epsestimate", "epsactual", "epssurprisepct", "eventtype"]
         }
         response = self._data.post(url, params=params, body=body)
         json_data = response.json()
@@ -757,6 +896,14 @@ class TickerBase:
             logger.error(f'{self.ticker}: {err_msg}')
             return None
 
+        # Convert eventtype
+        # - 1 = earnings call (manually confirmed)
+        # - 2 = earnings report
+        # - 11 = stockholders meeting (manually confirmed)
+        df['Event Type'] = df['Event Type'].replace('^1$', 'Call', regex=True)
+        df['Event Type'] = df['Event Type'].replace('^2$', 'Earnings', regex=True)
+        df['Event Type'] = df['Event Type'].replace('^11$', 'Meeting', regex=True)
+
         # Calculate earnings date
         df['Earnings Date'] = pd.to_datetime(df['Event Start Date'])
         tz = self._get_ticker_tz(timeout=30)
@@ -779,17 +926,24 @@ class TickerBase:
 
     def get_history_metadata(self, proxy=_SENTINEL_) -> dict:
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         return self._lazy_load_price_history().get_history_metadata(proxy)
 
     def get_funds_data(self, proxy=_SENTINEL_) -> Optional[FundsData]:
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         if not self._funds_data:
             self._funds_data = FundsData(self._data, self.ticker)
         
         return self._funds_data
+
+    def live(self, message_handler=None, verbose=True):
+        self._message_handler = message_handler
+
+        self.ws = WebSocket(verbose=verbose)
+        self.ws.subscribe(self.ticker)
+        self.ws.listen(self._message_handler)

yfinance/cache.py

@@ -3,13 +3,15 @@ from threading import Lock
 import os as _os
 import platformdirs as _ad
 import atexit as _atexit
-import datetime as _datetime
+import datetime as _dt
 import pickle as _pkl
 
 from .utils import get_yf_logger
 
 _cache_init_lock = Lock()
 
+
+
 # --------------
 # TimeZone cache
 # --------------
@@ -105,7 +107,7 @@ _atexit.register(_TzDBManager.close_db)
 
 
 tz_db_proxy = _peewee.Proxy()
-class _KV(_peewee.Model):
+class _TZ_KV(_peewee.Model):
     key = _peewee.CharField(primary_key=True)
     value = _peewee.CharField(null=True)
     
@@ -146,11 +148,11 @@ class _TzCache:
         db.connect()
         tz_db_proxy.initialize(db)
         try:
-            db.create_tables([_KV])
+            db.create_tables([_TZ_KV])
         except _peewee.OperationalError as e:
             if 'WITHOUT' in str(e):
-                _KV._meta.without_rowid = False
-                db.create_tables([_KV])
+                _TZ_KV._meta.without_rowid = False
+                db.create_tables([_TZ_KV])
             else:
                 raise
         self.initialised = 1  # success
@@ -166,8 +168,8 @@ class _TzCache:
             return None
 
         try:
-            return _KV.get(_KV.key == key).value
-        except _KV.DoesNotExist:
+            return _TZ_KV.get(_TZ_KV.key == key).value
+        except _TZ_KV.DoesNotExist:
             return None
 
     def store(self, key, value):
@@ -185,18 +187,18 @@ class _TzCache:
             return
         try:
             if value is None:
-                q = _KV.delete().where(_KV.key == key)
+                q = _TZ_KV.delete().where(_TZ_KV.key == key)
                 q.execute()
                 return
             with db.atomic():
-                _KV.insert(key=key, value=value).execute()
+                _TZ_KV.insert(key=key, value=value).execute()
         except _peewee.IntegrityError:
             # Integrity error means the key already exists. Try updating the key.
             old_value = self.lookup(key)
             if old_value != value:
                 get_yf_logger().debug(f"Value for key {key} changed from {old_value} to {value}.")
                 with db.atomic():
-                    q = _KV.update(value=value).where(_KV.key == key)
+                    q = _TZ_KV.update(value=value).where(_TZ_KV.key == key)
                     q.execute()
 
 
@@ -301,16 +303,16 @@ class ISODateTimeField(_peewee.DateTimeField):
     # because user discovered peewee allowed an invalid datetime
     # to get written.
     def db_value(self, value):
-        if value and isinstance(value, _datetime.datetime):
+        if value and isinstance(value, _dt.datetime):
             return value.isoformat()
         return super().db_value(value)
     def python_value(self, value):
         if value and isinstance(value, str) and 'T' in value:
-            return _datetime.datetime.fromisoformat(value)
+            return _dt.datetime.fromisoformat(value)
         return super().python_value(value)
 class _CookieSchema(_peewee.Model):
     strategy = _peewee.CharField(primary_key=True)
-    fetch_date = ISODateTimeField(default=_datetime.datetime.now)
+    fetch_date = ISODateTimeField(default=_dt.datetime.now)
     
     # Which cookie type depends on strategy
     cookie_bytes = _peewee.BlobField()
@@ -374,7 +376,7 @@ class _CookieCache:
         try:
             data =  _CookieSchema.get(_CookieSchema.strategy == strategy)
             cookie = _pkl.loads(data.cookie_bytes)
-            return {'cookie':cookie, 'age':_datetime.datetime.now()-data.fetch_date}
+            return {'cookie':cookie, 'age':_dt.datetime.now()-data.fetch_date}
         except _CookieSchema.DoesNotExist:
             return None
 
@@ -415,6 +417,211 @@ def get_cookie_cache():
 
 
 
+# --------------
+# ISIN cache
+# --------------
+
+class _ISINCacheException(Exception):
+    pass
+
+
+class _ISINCacheDummy:
+    """Dummy cache to use if isin cache is disabled"""
+
+    def lookup(self, isin):
+        return None
+
+    def store(self, isin, tkr):
+        pass
+
+    @property
+    def tz_db(self):
+        return None
+
+
+class _ISINCacheManager:
+    _isin_cache = None
+
+    @classmethod
+    def get_isin_cache(cls):
+        if cls._isin_cache is None:
+            with _cache_init_lock:
+                cls._initialise()
+        return cls._isin_cache
+
+    @classmethod
+    def _initialise(cls, cache_dir=None):
+        cls._isin_cache = _ISINCache()
+
+
+class _ISINDBManager:
+    _db = None
+    _cache_dir = _os.path.join(_ad.user_cache_dir(), "py-yfinance")
+
+    @classmethod
+    def get_database(cls):
+        if cls._db is None:
+            cls._initialise()
+        return cls._db
+
+    @classmethod
+    def close_db(cls):
+        if cls._db is not None:
+            try:
+                cls._db.close()
+            except Exception:
+                # Must discard exceptions because Python trying to quit.
+                pass
+
+
+    @classmethod
+    def _initialise(cls, cache_dir=None):
+        if cache_dir is not None:
+            cls._cache_dir = cache_dir
+
+        if not _os.path.isdir(cls._cache_dir):
+            try:
+                _os.makedirs(cls._cache_dir)
+            except OSError as err:
+                raise _ISINCacheException(f"Error creating ISINCache folder: '{cls._cache_dir}' reason: {err}")
+        elif not (_os.access(cls._cache_dir, _os.R_OK) and _os.access(cls._cache_dir, _os.W_OK)):
+            raise _ISINCacheException(f"Cannot read and write in ISINCache folder: '{cls._cache_dir}'")
+
+        cls._db = _peewee.SqliteDatabase(
+            _os.path.join(cls._cache_dir, 'isin-tkr.db'),
+            pragmas={'journal_mode': 'wal', 'cache_size': -64}
+        )
+
+    @classmethod
+    def set_location(cls, new_cache_dir):
+        if cls._db is not None:
+            cls._db.close()
+            cls._db = None
+        cls._cache_dir = new_cache_dir
+
+    @classmethod
+    def get_location(cls):
+        return cls._cache_dir
+
+# close DB when Python exists
+_atexit.register(_ISINDBManager.close_db)
+
+
+isin_db_proxy = _peewee.Proxy()
+class _ISIN_KV(_peewee.Model):
+    key = _peewee.CharField(primary_key=True)
+    value = _peewee.CharField(null=True)
+    created_at = _peewee.DateTimeField(default=_dt.datetime.now)
+    
+    class Meta:
+        database = isin_db_proxy
+        without_rowid = True
+
+
+class _ISINCache:
+    def __init__(self):
+        self.initialised = -1
+        self.db = None
+        self.dummy = False
+
+    def get_db(self):
+        if self.db is not None:
+            return self.db
+
+        try:
+            self.db = _ISINDBManager.get_database()
+        except _ISINCacheException as err:
+            get_yf_logger().info(f"Failed to create ISINCache, reason: {err}. "
+                                 "ISINCache will not be used. "
+                                 "Tip: You can direct cache to use a different location with 'set_isin_cache_location(mylocation)'")
+            self.dummy = True
+            return None
+        return self.db
+
+    def initialise(self):
+        if self.initialised != -1:
+            return
+
+        db = self.get_db()
+        if db is None:
+            self.initialised = 0  # failure
+            return
+
+        db.connect()
+        isin_db_proxy.initialize(db)
+        try:
+            db.create_tables([_ISIN_KV])
+        except _peewee.OperationalError as e:
+            if 'WITHOUT' in str(e):
+                _ISIN_KV._meta.without_rowid = False
+                db.create_tables([_ISIN_KV])
+            else:
+                raise
+        self.initialised = 1  # success
+
+    def lookup(self, key):
+        if self.dummy:
+            return None
+
+        if self.initialised == -1:
+            self.initialise()
+
+        if self.initialised == 0:  # failure
+            return None
+
+        try:
+            return _ISIN_KV.get(_ISIN_KV.key == key).value
+        except _ISIN_KV.DoesNotExist:
+            return None
+
+    def store(self, key, value):
+        if self.dummy:
+            return
+
+        if self.initialised == -1:
+            self.initialise()
+
+        if self.initialised == 0:  # failure
+            return
+
+        db = self.get_db()
+        if db is None:
+            return
+        try:
+            if value is None:
+                q = _ISIN_KV.delete().where(_ISIN_KV.key == key)
+                q.execute()
+                return
+
+            # Remove existing rows with same value that are older than 1 week
+            one_week_ago = _dt.datetime.now() - _dt.timedelta(weeks=1)
+            old_rows_query = _ISIN_KV.delete().where(
+                (_ISIN_KV.value == value) & 
+                (_ISIN_KV.created_at < one_week_ago)
+            )
+            old_rows_query.execute()
+
+            with db.atomic():
+                _ISIN_KV.insert(key=key, value=value).execute()
+
+        except _peewee.IntegrityError:
+            # Integrity error means the key already exists. Try updating the key.
+            old_value = self.lookup(key)
+            if old_value != value:
+                get_yf_logger().debug(f"Value for key {key} changed from {old_value} to {value}.")
+                with db.atomic():
+                    q = _ISIN_KV.update(value=value, created_at=_dt.datetime.now()).where(_ISIN_KV.key == key)
+                    q.execute()
+
+
+def get_isin_cache():
+    return _ISINCacheManager.get_isin_cache()
+
+
+# --------------
+# Utils
+# --------------
+
 def set_cache_location(cache_dir: str):
     """
     Sets the path to create the "py-yfinance" cache folder in.
@@ -425,6 +632,7 @@ def set_cache_location(cache_dir: str):
     """
     _TzDBManager.set_location(cache_dir)
     _CookieDBManager.set_location(cache_dir)
+    _ISINDBManager.set_location(cache_dir)
 
 def set_tz_cache_location(cache_dir: str):
     set_cache_location(cache_dir)

yfinance/config.py

@@ -0,0 +1,33 @@
+import threading
+
+class SingletonMeta(type):
+    """
+    Metaclass that creates a Singleton instance.
+    """
+    _instances = {}
+    _lock = threading.Lock()
+
+    def __call__(cls, *args, **kwargs):
+        with cls._lock:
+            if cls not in cls._instances:
+                instance = super().__call__(*args, **kwargs)
+                cls._instances[cls] = instance
+            else:
+                # Update the existing instance
+                if 'hide_exceptions' in kwargs or (args and len(args) > 0):
+                    hide_exceptions = kwargs.get('hide_exceptions') if 'hide_exceptions' in kwargs else args[0]
+                    cls._instances[cls]._set_hide_exceptions(hide_exceptions)
+            return cls._instances[cls]
+
+
+class YfConfig(metaclass=SingletonMeta):
+    def __init__(self, hide_exceptions=True):
+        self._hide_exceptions = hide_exceptions
+
+    def _set_hide_exceptions(self, hide_exceptions):
+        self._hide_exceptions = hide_exceptions
+
+    @property
+    def hide_exceptions(self):
+        return self._hide_exceptions
+    

yfinance/const.py

@@ -161,151 +161,221 @@ quote_summary_valid_modules = (
 
 # map last updated as of 2024.09.18
 SECTOR_INDUSTY_MAPPING = {
-    'basic-materials': {'specialty-chemicals',
-                        'gold',
-                        'building-materials',
-                        'copper',
-                        'steel',
-                        'agricultural-inputs',
-                        'chemicals',
-                        'other-industrial-metals-mining',
-                        'lumber-wood-production',
-                        'aluminum',
-                        'other-precious-metals-mining',
-                        'coking-coal',
-                        'paper-paper-products',
-                        'silver'},
-    'communication-services': {'internet-content-information',
-                                'telecom-services',
-                                'entertainment',
-                                'electronic-gaming-multimedia',
-                                'advertising-agencies',
-                                'broadcasting',
-                                'publishing'},
-    'consumer-cyclical': {'internet-retail',
-                            'auto-manufacturers',
-                            'restaurants',
-                            'home-improvement-retail',
-                            'travel-services',
-                            'specialty-retail',
-                            'apparel-retail',
-                            'residential-construction',
-                            'footwear-accessories',
-                            'packaging-containers',
-                            'lodging',
-                            'auto-parts',
-                            'auto-truck-dealerships',
-                            'gambling',
-                            'resorts-casinos',
-                            'leisure',
-                            'apparel-manufacturing',
-                            'personal-services',
-                            'furnishings-fixtures-appliances',
-                            'recreational-vehicles',
-                            'luxury-goods',
-                            'department-stores',
-                            'textile-manufacturing'},
-    'consumer-defensive': {'discount-stores',
-                            'beverages-non-alcoholic',
-                            'household-personal-products',
-                            'packaged-foods',
-                            'tobacco',
-                            'confectioners',
-                            'farm-products',
-                            'food-distribution',
-                            'grocery-stores',
-                            'beverages-brewers',
-                            'education-training-services',
-                            'beverages-wineries-distilleries'},
-    'energy': {'oil-gas-integrated',
-                'oil-gas-midstream',
-                'oil-gas-e-p',
-                'oil-gas-equipment-services',
-                'oil-gas-refining-marketing',
-                'uranium',
-                'oil-gas-drilling',
-                'thermal-coal'},
-    'financial-services': {'banks-diversified',
-                            'credit-services',
-                            'asset-management',
-                            'insurance-diversified',
-                            'banks-regional',
-                            'capital-markets',
-                            'financial-data-stock-exchanges',
-                            'insurance-property-casualty',
-                            'insurance-brokers',
-                            'insurance-life',
-                            'insurance-specialty',
-                            'mortgage-finance',
-                            'insurance-reinsurance',
-                            'shell-companies',
-                            'financial-conglomerates'},
-    'healthcare': {'drug-manufacturers-general',
-                    'healthcare-plans',
-                    'biotechnology',
-                    'medical-devices',
-                    'diagnostics-research',
-                    'medical-instruments-supplies',
-                    'medical-care-facilities',
-                    'drug-manufacturers-specialty-generic',
-                    'health-information-services',
-                    'medical-distribution',
-                    'pharmaceutical-retailers'},
-    'industrials': {'aerospace-defense',
-                    'specialty-industrial-machinery',
-                    'railroads',
-                    'building-products-equipment',
-                    'farm-heavy-construction-machinery',
-                    'specialty-business-services',
-                    'integrated-freight-logistics',
-                    'waste-management',
-                    'conglomerates',
-                    'industrial-distribution',
-                    'engineering-construction',
-                    'rental-leasing-services',
-                    'consulting-services',
-                    'trucking',
-                    'electrical-equipment-parts',
-                    'airlines',
-                    'tools-accessories',
-                    'pollution-treatment-controls',
-                    'security-protection-services',
-                    'marine-shipping',
-                    'metal-fabrication',
-                    'infrastructure-operations',
-                    'staffing-employment-services',
-                    'airports-air-services',
-                    'business-equipment-supplies'},
-    'real-estate': {'reit-specialty',
-                    'reit-industrial',
-                    'reit-retail',
-                    'reit-residential',
-                    'reit-healthcare-facilities',
-                    'real-estate-services',
-                    'reit-office',
-                    'reit-diversified',
-                    'reit-mortgage',
-                    'reit-hotel-motel',
-                    'real-estate-development',
-                    'real-estate-diversified'},
-    'technology': {'software-infrastructure',
-                    'semiconductors',
-                    'consumer-electronics',
-                    'software-application',
-                    'information-technology-services',
-                    'semiconductor-equipment-materials',
-                    'communication-equipment',
-                    'computer-hardware',
-                    'electronic-components',
-                    'scientific-technical-instruments',
-                    'solar',
-                    'electronics-computer-distribution'},
-    'utilities': {'utilities-regulated-electric',
-                    'utilities-renewable',
-                    'utilities-diversified',
-                    'utilities-regulated-gas',
-                    'utilities-independent-power-producers',
-                    'utilities-regulated-water'}
+    'Basic Materials': {'Specialty Chemicals',
+                        'Gold',
+                        'Building Materials',
+                        'Copper',
+                        'Steel',
+                        'Agricultural Inputs',
+                        'Chemicals',
+                        'Other Industrial Metals & Mining',
+                        'Lumber & Wood Production',
+                        'Aluminum',
+                        'Other Precious Metals & Mining',
+                        'Coking Coal',
+                        'Paper & Paper Products',
+                        'Silver'},
+    'Communication Services': {'Advertising Agencies',
+                                'Broadcasting',
+                                'Electronic Gaming & Multimedia',
+                                'Entertainment',
+                                'Internet Content & Information',
+                                'Publishing',
+                                'Telecom Services'},
+    'Consumer Cyclical': {'Apparel Manufacturing',
+                            'Apparel Retail',
+                            'Auto & Truck Dealerships',
+                            'Auto Manufacturers',
+                            'Auto Parts',
+                            'Department Stores',
+                            'Footwear & Accessories',
+                            'Furnishings, Fixtures & Appliances',
+                            'Gambling',
+                            'Home Improvement Retail',
+                            'Internet Retail',
+                            'Leisure',
+                            'Lodging',
+                            'Luxury Goods',
+                            'Packaging & Containers',
+                            'Personal Services',
+                            'Recreational Vehicles',
+                            'Residential Construction',
+                            'Resorts & Casinos',
+                            'Restaurants',
+                            'Specialty Retail',
+                            'Textile Manufacturing',
+                            'Travel Services'},
+    'Consumer Defensive': {'Beverages - Brewers',
+                            'Beverages - Non-Alcoholic',
+                            'Beverages - Wineries & Distilleries',
+                            'Confectioners',
+                            'Discount Stores',
+                            'Education & Training Services',
+                            'Farm Products',
+                            'Food Distribution',
+                            'Grocery Stores',
+                            'Household & Personal Products',
+                            'Packaged Foods',
+                            'Tobacco'},
+    'Energy': {'Oil Gas Drilling',
+                'Oil Gas E P',
+                'Oil Gas Equipment Services',
+                'Oil Gas Integrated',
+                'Oil Gas Midstream',
+                'Oil Gas Refining Marketing',
+                'Thermal Coal',
+                'Uranium'},
+    'Financial Services': {'Asset Management',
+                            'Banks Diversified',
+                            'Banks Regional',
+                            'Capital Markets',
+                            'Credit Services',
+                            'Financial Conglomerates',
+                            'Financial Data Stock Exchanges',
+                            'Insurance Brokers',
+                            'Insurance Diversified',
+                            'Insurance Life',
+                            'Insurance Property Casualty',
+                            'Insurance Reinsurance',
+                            'Insurance Specialty',
+                            'Mortgage Finance',
+                            'Shell Companies'},
+    'Healthcare': {'Biotechnology',
+                    'Diagnostics Research',
+                    'Drug Manufacturers General',
+                    'Drug Manufacturers Specialty Generic',
+                    'Health Information Services',
+                    'Healthcare Plans',
+                    'Medical Care Facilities',
+                    'Medical Devices',
+                    'Medical Distribution',
+                    'Medical Instruments Supplies',
+                    'Pharmaceutical Retailers'},
+    'Industrials': {'Aerospace Defense',
+                    'Airlines',
+                    'Airports Air Services',
+                    'Building Products Equipment',
+                    'Business Equipment Supplies',
+                    'Conglomerates',
+                    'Consulting Services',
+                    'Electrical Equipment Parts',
+                    'Engineering Construction',
+                    'Farm Heavy Construction Machinery',
+                    'Industrial Distribution',
+                    'Infrastructure Operations',
+                    'Integrated Freight Logistics',
+                    'Marine Shipping',
+                    'Metal Fabrication',
+                    'Pollution Treatment Controls',
+                    'Railroads',
+                    'Rental Leasing Services',
+                    'Security Protection Services',
+                    'Specialty Business Services',
+                    'Specialty Industrial Machinery',
+                    'Staffing Employment Services',
+                    'Tools Accessories',
+                    'Trucking',
+                    'Waste Management'},
+    'Real Estate': {'Real Estate Development',
+                    'Real Estate Diversified',
+                    'Real Estate Services',
+                    'Reit Diversified',
+                    'Reit Healthcare Facilities',
+                    'Reit Hotel Motel',
+                    'Reit Industrial',
+                    'Reit Mortgage',
+                    'Reit Office',
+                    'Reit Residential',
+                    'Reit Retail',
+                    'Reit Specialty'},
+    'Technology': {'Communication Equipment',
+                    'Computer Hardware',
+                    'Consumer Electronics',
+                    'Electronic Components',
+                    'Electronics Computer Distribution',
+                    'Information Technology Services',
+                    'Scientific Technical Instruments',
+                    'Semiconductor Equipment Materials',
+                    'Semiconductors',
+                    'Software Application',
+                    'Software Infrastructure',
+                    'Solar'},
+    'Utilities': {'Utilities Diversified',
+                    'Utilities Independent Power Producers',
+                    'Utilities Regulated Electric',
+                    'Utilities Regulated Gas',
+                    'Utilities Regulated Water',
+                    'Utilities Renewable'},
+}
+SECTOR_INDUSTY_MAPPING_LC = {}
+for k in SECTOR_INDUSTY_MAPPING.keys():
+    k2 = k.lower().replace('& ', '').replace('- ', '').replace(', ', ' ').replace(' ', '-')
+    SECTOR_INDUSTY_MAPPING_LC[k2] = []
+    for v in SECTOR_INDUSTY_MAPPING[k]:
+        v2 = v.lower().replace('& ', '').replace('- ', '').replace(', ', ' ').replace(' ', '-')
+        SECTOR_INDUSTY_MAPPING_LC[k2].append(v2)
+
+# _MIC_TO_YAHOO_SUFFIX maps Market Identifier Codes (MIC) to Yahoo Finance market suffixes.
+# c.f. :
+# https://help.yahoo.com/kb/finance-for-web/SLN2310.html;_ylt=AwrJKiCZFo9g3Y8AsDWPAwx.;_ylu=Y29sbwMEcG9zAzEEdnRpZAMEc2VjA3Ny?locale=en_US
+# https://www.iso20022.org/market-identifier-codes
+
+_MIC_TO_YAHOO_SUFFIX = {
+    'XCBT': 'CBT', 'XCME': 'CME', 'IFUS': 'NYB', 'CECS': 'CMX', 'XNYM': 'NYM', 'XNYS': '', 'XNAS': '',  # United States
+    'XBUE': 'BA',  # Argentina
+    'XVIE': 'VI',  # Austria
+    'XASX': 'AX', 'XAUS': 'XA',  # Australia
+    'XBRU': 'BR',  # Belgium
+    'BVMF': 'SA',  # Brazil
+    'CNSX': 'CN', 'NEOE': 'NE', 'XTSE': 'TO', 'XTSX': 'V',  # Canada
+    'XSGO': 'SN',  # Chile
+    'XSHG': 'SS', 'XSHE': 'SZ',  # China
+    'XBOG': 'CL',  # Colombia
+    'XPRA': 'PR',  # Czech Republic
+    'XCSE': 'CO',  # Denmark
+    'XCAI': 'CA',  # Egypt
+    'XTAL': 'TL',  # Estonia
+    'CEUX': 'XD', 'XEUR': 'NX',  # Europe (Cboe Europe, Euronext)
+    'XHEL': 'HE',  # Finland
+    'XPAR': 'PA',  # France
+    'XBER': 'BE', 'XBMS': 'BM', 'XDUS': 'DU', 'XFRA': 'F', 'XHAM': 'HM', 'XHAN': 'HA', 'XMUN': 'MU', 'XSTU': 'SG', 'XETR': 'DE',  # Germany
+    'XATH': 'AT',  # Greece
+    'XHKG': 'HK',  # Hong Kong
+    'XBUD': 'BD',  # Hungary
+    'XICE': 'IC',  # Iceland
+    'XBOM': 'BO', 'XNSE': 'NS',  # India
+    'XIDX': 'JK',  # Indonesia
+    'XDUB': 'IR',  # Ireland
+    'XTAE': 'TA',  # Israel
+    'MTAA': 'MI', 'EUTL': 'TI',  # Italy
+    'XTKS': 'T',  # Japan
+    'XKFE': 'KW',  # Kuwait
+    'XRIS': 'RG',  # Latvia
+    'XVIL': 'VS',  # Lithuania
+    'XKLS': 'KL',  # Malaysia
+    'XMEX': 'MX',  # Mexico
+    'XAMS': 'AS',  # Netherlands
+    'XNZE': 'NZ',  # New Zealand
+    'XOSL': 'OL',  # Norway
+    'XPHS': 'PS',  # Philippines
+    'XWAR': 'WA',  # Poland
+    'XLIS': 'LS',  # Portugal
+    'XQAT': 'QA',  # Qatar
+    'XBSE': 'RO',  # Romania
+    'XSES': 'SI',  # Singapore
+    'XJSE': 'JO',  # South Africa
+    'XKRX': 'KS', 'KQKS': 'KQ',  # South Korea
+    'BMEX': 'MC',  # Spain
+    'XTAD': 'SAU',  # Saudi Arabia
+    'XSTO': 'ST',  # Sweden
+    'XSWX': 'SW',  # Switzerland
+    'ROCO': 'TWO', 'XTAI': 'TW',  # Taiwan
+    'XBKK': 'BK',  # Thailand
+    'XIST': 'IS',  # Turkey
+    'XDFM': 'AE',  # UAE
+    'AQXE': 'AQ', 'XCHI': 'XC', 'XLON': 'L', 'ILSE': 'IL',  # United Kingdom
+    'XCAR': 'CR',  # Venezuela
+    'XSTC': 'VN'  # Vietnam
 }
 
 def merge_two_level_dicts(dict1, dict2):
@@ -380,6 +450,7 @@ EQUITY_SCREENER_EQ_MAP = {
         'se': {'STO'},
         'sg': {'SES'},
         'sr': {},
+        'sw': {'EBS'},
         'th': {'SET'},
         'tr': {'IST'},
         'tw': {'TAI', 'TWO'},
@@ -393,6 +464,7 @@ EQUITY_SCREENER_EQ_MAP = {
         "Real Estate", "Technology", "Energy", "Utilities", "Financial Services",
         "Consumer Defensive", "Consumer Cyclical"
     },
+    "industry": SECTOR_INDUSTY_MAPPING,
     "peer_group": {
         "US Fund Equity Energy",
         "US CE Convertibles",
@@ -529,7 +601,8 @@ EQUITY_SCREENER_FIELDS = {
     "eq_fields": {
         "region",
         "sector",
-        "peer_group"}, 
+        "peer_group",
+        "industry"}, 
     "price":{
         "lastclosemarketcap.lasttwelvemonths",
         "percentchange",

yfinance/data.py

@@ -1,8 +1,8 @@
 import functools
-import random
 from functools import lru_cache
 
 from curl_cffi import requests
+from urllib.parse import urlsplit, urljoin
 from bs4 import BeautifulSoup
 import datetime
 
@@ -11,8 +11,7 @@ from frozendict import frozendict
 from . import utils, cache
 import threading
 
-from .const import USER_AGENTS
-from .exceptions import YFRateLimitError
+from .exceptions import YFException, YFDataException, YFRateLimitError
 
 cache_maxsize = 64
 
@@ -66,9 +65,6 @@ class YfData(metaclass=SingletonMeta):
     Have one place to retrieve data from Yahoo API in order to ease caching and speed up operations.
     Singleton means one session one cookie shared by all threads.
     """
-    user_agent_headers = {
-        'User-Agent': random.choice(USER_AGENTS)
-    }
 
     def __init__(self, session=None, proxy=None):
         self._crumb = None
@@ -85,18 +81,12 @@ class YfData(metaclass=SingletonMeta):
         self._set_session(session or requests.Session(impersonate="chrome"))
         self._set_proxy(proxy)
 
-        utils.get_yf_logger().debug(f"Using User-Agent: {self.user_agent_headers['User-Agent']}")
-
     def _set_session(self, session):
         if session is None:
             return
-        with self._cookie_lock:
-            self._session = session
-            if self._proxy is not None:
-                self._session.proxies = self._proxy
 
         try:
-            self._session.cache
+            session.cache
         except AttributeError:
             # Not caching
             self._session_is_caching = False
@@ -105,8 +95,16 @@ class YfData(metaclass=SingletonMeta):
             # Can't simply use a non-caching session to fetch cookie & crumb,
             # because then the caching-session won't have cookie.
             self._session_is_caching = True
-            from requests_cache import DO_NOT_CACHE
-            self._expire_after = DO_NOT_CACHE
+            # But since switch to curl_cffi, can't use requests_cache with it.
+            raise YFDataException("request_cache sessions don't work with curl_cffi, which is necessary now for Yahoo API. Solution: stop setting session, let YF handle.")
+
+        if not isinstance(session, requests.session.Session):
+            raise YFDataException(f"Yahoo API requires curl_cffi session not {type(session)}. Solution: stop setting session, let YF handle.")
+
+        with self._cookie_lock:
+            self._session = session
+            if self._proxy is not None:
+                self._session.proxies = self._proxy
 
     def _set_proxy(self, proxy=None):
         with self._cookie_lock:
@@ -140,58 +138,85 @@ class YfData(metaclass=SingletonMeta):
         if not have_lock:
             self._cookie_lock.release()
 
-    def _save_session_cookies(self):
-        try:
-            cache.get_cookie_cache().store('csrf', self._session.cookies)
-        except Exception:
+    @utils.log_indent_decorator
+    def _save_cookie_curlCffi(self):
+        if self._session is None:
             return False
+        cookies = self._session.cookies.jar._cookies
+        if len(cookies) == 0:
+            return False
+        yh_domains = [k for k in cookies.keys() if 'yahoo' in k]
+        if len(yh_domains) > 1:
+            # Possible when cookie fetched with CSRF method. Discard consent cookie.
+            yh_domains = [k for k in yh_domains if 'consent' not in k]
+        if len(yh_domains) > 1:
+            utils.get_yf_logger().debug(f'Multiple Yahoo cookies, not sure which to cache: {yh_domains}')
+            return False
+        if len(yh_domains) == 0:
+            return False
+        yh_domain = yh_domains[0]
+        yh_cookie = {yh_domain: cookies[yh_domain]}
+        cache.get_cookie_cache().store('curlCffi', yh_cookie)
         return True
 
-    def _load_session_cookies(self):
-        cookie_dict = cache.get_cookie_cache().lookup('csrf')
-        if cookie_dict is None:
+    @utils.log_indent_decorator
+    def _load_cookie_curlCffi(self):
+        if self._session is None:
             return False
-        # Periodically refresh, 24 hours seems fair.
-        if cookie_dict['age'] > datetime.timedelta(days=1):
+        cookie_dict = cache.get_cookie_cache().lookup('curlCffi')
+        if cookie_dict is None or len(cookie_dict) == 0:
             return False
-        self._session.cookies.update(cookie_dict['cookie'])
-        utils.get_yf_logger().debug('loaded persistent cookie')
-
-    def _save_cookie_basic(self, cookie):
-        try:
-            cache.get_cookie_cache().store('basic', cookie)
-        except Exception:
+        cookies = cookie_dict['cookie']
+        domain = list(cookies.keys())[0]
+        cookie = cookies[domain]['/']['A3']
+        expiry_ts = cookie.expires
+        if expiry_ts > 2e9:
+            # convert ms to s
+            expiry_ts //= 1e3
+        expiry_dt = datetime.datetime.fromtimestamp(expiry_ts, tz=datetime.timezone.utc)
+        expired = expiry_dt < datetime.datetime.now(datetime.timezone.utc)
+        if expired:
+            utils.get_yf_logger().debug('cached cookie expired')
             return False
+        self._session.cookies.jar._cookies.update(cookies)
+        self._cookie = cookie
         return True
-    def _load_cookie_basic(self):
-        cookie_dict = cache.get_cookie_cache().lookup('basic')
-        if cookie_dict is None:
-            return None
-        # Periodically refresh, 24 hours seems fair.
-        if cookie_dict['age'] > datetime.timedelta(days=1):
-            return None
-        utils.get_yf_logger().debug('loaded persistent cookie')
-        return cookie_dict['cookie']
 
+    @utils.log_indent_decorator
     def _get_cookie_basic(self, timeout=30):
+        if self._cookie is not None:
+            utils.get_yf_logger().debug('reusing cookie')
+            return True
+        elif self._load_cookie_curlCffi():
+            utils.get_yf_logger().debug('reusing persistent cookie')
+            return True
+
         # To avoid infinite recursion, do NOT use self.get()
         # - 'allow_redirects' copied from @psychoz971 solution - does it help USA?
-        response = self._session.get(
-            url='https://fc.yahoo.com',
-            headers=self.user_agent_headers,
-            timeout=timeout,
-            allow_redirects=True)
+        try:
+            self._session.get(
+                url='https://fc.yahoo.com',
+                timeout=timeout,
+                allow_redirects=True)
+        except requests.exceptions.DNSError as e:
+            # Possible because url on some privacy/ad blocklists.
+            # Can ignore because have second strategy.
+            utils.get_yf_logger().debug("Handling DNS error on cookie fetch: " + str(e))
+            return False
+        self._save_cookie_curlCffi()
+        return True
 
+    @utils.log_indent_decorator
     def _get_crumb_basic(self, timeout=30):
         if self._crumb is not None:
             utils.get_yf_logger().debug('reusing crumb')
             return self._crumb
 
-        self._get_cookie_basic()        
+        if not self._get_cookie_basic():
+            return None
         # - 'allow_redirects' copied from @psychoz971 solution - does it help USA?
         get_args = {
             'url': "https://query1.finance.yahoo.com/v1/test/getcrumb",
-            'headers': self.user_agent_headers,
             'timeout': timeout,
             'allow_redirects': True
         }
@@ -201,6 +226,10 @@ class YfData(metaclass=SingletonMeta):
         else:
             crumb_response = self._session.get(**get_args)
         self._crumb = crumb_response.text
+        if crumb_response.status_code == 429 or "Too Many Requests" in self._crumb:
+            utils.get_yf_logger().debug(f"Didn't receive crumb {self._crumb}")
+            raise YFRateLimitError()
+
         if self._crumb is None or '<html>' in self._crumb:
             utils.get_yf_logger().debug("Didn't receive crumb")
             return None
@@ -210,22 +239,22 @@ class YfData(metaclass=SingletonMeta):
 
     @utils.log_indent_decorator
     def _get_cookie_and_crumb_basic(self, timeout):
-        self._get_cookie_basic(timeout)
-        crumb = self._get_crumb_basic(timeout)
-        return crumb
+        if not self._get_cookie_basic(timeout):
+            return None
+        return self._get_crumb_basic(timeout)
 
+    @utils.log_indent_decorator
     def _get_cookie_csrf(self, timeout):
         if self._cookie is not None:
             utils.get_yf_logger().debug('reusing cookie')
             return True
 
-        elif self._load_session_cookies():
+        elif self._load_cookie_curlCffi():
             utils.get_yf_logger().debug('reusing persistent cookie')
             self._cookie = True
             return True
 
         base_args = {
-            'headers': self.user_agent_headers,
             'timeout': timeout}
 
         get_args = {**base_args, 'url': 'https://guce.yahoo.com/consent'}
@@ -280,7 +309,7 @@ class YfData(metaclass=SingletonMeta):
             # No idea why happens, but handle nicely so can switch to other cookie method.
             utils.get_yf_logger().debug('_get_cookie_csrf() encountering requests.exceptions.ChunkedEncodingError, aborting')
         self._cookie = True
-        self._save_session_cookies()
+        self._save_cookie_curlCffi()
         return True
 
     @utils.log_indent_decorator
@@ -297,7 +326,6 @@ class YfData(metaclass=SingletonMeta):
 
         get_args = {
             'url': 'https://query2.finance.yahoo.com/v1/test/getcrumb',
-            'headers': self.user_agent_headers,
             'timeout': timeout}
         if self._session_is_caching:
             get_args['expire_after'] = self._expire_after
@@ -306,6 +334,10 @@ class YfData(metaclass=SingletonMeta):
             r = self._session.get(**get_args)
         self._crumb = r.text
 
+        if r.status_code == 429 or "Too Many Requests" in self._crumb:
+            utils.get_yf_logger().debug(f"Didn't receive crumb {self._crumb}")
+            raise YFRateLimitError()
+
         if self._crumb is None or '<html>' in self._crumb or self._crumb == '':
             utils.get_yf_logger().debug("Didn't receive crumb")
             return None
@@ -337,15 +369,25 @@ class YfData(metaclass=SingletonMeta):
         return crumb, strategy
 
     @utils.log_indent_decorator
-    def get(self, url, user_agent_headers=None, params=None, timeout=30):
-        return self._make_request(url, request_method = self._session.get, user_agent_headers=user_agent_headers, params=params, timeout=timeout)
-    
+    def get(self, url, params=None, timeout=30):
+        response = self._make_request(url, request_method = self._session.get, params=params, timeout=timeout)
+
+        # Accept cookie-consent if redirected to consent page
+        if not self._is_this_consent_url(response.url):
+            # "Consent Page not detected"
+            pass
+        else:
+            # "Consent Page detected"
+            response = self._accept_consent_form(response, timeout)
+
+        return response
+
     @utils.log_indent_decorator
-    def post(self, url, body, user_agent_headers=None, params=None, timeout=30):
-        return self._make_request(url, request_method = self._session.post, user_agent_headers=user_agent_headers, body=body, params=params, timeout=timeout)
-    
+    def post(self, url, body, params=None, timeout=30):
+        return self._make_request(url, request_method = self._session.post, body=body, params=params, timeout=timeout)
+
     @utils.log_indent_decorator
-    def _make_request(self, url, request_method, user_agent_headers=None, body=None, params=None, timeout=30):
+    def _make_request(self, url, request_method, body=None, params=None, timeout=30):
         # Important: treat input arguments as immutable.
 
         if len(url) > 200:
@@ -357,7 +399,7 @@ class YfData(metaclass=SingletonMeta):
         if params is None:
             params = {}
         if 'crumb' in params:
-            raise Exception("Don't manually add 'crumb' to params dict, let data.py handle it")
+            raise YFException("Don't manually add 'crumb' to params dict, let data.py handle it")
 
         crumb, strategy = self._get_cookie_and_crumb()
         if crumb is not None:
@@ -368,13 +410,12 @@ class YfData(metaclass=SingletonMeta):
         request_args = {
             'url': url,
             'params': {**params, **crumbs},
-            'timeout': timeout,
-            'headers': user_agent_headers or self.user_agent_headers
+            'timeout': timeout
         }
 
         if body:
             request_args['json'] = body
-            
+
         response = request_method(**request_args)
         utils.get_yf_logger().debug(f'response code={response.status_code}')
         if response.status_code >= 400:
@@ -396,11 +437,91 @@ class YfData(metaclass=SingletonMeta):
 
     @lru_cache_freezeargs
     @lru_cache(maxsize=cache_maxsize)
-    def cache_get(self, url, user_agent_headers=None, params=None, timeout=30):
-        return self.get(url, user_agent_headers, params, timeout)
+    def cache_get(self, url, params=None, timeout=30):
+        return self.get(url, params, timeout)
 
-    def get_raw_json(self, url, user_agent_headers=None, params=None, timeout=30):
+    def get_raw_json(self, url, params=None, timeout=30):
         utils.get_yf_logger().debug(f'get_raw_json(): {url}')
-        response = self.get(url, user_agent_headers=user_agent_headers, params=params, timeout=timeout)
+        response = self.get(url, params=params, timeout=timeout)
         response.raise_for_status()
-        return response.json()
+        return response.json()
+
+    def _is_this_consent_url(self, response_url: str) -> bool:
+        """
+        Check if given response_url is consent page
+
+        Args:
+            response_url (str) : response.url
+    
+        Returns:
+            True : This is cookie-consent page
+            False : This is not cookie-consent page
+        """
+        try:
+            return urlsplit(response_url).hostname and urlsplit(
+                response_url
+            ).hostname.endswith("consent.yahoo.com")
+        except Exception:
+            return False
+
+    def _accept_consent_form(
+        self, consent_resp: requests.Response, timeout: int
+    ) -> requests.Response:
+        """
+        Click 'Accept all' to cookie-consent form and return response object.
+
+        Args:
+            consent_resp (requests.Response) : Response instance of cookie-consent page
+            timeout (int) : Raise TimeoutError if post doesn't respond
+    
+        Returns:
+            response (requests.Response) : Reponse instance received from the server after accepting cookie-consent post.
+        """
+        soup = BeautifulSoup(consent_resp.text, "html.parser")
+    
+        # Heuristic: pick the first form; Yahoo's CMP tends to have a single form for consent
+        form = soup.find("form")
+        if not form:
+            return consent_resp
+    
+        # action : URL to send "Accept Cookies"
+        action = form.get("action") or consent_resp.url
+        action = urljoin(consent_resp.url, action)
+    
+        # Collect inputs (hidden tokens, etc.)
+        """
+        <input name="csrfToken" type="hidden" value="..."/>
+        <input name="sessionId" type="hidden" value="..."/>
+        <input name="originalDoneUrl" type="hidden" value="..."/>
+        <input name="namespace" type="hidden" value="yahoo"/>
+        """
+        data = {}
+        for inp in form.find_all("input"):
+            name = inp.get("name")
+            if not name:
+                continue
+            typ = (inp.get("type") or "text").lower()
+            val = inp.get("value") or ""
+    
+            if typ in ("checkbox", "radio"):
+                # If it's clearly an "agree"/"accept" field or already checked, include it
+                if (
+                    "agree" in name.lower()
+                    or "accept" in name.lower()
+                    or inp.has_attr("checked")
+                ):
+                    data[name] = val if val != "" else "1"
+            else:
+                data[name] = val
+    
+        # If no explicit agree/accept in inputs, add a best-effort flag
+        lowered = {k.lower() for k in data.keys()}
+        if not any(("agree" in k or "accept" in k) for k in lowered):
+            data["agree"] = "1"
+    
+        # Submit the form with "Referer". Some servers check this header as a simple CSRF protection measure.
+        headers = {"Referer": consent_resp.url}
+        response = self._session.post(
+            action, data=data, headers=headers, timeout=timeout, allow_redirects=True
+        )
+        return response

yfinance/domain/domain.py

@@ -1,10 +1,11 @@
 from abc import ABC, abstractmethod
-from ..ticker import Ticker
+import pandas as _pd
+from typing import Dict, List, Optional
+import warnings
+
 from ..const import _QUERY1_URL_, _SENTINEL_
 from ..data import YfData
-from ..utils import print_once
-from typing import Dict, List, Optional
-import pandas as _pd
+from ..ticker import Ticker
 
 _QUERY_URL_ = f'{_QUERY1_URL_}/v1/finance'
 
@@ -26,7 +27,7 @@ class Domain(ABC):
         self.session = session
         self._data: YfData = YfData(session=session)
         if proxy is not _SENTINEL_:
-            print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         self._name: Optional[str] = None
@@ -122,7 +123,7 @@ class Domain(ABC):
             Dict: The JSON response data from the request.
         """
         params_dict = {"formatted": "true", "withReturns": "true", "lang": "en-US", "region": "US"}
-        result = self._data.get_raw_json(query_url, user_agent_headers=self._data.user_agent_headers, params=params_dict)
+        result = self._data.get_raw_json(query_url, params=params_dict)
         return result
 
     def _parse_and_assign_common(self, data) -> None:

yfinance/domain/industry.py

@@ -1,12 +1,15 @@
 from __future__ import print_function
-from typing import Dict, Optional
 
 import pandas as _pd
+from typing import Dict, Optional
+import warnings
+
+from .. import utils
+from ..config import YfConfig
+from ..const import _SENTINEL_
+from ..data import YfData
 
 from .domain import Domain, _QUERY_URL_
-from .. import utils
-from ..data import YfData
-from ..const import _SENTINEL_
 
 class Industry(Domain):
     """
@@ -20,8 +23,9 @@ class Industry(Domain):
             session (optional): The session to use for requests.
         """
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
-            YfData(session=session, proxy=proxy)
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
+            YfData(proxy=proxy)
+        YfData(session=session)
         super(Industry, self).__init__(key, session)
         self._query_url = f'{_QUERY_URL_}/industries/{self._key}'
 
@@ -93,7 +97,7 @@ class Industry(Domain):
         Returns:
             Optional[pd.DataFrame]: DataFrame containing parsed top performing companies data.
         """
-        compnaies_column = ['symbol','name','ytd return',' last price','target price']
+        compnaies_column = ['symbol','name','ytd return','last price','target price']
         compnaies_values = [(c.get('symbol', None),
                              c.get('name', None),
                              c.get('ytdReturn',{}).get('raw', None),
@@ -115,7 +119,7 @@ class Industry(Domain):
         Returns:
             Optional[pd.DataFrame]: DataFrame containing parsed top growth companies data.
         """
-        compnaies_column = ['symbol','name','ytd return',' growth estimate']
+        compnaies_column = ['symbol','name','ytd return','growth estimate']
         compnaies_values = [(c.get('symbol', None),
                              c.get('name', None),
                              c.get('ytdReturn',{}).get('raw', None),
@@ -144,6 +148,8 @@ class Industry(Domain):
 
             return result
         except Exception as e:
+            if not YfConfig().hide_exceptions:
+                raise
             logger = utils.get_yf_logger()
             logger.error(f"Failed to get industry data for '{self._key}' reason: {e}")
             logger.debug("Got response: ")

yfinance/domain/market.py

@@ -1,9 +1,11 @@
 import datetime as dt
-
-from ..data import YfData
-from ..data import utils
-from ..const import _QUERY1_URL_, _SENTINEL_
 import json as _json
+import warnings
+
+from ..config import YfConfig
+from ..const import _QUERY1_URL_, _SENTINEL_
+from ..data import utils, YfData
+from ..exceptions import YFDataException
 
 class Market:
     def __init__(self, market:'str', session=None, proxy=_SENTINEL_, timeout=30):
@@ -13,7 +15,7 @@ class Market:
 
         self._data = YfData(session=self.session)
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         self._logger = utils.get_yf_logger()
@@ -24,12 +26,12 @@ class Market:
     def _fetch_json(self, url, params):
         data = self._data.cache_get(url=url, params=params, timeout=self.timeout)
         if data is None or "Will be right back" in data.text:
-            raise RuntimeError("*** YAHOO! FINANCE IS CURRENTLY DOWN! ***\n"
-                               "Our engineers are working quickly to resolve "
-                               "the issue. Thank you for your patience.")
+            raise YFDataException("*** YAHOO! FINANCE IS CURRENTLY DOWN! ***")
         try:
             return data.json()
         except _json.JSONDecodeError:
+            if not YfConfig().hide_exceptions:
+                raise
             self._logger.error(f"{self.market}: Failed to retrieve market data and recieved faulty data.")
             return {}
         
@@ -66,6 +68,8 @@ class Market:
             self._summary = self._summary['marketSummaryResponse']['result']
             self._summary = {x['exchange']:x for x in self._summary}
         except Exception as e:
+            if not YfConfig().hide_exceptions:
+                raise
             self._logger.error(f"{self.market}: Failed to parse market summary")
             self._logger.debug(f"{type(e)}: {e}")
 
@@ -75,18 +79,22 @@ class Market:
             self._status = self._status['finance']['marketTimes'][0]['marketTime'][0]
             self._status['timezone'] = self._status['timezone'][0]
             del self._status['time']  # redundant
-            try:
-                self._status.update({
-                    "open": dt.datetime.fromisoformat(self._status["open"]),
-                    "close": dt.datetime.fromisoformat(self._status["close"]),
-                    "tz": dt.timezone(dt.timedelta(hours=int(self._status["timezone"]["gmtoffset"]))/1000, self._status["timezone"]["short"])
-                })
-            except Exception as e:
-                self._logger.error(f"{self.market}: Failed to update market status")
-                self._logger.debug(f"{type(e)}: {e}")
         except Exception as e:
+            if not YfConfig().hide_exceptions:
+                raise
             self._logger.error(f"{self.market}: Failed to parse market status")
             self._logger.debug(f"{type(e)}: {e}")
+        try:
+            self._status.update({
+                "open": dt.datetime.fromisoformat(self._status["open"]),
+                "close": dt.datetime.fromisoformat(self._status["close"]),
+                "tz": dt.timezone(dt.timedelta(hours=int(self._status["timezone"]["gmtoffset"]))/1000, self._status["timezone"]["short"])
+            })
+        except Exception as e:
+            if not YfConfig().hide_exceptions:
+                raise
+            self._logger.error(f"{self.market}: Failed to update market status")
+            self._logger.debug(f"{type(e)}: {e}")
 
 
 

yfinance/domain/sector.py

@@ -1,13 +1,15 @@
 from __future__ import print_function
-from typing import Dict, Optional
-from ..utils import dynamic_docstring, generate_list_table_from_dict
-from ..const import SECTOR_INDUSTY_MAPPING, _SENTINEL_
 
 import pandas as _pd
+from typing import Dict, Optional
+import warnings
+
+from ..config import YfConfig
+from ..const import SECTOR_INDUSTY_MAPPING_LC, _SENTINEL_
+from ..data import YfData
+from ..utils import dynamic_docstring, generate_list_table_from_dict, get_yf_logger
 
 from .domain import Domain, _QUERY_URL_
-from .. import utils
-from ..data import YfData
 
 class Sector(Domain):
     """
@@ -28,7 +30,7 @@ class Sector(Domain):
                 Map of sector and industry
         """
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             YfData(session=session, proxy=proxy)
 
         super(Sector, self).__init__(key, session)
@@ -68,7 +70,7 @@ class Sector(Domain):
         self._ensure_fetched(self._top_mutual_funds)
         return self._top_mutual_funds
 
-    @dynamic_docstring({"sector_industry": generate_list_table_from_dict(SECTOR_INDUSTY_MAPPING,bullets=True)})
+    @dynamic_docstring({"sector_industry": generate_list_table_from_dict(SECTOR_INDUSTY_MAPPING_LC,bullets=True)})
     @property
     def industries(self) -> _pd.DataFrame:
         """
@@ -147,7 +149,9 @@ class Sector(Domain):
             self._industries = self._parse_industries(data.get('industries', {}))
 
         except Exception as e:
-            logger = utils.get_yf_logger()
+            if not YfConfig().hide_exceptions:
+                raise
+            logger = get_yf_logger()
             logger.error(f"Failed to get sector data for '{self._key}' reason: {e}")
             logger.debug("Got response: ")
             logger.debug("-------------")

yfinance/exceptions.py

@@ -45,7 +45,7 @@ class YFInvalidPeriodError(YFException):
         self.invalid_period = invalid_period
         self.valid_ranges = valid_ranges
         super().__init__(f"{self.ticker}: Period '{invalid_period}' is invalid, "
-                         f"must be of the format {valid_ranges}, etc.")
+                         f"must be one of: {valid_ranges}")
 
 
 class YFRateLimitError(YFException):

yfinance/live.py

@@ -0,0 +1,350 @@
+import asyncio
+import base64
+import json
+from typing import List, Optional, Callable, Union
+
+from websockets.sync.client import connect as sync_connect
+from websockets.asyncio.client import connect as async_connect
+
+from yfinance import utils
+from yfinance.config import YfConfig
+from yfinance.pricing_pb2 import PricingData
+from google.protobuf.json_format import MessageToDict
+
+
+class BaseWebSocket:
+    def __init__(self, url: str = "wss://streamer.finance.yahoo.com/?version=2", verbose=True):
+        self.url = url
+        self.verbose = verbose
+        self.logger = utils.get_yf_logger()
+        self._ws = None
+        self._subscriptions = set()
+        self._subscription_interval = 15  # seconds
+
+    def _decode_message(self, base64_message: str) -> dict:
+        try:
+            decoded_bytes = base64.b64decode(base64_message)
+            pricing_data = PricingData()
+            pricing_data.ParseFromString(decoded_bytes)
+            return MessageToDict(pricing_data, preserving_proto_field_name=True)
+        except Exception as e:
+            if not YfConfig().hide_exceptions:
+                raise
+            self.logger.error("Failed to decode message: %s", e, exc_info=True)
+            if self.verbose:
+                print("Failed to decode message: %s", e)
+            return {
+                'error': str(e),
+                'raw_base64': base64_message
+            }
+
+
+class AsyncWebSocket(BaseWebSocket):
+    """
+    Asynchronous WebSocket client for streaming real time pricing data.
+    """
+
+    def __init__(self, url: str = "wss://streamer.finance.yahoo.com/?version=2", verbose=True):
+        """
+        Initialize the AsyncWebSocket client.
+
+        Args:
+            url (str): The WebSocket server URL. Defaults to Yahoo Finance's WebSocket URL.
+            verbose (bool): Flag to enable or disable print statements. Defaults to True.
+        """
+        super().__init__(url, verbose)
+        self._message_handler = None  # Callable to handle messages
+        self._heartbeat_task = None  # Task to send heartbeat subscribe
+
+    async def _connect(self):
+        try:
+            if self._ws is None:
+                self._ws = await async_connect(self.url)
+                self.logger.info("Connected to WebSocket.")
+                if self.verbose:
+                    print("Connected to WebSocket.")
+        except Exception as e:
+            if not YfConfig().hide_exceptions:
+                raise
+            self.logger.error("Failed to connect to WebSocket: %s", e, exc_info=True)
+            if self.verbose:
+                print(f"Failed to connect to WebSocket: {e}")
+            self._ws = None
+            raise
+
+    async def _periodic_subscribe(self):
+        while True:
+            try:
+                await asyncio.sleep(self._subscription_interval)
+
+                if self._subscriptions:
+                    message = {"subscribe": list(self._subscriptions)}
+                    await self._ws.send(json.dumps(message))
+
+                    if self.verbose:
+                        print(f"Heartbeat subscription sent for symbols: {self._subscriptions}")
+            except Exception as e:
+                if not YfConfig().hide_exceptions:
+                    raise
+                self.logger.error("Error in heartbeat subscription: %s", e, exc_info=True)
+                if self.verbose:
+                    print(f"Error in heartbeat subscription: {e}")
+                break
+
+    async def subscribe(self, symbols: Union[str, List[str]]):
+        """
+        Subscribe to a stock symbol or a list of stock symbols.
+
+        Args:
+            symbols (Union[str, List[str]]): Stock symbol(s) to subscribe to.
+        """
+        await self._connect()
+
+        if isinstance(symbols, str):
+            symbols = [symbols]
+
+        self._subscriptions.update(symbols)
+
+        message = {"subscribe": list(self._subscriptions)}
+        await self._ws.send(json.dumps(message))
+
+        # Start heartbeat subscription task
+        if self._heartbeat_task is None:
+            self._heartbeat_task = asyncio.create_task(self._periodic_subscribe())
+
+        self.logger.info(f"Subscribed to symbols: {symbols}")
+        if self.verbose:
+            print(f"Subscribed to symbols: {symbols}")
+
+    async def unsubscribe(self, symbols: Union[str, List[str]]):
+        """
+        Unsubscribe from a stock symbol or a list of stock symbols.
+
+        Args:
+            symbols (Union[str, List[str]]): Stock symbol(s) to unsubscribe from.
+        """
+        await self._connect()
+
+        if isinstance(symbols, str):
+            symbols = [symbols]
+
+        self._subscriptions.difference_update(symbols)
+
+        message = {"unsubscribe": symbols}
+        await self._ws.send(json.dumps(message))
+
+        self.logger.info(f"Unsubscribed from symbols: {symbols}")
+        if self.verbose:
+            print(f"Unsubscribed from symbols: {symbols}")
+
+    async def listen(self, message_handler=None):
+        """
+        Start listening to messages from the WebSocket server.
+
+        Args:
+            message_handler (Optional[Callable[[dict], None]]): Optional function to handle received messages.
+        """
+        await self._connect()
+        self._message_handler = message_handler
+
+        self.logger.info("Listening for messages...")
+        if self.verbose:
+            print("Listening for messages...")
+
+        # Start heartbeat subscription task
+        if self._heartbeat_task is None:
+            self._heartbeat_task = asyncio.create_task(self._periodic_subscribe())
+
+        while True:
+            try:
+                async for message in self._ws:
+                    message_json = json.loads(message)
+                    encoded_data = message_json.get("message", "")
+                    decoded_message = self._decode_message(encoded_data)
+
+                    if self._message_handler:
+                        try:
+                            if asyncio.iscoroutinefunction(self._message_handler):
+                                await self._message_handler(decoded_message)
+                            else:
+                                self._message_handler(decoded_message)
+                        except Exception as handler_exception:
+                            if not YfConfig().hide_exceptions:
+                                raise
+                            self.logger.error("Error in message handler: %s", handler_exception, exc_info=True)
+                            if self.verbose:
+                                print("Error in message handler:", handler_exception)
+                    else:
+                        print(decoded_message)
+
+            except (KeyboardInterrupt, asyncio.CancelledError):
+                self.logger.info("WebSocket listening interrupted. Closing connection...")
+                if self.verbose:
+                    print("WebSocket listening interrupted. Closing connection...")
+                await self.close()
+                break
+
+            except Exception as e:
+                if not YfConfig().hide_exceptions:
+                    raise
+                self.logger.error("Error while listening to messages: %s", e, exc_info=True)
+                if self.verbose:
+                    print("Error while listening to messages: %s", e)
+
+                # Attempt to reconnect if connection drops
+                self.logger.info("Attempting to reconnect...")
+                if self.verbose:
+                    print("Attempting to reconnect...")
+                await asyncio.sleep(3)  # backoff
+                await self._connect()
+
+    async def close(self):
+        """Close the WebSocket connection."""
+        if self._heartbeat_task:
+            self._heartbeat_task.cancel()
+
+        if self._ws is not None:  # and not self._ws.closed:
+            await self._ws.close()
+            self.logger.info("WebSocket connection closed.")
+            if self.verbose:
+                print("WebSocket connection closed.")
+
+    async def __aenter__(self):
+        await self._connect()
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        await self.close()
+
+
+class WebSocket(BaseWebSocket):
+    """
+    Synchronous WebSocket client for streaming real time pricing data.
+    """
+
+    def __init__(self, url: str = "wss://streamer.finance.yahoo.com/?version=2", verbose=True):
+        """
+        Initialize the WebSocket client.
+
+        Args:
+            url (str): The WebSocket server URL. Defaults to Yahoo Finance's WebSocket URL.
+            verbose (bool): Flag to enable or disable print statements. Defaults to True.
+        """
+        super().__init__(url, verbose)
+
+    def _connect(self):
+        try:
+            if self._ws is None:
+                self._ws = sync_connect(self.url)
+                self.logger.info("Connected to WebSocket.")
+                if self.verbose:
+                    print("Connected to WebSocket.")
+        except Exception as e:
+            self.logger.error("Failed to connect to WebSocket: %s", e, exc_info=True)
+            if self.verbose:
+                print(f"Failed to connect to WebSocket: {e}")
+            self._ws = None
+            raise
+
+    def subscribe(self, symbols: Union[str, List[str]]):
+        """
+        Subscribe to a stock symbol or a list of stock symbols.
+
+        Args:
+            symbols (Union[str, List[str]]): Stock symbol(s) to subscribe to.
+        """
+        self._connect()
+
+        if isinstance(symbols, str):
+            symbols = [symbols]
+
+        self._subscriptions.update(symbols)
+
+        message = {"subscribe": list(self._subscriptions)}
+        self._ws.send(json.dumps(message))
+
+        self.logger.info(f"Subscribed to symbols: {symbols}")
+        if self.verbose:
+            print(f"Subscribed to symbols: {symbols}")
+
+    def unsubscribe(self, symbols: Union[str, List[str]]):
+        """
+        Unsubscribe from a stock symbol or a list of stock symbols.
+
+        Args:
+            symbols (Union[str, List[str]]): Stock symbol(s) to unsubscribe from.
+        """
+        self._connect()
+
+        if isinstance(symbols, str):
+            symbols = [symbols]
+
+        self._subscriptions.difference_update(symbols)
+
+        message = {"unsubscribe": symbols}
+        self._ws.send(json.dumps(message))
+
+        self.logger.info(f"Unsubscribed from symbols: {symbols}")
+        if self.verbose:
+            print(f"Unsubscribed from symbols: {symbols}")
+
+    def listen(self, message_handler: Optional[Callable[[dict], None]] = None):
+        """
+        Start listening to messages from the WebSocket server.
+
+        Args:
+            message_handler (Optional[Callable[[dict], None]]): Optional function to handle received messages.
+        """
+        self._connect()
+
+        self.logger.info("Listening for messages...")
+        if self.verbose:
+            print("Listening for messages...")
+
+        while True:
+            try:
+                message = self._ws.recv()
+                message_json = json.loads(message)
+                encoded_data = message_json.get("message", "")
+                decoded_message = self._decode_message(encoded_data)
+
+                if message_handler:
+                    try:
+                        message_handler(decoded_message)
+                    except Exception as handler_exception:
+                        if not YfConfig().hide_exceptions:
+                            raise
+                        self.logger.error("Error in message handler: %s", handler_exception, exc_info=True)
+                        if self.verbose:
+                            print("Error in message handler:", handler_exception)
+                else:
+                    print(decoded_message)
+
+            except KeyboardInterrupt:
+                if self.verbose:
+                    print("Received keyboard interrupt.")
+                self.close()
+                break
+
+            except Exception as e:
+                if not YfConfig().hide_exceptions:
+                    raise
+                self.logger.error("Error while listening to messages: %s", e, exc_info=True)
+                if self.verbose:
+                    print("Error while listening to messages: %s", e)
+                break
+
+    def close(self):
+        """Close the WebSocket connection."""
+        if self._ws is not None:
+            self._ws.close()
+            self.logger.info("WebSocket connection closed.")
+            if self.verbose:
+                print("WebSocket connection closed.")
+
+    def __enter__(self):
+        self._connect()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()

yfinance/lookup.py

@@ -20,13 +20,14 @@
 #
 
 import json as _json
-
 import pandas as pd
+import warnings
 
 from . import utils
+from .config import YfConfig
 from .const import _QUERY1_URL_, _SENTINEL_
 from .data import YfData
-from .exceptions import YFException
+from .exceptions import YFDataException
 
 LOOKUP_TYPES = ["all", "equity", "mutualfund", "etf", "index", "future", "currency", "cryptocurrency"]
 
@@ -43,12 +44,12 @@ class Lookup:
     :param raise_errors: Raise exceptions on error (default True).
     """
 
-    def __init__(self, query: str, session=None, proxy=None, timeout=30, raise_errors=True):
+    def __init__(self, query: str, session=None, proxy=_SENTINEL_, timeout=30, raise_errors=True):
         self.session = session
         self._data = YfData(session=self.session)
 
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         self.query = query
@@ -81,18 +82,19 @@ class Lookup:
 
         data = self._data.get(url=url, params=params, timeout=self.timeout)
         if data is None or "Will be right back" in data.text:
-            raise RuntimeError("*** YAHOO! FINANCE IS CURRENTLY DOWN! ***\n"
-                               "Our engineers are working quickly to resolve "
-                               "the issue. Thank you for your patience.")
+            raise YFDataException("*** YAHOO! FINANCE IS CURRENTLY DOWN! ***")
         try:
             data = data.json()
         except _json.JSONDecodeError:
-            self._logger.error(f"{self.query}: Failed to retrieve lookup results and received faulty response instead.")
+            if not YfConfig().hide_exceptions:
+                raise
+            self._logger.error(f"{self.ticker}: 'lookup' fetch received faulty data")
             data = {}
 
         # Error returned
         if data.get("finance", {}).get("error", {}):
-            raise YFException(data.get("finance", {}).get("error", {}))
+            error = data.get("finance", {}).get("error", {})
+            raise YFDataException(f"{self.ticker}: 'lookup' fetch returned error: {error}")
 
         self._cache[cache_key] = data
         return data

yfinance/multi.py

@@ -25,6 +25,7 @@ import logging
 import time as _time
 import traceback
 from typing import Union
+import warnings
 
 import multitasking as _multitasking
 import pandas as _pd
@@ -38,7 +39,7 @@ from .const import _SENTINEL_
 @utils.log_indent_decorator
 def download(tickers, start=None, end=None, actions=False, threads=True,
              ignore_tz=None, group_by='column', auto_adjust=None, back_adjust=False,
-             repair=False, keepna=False, progress=True, period="max", interval="1d",
+             repair=False, keepna=False, progress=True, period=None, interval="1d",
              prepost=False, proxy=_SENTINEL_, rounding=False, timeout=10, session=None,
              multi_level_index=True) -> Union[_pd.DataFrame, None]:
     """
@@ -48,6 +49,7 @@ def download(tickers, start=None, end=None, actions=False, threads=True,
             List of tickers to download
         period : str
             Valid periods: 1d,5d,1mo,3mo,6mo,1y,2y,5y,10y,ytd,max
+            Default: 1mo
             Either Use period parameter or use start and end
         interval : str
             Valid intervals: 1m,2m,5m,15m,30m,60m,90m,1h,1d,5d,1wk,1mo,3mo
@@ -93,9 +95,15 @@ def download(tickers, start=None, end=None, actions=False, threads=True,
     logger = utils.get_yf_logger()
     session = session or requests.Session(impersonate="chrome")
 
+    # Ensure data initialised with session.
+    if proxy is not _SENTINEL_:
+        warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=3)
+        YfData(proxy=proxy)
+    YfData(session=session)
+
     if auto_adjust is None:
         # Warn users that default has changed to True
-        utils.print_once("YF.download() has changed argument auto_adjust default to True")
+        warnings.warn("YF.download() has changed argument auto_adjust default to True", FutureWarning, stacklevel=3)
         auto_adjust = True
 
     if logger.isEnabledFor(logging.DEBUG):
@@ -127,7 +135,7 @@ def download(tickers, start=None, end=None, actions=False, threads=True,
     for ticker in tickers:
         if utils.is_isin(ticker):
             isin = ticker
-            ticker = utils.get_ticker_by_isin(ticker, session=session)
+            ticker = utils.get_ticker_by_isin(ticker)
             shared._ISINS[ticker] = isin
         _tickers_.append(ticker)
 
@@ -143,13 +151,6 @@ def download(tickers, start=None, end=None, actions=False, threads=True,
     shared._ERRORS = {}
     shared._TRACEBACKS = {}
 
-    # Ensure data initialised with session.
-    if proxy is not _SENTINEL_:
-        utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
-        YfData(session=session, proxy=proxy)
-    else:
-        YfData(session=session)
-
     # download using threads
     if threads:
         if threads is True:

yfinance/pricing.proto

@@ -0,0 +1,37 @@
+syntax = "proto3";
+
+message PricingData {
+    string id = 1;
+    float price = 2;
+    sint64 time = 3;
+    string currency = 4;
+    string exchange = 5;
+    int32 quote_type = 6;
+    int32 market_hours = 7;
+    float change_percent = 8;
+    sint64 day_volume = 9;
+    float day_high = 10;
+    float day_low = 11;
+    float change = 12;
+    string short_name = 13;
+    sint64 expire_date = 14;
+    float open_price = 15;
+    float previous_close = 16;
+    float strike_price = 17;
+    string underlying_symbol = 18;
+    sint64 open_interest = 19;
+    sint64 options_type = 20;
+    sint64 mini_option = 21;
+    sint64 last_size = 22;
+    float bid = 23;
+    sint64 bid_size = 24;
+    float ask = 25;
+    sint64 ask_size = 26;
+    sint64 price_hint = 27;
+    sint64 vol_24hr = 28;
+    sint64 vol_all_currencies = 29;
+    string from_currency = 30;
+    string last_market = 31;
+    double circulating_supply = 32;
+    double market_cap = 33;
+}

yfinance/pricing_pb2.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: pricing.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rpricing.proto\"\x9a\x05\n\x0bPricingData\x12\n\n\x02id\x18\x01 \x01(\t\x12\r\n\x05price\x18\x02 \x01(\x02\x12\x0c\n\x04time\x18\x03 \x01(\x12\x12\x10\n\x08\x63urrency\x18\x04 \x01(\t\x12\x10\n\x08\x65xchange\x18\x05 \x01(\t\x12\x12\n\nquote_type\x18\x06 \x01(\x05\x12\x14\n\x0cmarket_hours\x18\x07 \x01(\x05\x12\x16\n\x0e\x63hange_percent\x18\x08 \x01(\x02\x12\x12\n\nday_volume\x18\t \x01(\x12\x12\x10\n\x08\x64\x61y_high\x18\n \x01(\x02\x12\x0f\n\x07\x64\x61y_low\x18\x0b \x01(\x02\x12\x0e\n\x06\x63hange\x18\x0c \x01(\x02\x12\x12\n\nshort_name\x18\r \x01(\t\x12\x13\n\x0b\x65xpire_date\x18\x0e \x01(\x12\x12\x12\n\nopen_price\x18\x0f \x01(\x02\x12\x16\n\x0eprevious_close\x18\x10 \x01(\x02\x12\x14\n\x0cstrike_price\x18\x11 \x01(\x02\x12\x19\n\x11underlying_symbol\x18\x12 \x01(\t\x12\x15\n\ropen_interest\x18\x13 \x01(\x12\x12\x14\n\x0coptions_type\x18\x14 \x01(\x12\x12\x13\n\x0bmini_option\x18\x15 \x01(\x12\x12\x11\n\tlast_size\x18\x16 \x01(\x12\x12\x0b\n\x03\x62id\x18\x17 \x01(\x02\x12\x10\n\x08\x62id_size\x18\x18 \x01(\x12\x12\x0b\n\x03\x61sk\x18\x19 \x01(\x02\x12\x10\n\x08\x61sk_size\x18\x1a \x01(\x12\x12\x12\n\nprice_hint\x18\x1b \x01(\x12\x12\x10\n\x08vol_24hr\x18\x1c \x01(\x12\x12\x1a\n\x12vol_all_currencies\x18\x1d \x01(\x12\x12\x15\n\rfrom_currency\x18\x1e \x01(\t\x12\x13\n\x0blast_market\x18\x1f \x01(\t\x12\x1a\n\x12\x63irculating_supply\x18  \x01(\x01\x12\x12\n\nmarket_cap\x18! \x01(\x01\x62\x06proto3')
+
+
+
+_PRICINGDATA = DESCRIPTOR.message_types_by_name['PricingData']
+PricingData = _reflection.GeneratedProtocolMessageType('PricingData', (_message.Message,), {
+  'DESCRIPTOR' : _PRICINGDATA,
+  '__module__' : 'pricing_pb2'
+  # @@protoc_insertion_point(class_scope:PricingData)
+  })
+_sym_db.RegisterMessage(PricingData)
+
+if not _descriptor._USE_C_DESCRIPTORS:
+  DESCRIPTOR._options = None
+  _PRICINGDATA._serialized_start=18
+  _PRICINGDATA._serialized_end=684
+# @@protoc_insertion_point(module_scope)

yfinance/scrapers/analysis.py

@@ -1,17 +1,19 @@
+import curl_cffi
 import pandas as pd
-import requests
+import warnings
 
 from yfinance import utils
-from yfinance.data import YfData
+from yfinance.config import YfConfig
 from yfinance.const import quote_summary_valid_modules, _SENTINEL_
-from yfinance.scrapers.quote import _QUOTE_SUMMARY_URL_
+from yfinance.data import YfData
 from yfinance.exceptions import YFException
+from yfinance.scrapers.quote import _QUOTE_SUMMARY_URL_
 
 class Analysis:
 
     def __init__(self, data: YfData, symbol: str, proxy=_SENTINEL_):
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             data._set_proxy(proxy)
 
         self._data = data
@@ -83,6 +85,8 @@ class Analysis:
             data = self._fetch(['financialData'])
             data = data['quoteSummary']['result'][0]['financialData']
         except (TypeError, KeyError):
+            if not YfConfig().hide_exceptions:
+                raise
             self._analyst_price_targets = {}
             return self._analyst_price_targets
 
@@ -106,6 +110,8 @@ class Analysis:
             data = self._fetch(['earningsHistory'])
             data = data['quoteSummary']['result'][0]['earningsHistory']['history']
         except (TypeError, KeyError):
+            if not YfConfig().hide_exceptions:
+                raise
             self._earnings_history = pd.DataFrame()
             return self._earnings_history
 
@@ -142,6 +148,8 @@ class Analysis:
             trends = self._fetch(['industryTrend', 'sectorTrend', 'indexTrend'])
             trends = trends['quoteSummary']['result'][0]
         except (TypeError, KeyError):
+            if not YfConfig().hide_exceptions:
+                raise
             self._growth_estimates = pd.DataFrame()
             return self._growth_estimates
 
@@ -177,9 +185,11 @@ class Analysis:
             raise YFException("No valid modules provided, see available modules using `valid_modules`")
         params_dict = {"modules": modules, "corsDomain": "finance.yahoo.com", "formatted": "false", "symbol": self._symbol}
         try:
-            result = self._data.get_raw_json(_QUOTE_SUMMARY_URL_ + f"/{self._symbol}", user_agent_headers=self._data.user_agent_headers, params=params_dict)
-        except requests.exceptions.HTTPError as e:
-            utils.get_yf_logger().error(str(e))
+            result = self._data.get_raw_json(_QUOTE_SUMMARY_URL_ + f"/{self._symbol}", params=params_dict)
+        except curl_cffi.requests.exceptions.HTTPError as e:
+            if not YfConfig().hide_exceptions:
+                raise
+            utils.get_yf_logger().error(str(e) + e.response.text)
             return None
         return result
 
@@ -188,4 +198,6 @@ class Analysis:
             data = self._fetch(['earningsTrend'])
             self._earnings_trend = data['quoteSummary']['result'][0]['earningsTrend']['trend']
         except (TypeError, KeyError):
+            if not YfConfig().hide_exceptions:
+                raise
             self._earnings_trend = []

yfinance/scrapers/fundamentals.py

@@ -5,6 +5,7 @@ import warnings
 import pandas as pd
 
 from yfinance import utils, const
+from yfinance.config import YfConfig
 from yfinance.data import YfData
 from yfinance.exceptions import YFException, YFNotImplementedError
 
@@ -12,7 +13,7 @@ class Fundamentals:
 
     def __init__(self, data: YfData, symbol: str, proxy=const._SENTINEL_):
         if proxy is not const._SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             data._set_proxy(proxy)
 
         self._data = data
@@ -53,7 +54,7 @@ class Financials:
 
     def get_income_time_series(self, freq="yearly", proxy=const._SENTINEL_) -> pd.DataFrame:
         if proxy is not const._SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         res = self._income_time_series
@@ -63,7 +64,7 @@ class Financials:
 
     def get_balance_sheet_time_series(self, freq="yearly", proxy=const._SENTINEL_) -> pd.DataFrame:
         if proxy is not const._SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         res = self._balance_sheet_time_series
@@ -73,7 +74,7 @@ class Financials:
 
     def get_cash_flow_time_series(self, freq="yearly", proxy=const._SENTINEL_) -> pd.DataFrame:
         if proxy is not const._SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         res = self._cash_flow_time_series
@@ -104,6 +105,8 @@ class Financials:
             if statement is not None:
                 return statement
         except YFException as e:
+            if not YfConfig().hide_exceptions:
+                raise
             utils.get_yf_logger().error(f"{self._symbol}: Failed to create {name} financials table for reason: {e}")
         return pd.DataFrame()
 
@@ -117,6 +120,8 @@ class Financials:
         try:
             return self._get_financials_time_series(timescale, keys)
         except Exception:
+            if not YfConfig().hide_exceptions:
+                raise
             pass
 
     def _get_financials_time_series(self, timescale, keys: list) -> pd.DataFrame:
@@ -159,6 +164,10 @@ class Financials:
 
         df.index = df.index.str.replace("^" + timescale, "", regex=True)
 
+        # Ensure float type, not object
+        for d in df.columns:
+            df[d] = df[d].astype('float')
+
         # Reorder table to match order on Yahoo website
         df = df.reindex([k for k in keys if k in df.index])
         df = df[sorted(df.columns, reverse=True)]

yfinance/scrapers/funds.py

@@ -1,11 +1,12 @@
 import pandas as pd
-
-from yfinance.data import YfData
-from yfinance.const import _BASE_URL_, _SENTINEL_
-from yfinance.exceptions import YFDataException
-from yfinance import utils
-
 from typing import Dict, Optional
+import warnings
+
+from yfinance import utils
+from yfinance.config import YfConfig
+from yfinance.const import _BASE_URL_, _SENTINEL_
+from yfinance.data import YfData
+from yfinance.exceptions import YFDataException
 
 _QUOTE_SUMMARY_URL_ = f"{_BASE_URL_}/v10/finance/quoteSummary/"
 
@@ -26,7 +27,7 @@ class FundsData:
         self._data = data
         self._symbol = symbol
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
         
         # quoteType
@@ -175,7 +176,7 @@ class FundsData:
         """
         modules = ','.join(["quoteType", "summaryProfile", "topHoldings", "fundProfile"])
         params_dict = {"modules": modules, "corsDomain": "finance.yahoo.com", "symbol": self._symbol, "formatted": "false"}
-        result = self._data.get_raw_json(_QUOTE_SUMMARY_URL_+self._symbol, user_agent_headers=self._data.user_agent_headers, params=params_dict)
+        result = self._data.get_raw_json(_QUOTE_SUMMARY_URL_+self._symbol, params=params_dict)
         return result
 
     def _fetch_and_parse(self) -> None:
@@ -193,8 +194,12 @@ class FundsData:
             self._parse_top_holdings(data["topHoldings"])
             self._parse_fund_profile(data["fundProfile"])
         except KeyError:
-            raise YFDataException("No Fund data found.")
+            if not YfConfig().hide_exceptions:
+                raise
+            raise YFDataException(f"{self._symbol}: No Fund data found.")
         except Exception as e:
+            if not YfConfig().hide_exceptions:
+                raise
             logger = utils.get_yf_logger()
             logger.error(f"Failed to get fund data for '{self._symbol}' reason: {e}")
             logger.debug("Got response: ")

yfinance/scrapers/history.py

@@ -1,16 +1,18 @@
+from curl_cffi import requests
+from math import isclose
+import bisect
 import datetime as _datetime
 import dateutil as _dateutil
 import logging
 import numpy as np
 import pandas as pd
-from math import isclose
 import time as _time
-import bisect
-from curl_cffi import requests
+import warnings
 
 from yfinance import shared, utils
+from yfinance.config import YfConfig
 from yfinance.const import _BASE_URL_, _PRICE_COLNAMES_, _SENTINEL_
-from yfinance.exceptions import YFInvalidPeriodError, YFPricesMissingError, YFTzMissingError, YFRateLimitError
+from yfinance.exceptions import YFDataException, YFInvalidPeriodError, YFPricesMissingError, YFRateLimitError, YFTzMissingError
 
 class PriceHistory:
     def __init__(self, data, ticker, tz, session=None, proxy=_SENTINEL_):
@@ -18,7 +20,7 @@ class PriceHistory:
         self.ticker = ticker.upper()
         self.tz = tz
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=5)
             self._data._set_proxy(proxy)
         self.session = session or requests.Session(impersonate="chrome")
 
@@ -30,7 +32,7 @@ class PriceHistory:
         self._reconstruct_start_interval = None
 
     @utils.log_indent_decorator
-    def history(self, period="1mo", interval="1d",
+    def history(self, period=None, interval="1d",
                 start=None, end=None, prepost=False, actions=True,
                 auto_adjust=True, back_adjust=False, repair=False, keepna=False,
                 proxy=_SENTINEL_, rounding=False, timeout=10,
@@ -38,55 +40,60 @@ class PriceHistory:
         """
         :Parameters:
             period : str
-                Valid periods: 1d,5d,1mo,3mo,6mo,1y,2y,5y,10y,ytd,max
-                Either Use period parameter or use start and end
+              | Valid periods: 1d,5d,1mo,3mo,6mo,1y,2y,5y,10y,ytd,max
+              | Default: 1mo
+              | Can combine with start/end e.g. end = start + period
             interval : str
-                Valid intervals: 1m,2m,5m,15m,30m,60m,90m,1h,1d,5d,1wk,1mo,3mo
-                Intraday data cannot extend last 60 days
-            start: str
-                Download start date string (YYYY-MM-DD) or _datetime, inclusive.
-                Default is 99 years ago
-                E.g. for start="2020-01-01", the first data point will be on "2020-01-01"
-            end: str
-                Download end date string (YYYY-MM-DD) or _datetime, exclusive.
-                Default is now
-                E.g. for end="2023-01-01", the last data point will be on "2022-12-31"
+              | Valid intervals: 1m,2m,5m,15m,30m,60m,90m,1h,1d,5d,1wk,1mo,3mo
+              | Intraday data cannot extend last 60 days
+            start : str
+              | Download start date string (YYYY-MM-DD) or _datetime, inclusive.
+              | Default: 99 years ago
+              | E.g. for start="2020-01-01", first data point = "2020-01-01"
+            end : str
+              | Download end date string (YYYY-MM-DD) or _datetime, exclusive.
+              | Default: now
+              | E.g. for end="2023-01-01", last data point = "2022-12-31"
             prepost : bool
-                Include Pre and Post market data in results?
-                Default is False
-            auto_adjust: bool
-                Adjust all OHLC automatically? Default is True
-            back_adjust: bool
-                Back-adjusted data to mimic true historical prices
-            repair: bool
-                Detect currency unit 100x mixups and attempt repair.
-                Default is False
-            keepna: bool
-                Keep NaN rows returned by Yahoo?
-                Default is False
-            rounding: bool
-                Round values to 2 decimal places?
-                Optional. Default is False = precision suggested by Yahoo!
-            timeout: None or float
-                If not None stops waiting for a response after given number of
-                seconds. (Can also be a fraction of a second e.g. 0.01)
-                Default is 10 seconds.
-            raise_errors: bool
+              | Include Pre and Post market data in results?
+              | Default: False
+            auto_adjust : bool
+              | Adjust all OHLC automatically?
+              | Default: True
+            back_adjust : bool
+              | Back-adjusted data to mimic true historical prices
+            repair : bool
+              | Fixes price errors in Yahoo data: 100x, missing, bad dividend adjust.
+              | Default: False
+              | Full details at: :doc:`../advanced/price_repair`.
+            keepna : bool
+              | Keep NaN rows returned by Yahoo?
+              | Default: False
+            rounding : bool
+              | Optional: Round values to 2 decimal places?
+              | Default: False = use precision suggested by Yahoo!
+            timeout : None or float
+              | Optional: timeout fetches after N seconds
+              | Default: 10 seconds
+            raise_errors : bool
                 If True, then raise errors as Exceptions instead of logging.
         """
         logger = utils.get_yf_logger()
 
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=5)
             self._data._set_proxy(proxy)
 
+        if raise_errors:
+            warnings.warn("'raise_errors' deprecated, use yf.set_config(hide_exceptions=False)", DeprecationWarning, stacklevel=5)
+
         interval_user = interval
         period_user = period
         if repair and interval in ["5d", "1wk", "1mo", "3mo"]:
             # Yahoo's way of adjusting mutiday intervals is fundamentally broken.
             # Have to fetch 1d, adjust, then resample.
             if interval == '5d':
-                raise Exception("Yahoo's interval '5d' is nonsense, not supported with repair")
+                raise ValueError("Yahoo's interval '5d' is nonsense, not supported with repair")
             if start is None and end is None and period is not None:
                 tz = self.tz
                 if tz is None:
@@ -95,7 +102,7 @@ class PriceHistory:
                     err_msg = str(_exception)
                     shared._DFS[self.ticker] = utils.empty_df()
                     shared._ERRORS[self.ticker] = err_msg.split(': ', 1)[1]
-                    if raise_errors:
+                    if raise_errors or (not YfConfig().hide_exceptions):
                         raise _exception
                     else:
                         logger.error(err_msg)
@@ -112,7 +119,7 @@ class PriceHistory:
 
         start_user = start
         end_user = end
-        if start or period is None or period.lower() == "max":
+        if start or end or (period and period.lower() == "max"):
             # Check can get TZ. Fail => probably delisted
             tz = self.tz
             if tz is None:
@@ -121,27 +128,55 @@ class PriceHistory:
                 err_msg = str(_exception)
                 shared._DFS[self.ticker] = utils.empty_df()
                 shared._ERRORS[self.ticker] = err_msg.split(': ', 1)[1]
-                if raise_errors:
+                if raise_errors or (not YfConfig().hide_exceptions):
                     raise _exception
                 else:
                     logger.error(err_msg)
                 return utils.empty_df()
 
-            if end is None:
-                end = int(_time.time())
-            else:
-                end = utils._parse_user_dt(end, tz)
-            if start is None:
-                if interval == "1m":
-                    start = end - 604800   # 7 days
-                elif interval in ("5m", "15m", "30m", "90m"):
-                    start = end - 5184000  # 60 days
-                elif interval in ("1h", '60m'):
-                    start = end - 63072000  # 730 days
-                else:
-                    start = end - 3122064000  # 99 years
-            else:
-                start = utils._parse_user_dt(start, tz)
+        if start:
+            start_dt = utils._parse_user_dt(start, tz)
+            start = int(start_dt.timestamp())
+        if end:
+            end_dt = utils._parse_user_dt(end, tz)
+            end = int(end_dt.timestamp())
+
+        if period is None:
+            if not (start or end):
+                period = '1mo'  # default
+            elif not start:
+                start_dt = end_dt - utils._interval_to_timedelta('1mo')
+                start = int(start_dt.timestamp())
+            elif not end:
+                end_dt = pd.Timestamp.utcnow().tz_convert(tz)
+                end = int(end_dt.timestamp())
+        else:
+            if period.lower() == "max":
+                if end is None:
+                    end = int(_time.time())
+                if start is None:
+                    if interval == "1m":
+                        start = end - 691200  # 8 days
+                    elif interval in ("2m", "5m", "15m", "30m", "90m"):
+                        start = end - 5184000  # 60 days
+                    elif interval in ("1h", "60m"):
+                        start = end - 63072000  # 730 days
+                    else:
+                        start = end - 3122064000  # 99 years
+                    start += 5 # allow for processing time
+            elif start and end:
+                raise ValueError("Setting period, start and end is nonsense. Set maximum 2 of them.")
+            elif start or end:
+                period_td = utils._interval_to_timedelta(period)
+                if end is None:
+                    end_dt = start_dt + period_td
+                    end = int(end_dt.timestamp())
+                if start is None:
+                    start_dt = end_dt - period_td
+                    start = int(start_dt.timestamp())
+                period = None
+
+        if start or end:
             params = {"period1": start, "period2": end}
         else:
             period = period.lower()
@@ -182,22 +217,22 @@ class PriceHistory:
                 timeout=timeout
             )
             if "Will be right back" in data.text or data is None:
-                raise RuntimeError("*** YAHOO! FINANCE IS CURRENTLY DOWN! ***\n"
-                                   "Our engineers are working quickly to resolve "
-                                   "the issue. Thank you for your patience.")
+                raise YFDataException("*** YAHOO! FINANCE IS CURRENTLY DOWN! ***")
 
             data = data.json()
         # Special case for rate limits
         except YFRateLimitError:
             raise
         except Exception:
-            if raise_errors:
+            if raise_errors or (not YfConfig().hide_exceptions):
                 raise
 
         # Store the meta data that gets retrieved simultaneously
         try:
             self._history_metadata = data["chart"]["result"][0]["meta"]
         except Exception:
+            if not YfConfig().hide_exceptions:
+                raise
             self._history_metadata = {}
 
         intraday = params["interval"][-1] in ("m", 'h')
@@ -244,7 +279,7 @@ class PriceHistory:
             err_msg = str(_exception)
             shared._DFS[self.ticker] = utils.empty_df()
             shared._ERRORS[self.ticker] = err_msg.split(': ', 1)[1]
-            if raise_errors:
+            if raise_errors or (not YfConfig().hide_exceptions):
                 raise _exception
             else:
                 logger.error(err_msg)
@@ -296,6 +331,8 @@ class PriceHistory:
                 quotes['Dividends'] = quotes2['Dividends'].max()
                 quotes['Stock Splits'] = quotes2['Stock Splits'].max()
             except Exception:
+                if raise_errors or (not YfConfig().hide_exceptions):
+                    raise
                 pass
 
         # Note: ordering is important. If you change order, run the tests!
@@ -326,6 +363,24 @@ class PriceHistory:
             splits = utils.set_df_tz(splits, interval, tz_exchange)
         if dividends is not None:
             dividends = utils.set_df_tz(dividends, interval, tz_exchange)
+            if 'currency' in dividends.columns:
+                # Rare, only seen with Vietnam market
+                price_currency = self._history_metadata['currency']
+                if price_currency is None:
+                    price_currency = ''
+                f_currency_mismatch = dividends['currency'] != price_currency
+                if f_currency_mismatch.any():
+                    if not repair or price_currency == '':
+                        # Append currencies to values, let user decide action.
+                        dividends['Dividends'] = dividends['Dividends'].astype(str) + ' ' + dividends['currency']
+                    else:
+                        # Attempt repair = currency conversion
+                        dividends = self._dividends_convert_fx(dividends, price_currency, repair)
+                        if (dividends['currency'] != price_currency).any():
+                            # FX conversion failed
+                            dividends['Dividends'] = dividends['Dividends'].astype(str) + ' ' + dividends['currency']
+                dividends = dividends.drop('currency', axis=1)
+
         if capital_gains is not None:
             capital_gains = utils.set_df_tz(capital_gains, interval, tz_exchange)
         if start is not None:
@@ -403,6 +458,7 @@ class PriceHistory:
             # First make currency consistent. On some exchanges, dividends often in different currency
             # to prices, e.g. £ vs pence.
             df, currency = self._standardise_currency(df, currency)
+            self._history_metadata['currency'] = currency
 
             df = self._fix_bad_div_adjust(df, interval, currency)
 
@@ -426,16 +482,15 @@ class PriceHistory:
             elif back_adjust:
                 df = utils.back_adjust(df)
         except Exception as e:
+            if raise_errors or (not YfConfig().hide_exceptions):
+                raise
             if auto_adjust:
                 err_msg = "auto_adjust failed with %s" % e
             else:
                 err_msg = "back_adjust failed with %s" % e
             shared._DFS[self.ticker] = utils.empty_df()
             shared._ERRORS[self.ticker] = err_msg
-            if raise_errors:
-                raise Exception('%s: %s' % (self.ticker, err_msg))
-            else:
-                logger.error('%s: %s' % (self.ticker, err_msg))
+            logger.error('%s: %s' % (self.ticker, err_msg))
 
         if rounding:
             df = np.round(df, data["chart"]["result"][0]["meta"]["priceHint"])
@@ -481,7 +536,7 @@ class PriceHistory:
 
     def get_history_metadata(self, proxy=_SENTINEL_) -> dict:
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=3)
             self._data._set_proxy(proxy)
 
         if self._history_metadata is None or 'tradingPeriods' not in self._history_metadata:
@@ -496,7 +551,7 @@ class PriceHistory:
 
     def get_dividends(self, period="max", proxy=_SENTINEL_) -> pd.Series:
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=3)
             self._data._set_proxy(proxy)
 
         df = self._get_history_cache(period=period)
@@ -507,7 +562,7 @@ class PriceHistory:
 
     def get_capital_gains(self, period="max", proxy=_SENTINEL_) -> pd.Series:
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=3)
             self._data._set_proxy(proxy)
 
         df = self._get_history_cache(period=period)
@@ -518,7 +573,7 @@ class PriceHistory:
 
     def get_splits(self, period="max", proxy=_SENTINEL_) -> pd.Series:
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=3)
             self._data._set_proxy(proxy)
 
         df = self._get_history_cache(period=period)
@@ -529,7 +584,7 @@ class PriceHistory:
 
     def get_actions(self, period="max", proxy=_SENTINEL_) -> pd.Series:
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=3)
             self._data._set_proxy(proxy)
 
         df = self._get_history_cache(period=period)
@@ -566,7 +621,7 @@ class PriceHistory:
                 align_month = _datetime.datetime.now().strftime('%b').upper()
             resample_period = f"QS-{align_month}"
         else:
-            raise Exception(f"Not implemented resampling to interval '{target_interval}'")
+            raise ValueError(f"Not implemented resampling to interval '{target_interval}'")
         resample_map = {
             'Open': 'first', 'Low': 'min', 'High': 'max', 'Close': 'last',
             'Volume': 'sum', 'Dividends': 'sum', 'Stock Splits': 'prod'
@@ -589,7 +644,7 @@ class PriceHistory:
         log_extras = {'yf_cat': 'price-reconstruct', 'yf_interval': interval, 'yf_symbol': self.ticker}
 
         if not isinstance(df, pd.DataFrame):
-            raise Exception("'df' must be a Pandas DataFrame not", type(df))
+            raise ValueError("'df' must be a Pandas DataFrame not", type(df))
         if interval == "1m":
             # Can't go smaller than 1m so can't reconstruct
             return df
@@ -825,37 +880,39 @@ class PriceHistory:
                     # But in case are repairing a chunk of bad 1d data, back/forward-fill the
                     # good div-adjustments - not perfect, but a good backup.
                     div_adjusts[f_tag] = np.nan
-                    div_adjusts = div_adjusts.ffill().bfill()
-                    for idx in np.where(f_tag)[0]:
-                        dt = df_new_calib.index[idx]
-                        n = len(div_adjusts)
-                        if df_new.loc[dt, "Dividends"] != 0:
-                            if idx < n - 1:
-                                # Easy, take div-adjustment from next-day
-                                div_adjusts.iloc[idx] = div_adjusts.iloc[idx + 1]
+                    if not div_adjusts.isna().all():
+                        # Need some real values to calibrate
+                        div_adjusts = div_adjusts.ffill().bfill()
+                        for idx in np.where(f_tag)[0]:
+                            dt = df_new_calib.index[idx]
+                            n = len(div_adjusts)
+                            if df_new.loc[dt, "Dividends"] != 0:
+                                if idx < n - 1:
+                                    # Easy, take div-adjustment from next-day
+                                    div_adjusts.iloc[idx] = div_adjusts.iloc[idx + 1]
+                                else:
+                                    # Take previous-day div-adjustment and reverse todays adjustment
+                                    div_adj = 1.0 - df_new_calib["Dividends"].iloc[idx] / df_new_calib['Close'].iloc[
+                                        idx - 1]
+                                    div_adjusts.iloc[idx] = div_adjusts.iloc[idx - 1] / div_adj
                             else:
-                                # Take previous-day div-adjustment and reverse todays adjustment
-                                div_adj = 1.0 - df_new_calib["Dividends"].iloc[idx] / df_new_calib['Close'].iloc[
-                                    idx - 1]
-                                div_adjusts.iloc[idx] = div_adjusts.iloc[idx - 1] / div_adj
-                        else:
-                            if idx > 0:
-                                # Easy, take div-adjustment from previous-day
-                                div_adjusts.iloc[idx] = div_adjusts.iloc[idx - 1]
-                            else:
-                                # Must take next-day div-adjustment
-                                div_adjusts.iloc[idx] = div_adjusts.iloc[idx + 1]
-                                if df_new_calib["Dividends"].iloc[idx + 1] != 0:
-                                    div_adjusts.iloc[idx] *= 1.0 - df_new_calib["Dividends"].iloc[idx + 1] / \
-                                                        df_new_calib['Close'].iloc[idx]
-                    f_close_bad = df_block_calib['Close'] == tag
-                    div_adjusts = div_adjusts.reindex(df_block.index, fill_value=np.nan).ffill().bfill()
-                    df_new['Adj Close'] = df_block['Close'] * div_adjusts
-                    if f_close_bad.any():
-                        f_close_bad_new = f_close_bad.reindex(df_new.index, fill_value=False)
-                        div_adjusts_new = div_adjusts.reindex(df_new.index, fill_value=np.nan).ffill().bfill()
-                        div_adjusts_new_np = f_close_bad_new.to_numpy()
-                        df_new.loc[div_adjusts_new_np, 'Adj Close'] = df_new['Close'][div_adjusts_new_np] * div_adjusts_new[div_adjusts_new_np]
+                                if idx > 0:
+                                    # Easy, take div-adjustment from previous-day
+                                    div_adjusts.iloc[idx] = div_adjusts.iloc[idx - 1]
+                                else:
+                                    # Must take next-day div-adjustment
+                                    div_adjusts.iloc[idx] = div_adjusts.iloc[idx + 1]
+                                    if df_new_calib["Dividends"].iloc[idx + 1] != 0:
+                                        div_adjusts.iloc[idx] *= 1.0 - df_new_calib["Dividends"].iloc[idx + 1] / \
+                                                            df_new_calib['Close'].iloc[idx]
+                        f_close_bad = df_block_calib['Close'] == tag
+                        div_adjusts = div_adjusts.reindex(df_block.index, fill_value=np.nan).ffill().bfill()
+                        df_new['Adj Close'] = df_block['Close'] * div_adjusts
+                        if f_close_bad.any():
+                            f_close_bad_new = f_close_bad.reindex(df_new.index, fill_value=False)
+                            div_adjusts_new = div_adjusts.reindex(df_new.index, fill_value=np.nan).ffill().bfill()
+                            div_adjusts_new_np = f_close_bad_new.to_numpy()
+                            df_new.loc[div_adjusts_new_np, 'Adj Close'] = df_new['Close'][div_adjusts_new_np] * div_adjusts_new[div_adjusts_new_np]
 
             # Check whether 'df_fine' has different split-adjustment.
             # If different, then adjust to match 'df'
@@ -994,6 +1051,8 @@ class PriceHistory:
                     prices_in_subunits = False
             except Exception:
                 # Should never happen but just-in-case
+                if not YfConfig().hide_exceptions:
+                    raise
                 pass
         if prices_in_subunits:
             for c in _PRICE_COLNAMES_:
@@ -1017,6 +1076,45 @@ class PriceHistory:
 
         return df, currency2
 
+    def _dividends_convert_fx(self, dividends, fx, repair=False):
+        bad_div_currencies = [c for c in dividends['currency'].unique() if c != fx]
+        major_currencies = ['USD', 'JPY', 'EUR', 'CNY', 'GBP', 'CAD']
+        for c in bad_div_currencies:
+            fx2_tkr = None
+            if c == 'USD':
+                # Simple convert from USD to target FX
+                fx_tkr = f'{fx}=X'
+                reverse = False
+            elif fx == 'USD':
+                # Use same USD FX but reversed
+                fx_tkr = f'{fx}=X'
+                reverse = True
+            elif c in major_currencies and fx in major_currencies:
+                # Simple convert
+                fx_tkr = f'{c}{fx}=X'
+                reverse = False
+            else:
+                # No guarantee that Yahoo has direct FX conversion, so
+                # convert via USD
+                # - step 1: -> USD
+                fx_tkr = f'{c}=X'
+                reverse = True
+                # - step 2: USD -> FX
+                fx2_tkr = f'{fx}=X'
+
+            fx_dat = PriceHistory(self._data, fx_tkr, self.session)
+            fx_rate = fx_dat.history(period='1mo', repair=repair)['Close'].iloc[-1]
+            if reverse:
+                fx_rate = 1/fx_rate
+            dividends.loc[dividends['currency']==c, 'Dividends'] *= fx_rate
+            if fx2_tkr is not None:
+                fx2_dat = PriceHistory(self._data, fx2_tkr, self.session)
+                fx2_rate = fx2_dat.history(period='1mo', repair=repair)['Close'].iloc[-1]
+                dividends.loc[dividends['currency']==c, 'Dividends'] *= fx2_rate
+
+        dividends['currency'] = fx
+        return dividends
+
     @utils.log_indent_decorator
     def _fix_unit_mixups(self, df, interval, tz_exchange, prepost):
         if df.empty:
@@ -1645,7 +1743,7 @@ class PriceHistory:
                     # elif k == 'div_true_date':
                     #     div_status_df[k] = pd.Series(dtype='datetime64[ns, UTC]')
                     else:
-                        raise Exception(k,v,type(v))
+                        raise ValueError(k,v,type(v))
                 div_status_df.loc[dt, k] = v
         checks += ['adj_missing', 'adj_exceeds_div', 'div_exceeds_adj']
 
@@ -1885,7 +1983,7 @@ class PriceHistory:
                     elif k == 'div_true_date':
                         div_status_df[k] = pd.Series(dtype='datetime64[ns, UTC]')
                     else:
-                        raise Exception(k,v,type(v))
+                        raise ValueError(k,v,type(v))
                 div_status_df.loc[dt, k] = v
             if 'div_too_big' in div_status_df.columns and 'div_date_wrong' in div_status_df.columns:
                 # Where div_date_wrong = True, discard div_too_big. Helps with false-positive handling later.
@@ -2340,9 +2438,6 @@ class PriceHistory:
             msg = f"Repaired {k}: {[str(dt.date()) for dt in sorted(div_repairs[k])]}"
             logger.info(msg, extra=log_extras)
 
-        if 'Adj' in df2.columns:
-            raise Exception('"Adj" has snuck in df2')
-
         if not df2_nan.empty:
             df2 = pd.concat([df2, df2_nan]).sort_index()
 

yfinance/scrapers/holders.py

@@ -1,9 +1,11 @@
+import curl_cffi
 import pandas as pd
-import requests
+import warnings
 
 from yfinance import utils
-from yfinance.data import YfData
+from yfinance.config import YfConfig
 from yfinance.const import _BASE_URL_, _SENTINEL_
+from yfinance.data import YfData
 from yfinance.exceptions import YFDataException
 
 _QUOTE_SUMMARY_URL_ = f"{_BASE_URL_}/v10/finance/quoteSummary"
@@ -15,7 +17,7 @@ class Holders:
         self._data = data
         self._symbol = symbol
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             data._set_proxy(proxy)
 
         self._major = None
@@ -67,14 +69,16 @@ class Holders:
         modules = ','.join(
             ["institutionOwnership", "fundOwnership", "majorDirectHolders", "majorHoldersBreakdown", "insiderTransactions", "insiderHolders", "netSharePurchaseActivity"])
         params_dict = {"modules": modules, "corsDomain": "finance.yahoo.com", "formatted": "false"}
-        result = self._data.get_raw_json(f"{_QUOTE_SUMMARY_URL_}/{self._symbol}", user_agent_headers=self._data.user_agent_headers, params=params_dict)
+        result = self._data.get_raw_json(f"{_QUOTE_SUMMARY_URL_}/{self._symbol}", params=params_dict)
         return result
 
     def _fetch_and_parse(self):
         try:
             result = self._fetch()
-        except requests.exceptions.HTTPError as e:
-            utils.get_yf_logger().error(str(e))
+        except curl_cffi.requests.exceptions.HTTPError as e:
+            if not YfConfig().hide_exceptions:
+                raise
+            utils.get_yf_logger().error(str(e) + e.response.text)
 
             self._major = pd.DataFrame()
             self._major_direct_holders = pd.DataFrame()
@@ -97,6 +101,8 @@ class Holders:
             self._parse_insider_holders(data.get("insiderHolders", {}))
             self._parse_net_share_purchase_activity(data.get("netSharePurchaseActivity", {}))
         except (KeyError, IndexError):
+            if not YfConfig().hide_exceptions:
+                raise
             raise YFDataException("Failed to parse holders json data.")
 
     @staticmethod

yfinance/scrapers/quote.py

@@ -1,13 +1,14 @@
+import curl_cffi
 import datetime
 import json
-
 import numpy as _np
 import pandas as pd
-import requests
+import warnings
 
 from yfinance import utils
-from yfinance.data import YfData
+from yfinance.config import YfConfig
 from yfinance.const import quote_summary_valid_modules, _BASE_URL_, _QUERY1_URL_, _SENTINEL_
+from yfinance.data import YfData
 from yfinance.exceptions import YFDataException, YFException
 
 info_retired_keys_price = {"currentPrice", "dayHigh", "dayLow", "open", "previousClose", "volume", "volume24Hr"}
@@ -29,7 +30,7 @@ class FastInfo:
     def __init__(self, tickerBaseObject, proxy=_SENTINEL_):
         self._tkr = tickerBaseObject
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._tkr._data._set_proxy(proxy)
 
         self._prices_1y = None
@@ -106,7 +107,7 @@ class FastInfo:
 
     def __getitem__(self, k):
         if not isinstance(k, str):
-            raise KeyError("key must be a string")
+            raise KeyError(f"key must be a string not '{type(k)}'")
         if k not in self._keys:
             raise KeyError(f"'{k}' not valid key. Examine 'FastInfo.keys()'")
         if k in self._cc_to_sc_key:
@@ -467,8 +468,6 @@ class FastInfo:
         except Exception as e:
             if "Cannot retrieve share count" in str(e):
                 shares = None
-            elif "failed to decrypt Yahoo" in str(e):
-                shares = None
             else:
                 raise
 
@@ -490,7 +489,7 @@ class Quote:
         self._data = data
         self._symbol = symbol
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         self._info = None
@@ -523,6 +522,8 @@ class Quote:
                 try:
                     data = result["quoteSummary"]["result"][0]
                 except (KeyError, IndexError):
+                    if not YfConfig().hide_exceptions:
+                        raise
                     raise YFDataException(f"Failed to parse json response from Yahoo Finance: {result}")
                 self._sustainability = pd.DataFrame(data)
         return self._sustainability
@@ -537,6 +538,8 @@ class Quote:
                 try:
                     data = result["quoteSummary"]["result"][0]["recommendationTrend"]["trend"]
                 except (KeyError, IndexError):
+                    if not YfConfig().hide_exceptions:
+                        raise
                     raise YFDataException(f"Failed to parse json response from Yahoo Finance: {result}")
                 self._recommendations = pd.DataFrame(data)
         return self._recommendations
@@ -558,6 +561,8 @@ class Quote:
                     df.index = pd.to_datetime(df.index, unit='s')
                     self._upgrades_downgrades = df
                 except (KeyError, IndexError):
+                    if not YfConfig().hide_exceptions:
+                        raise
                     raise YFDataException(f"Failed to parse json response from Yahoo Finance: {result}")
         return self._upgrades_downgrades
 
@@ -587,20 +592,22 @@ class Quote:
             raise YFException("No valid modules provided, see available modules using `valid_modules`")
         params_dict = {"modules": modules, "corsDomain": "finance.yahoo.com", "formatted": "false", "symbol": self._symbol}
         try:
-            result = self._data.get_raw_json(_QUOTE_SUMMARY_URL_ + f"/{self._symbol}", user_agent_headers=self._data.user_agent_headers, params=params_dict)
-        except requests.exceptions.HTTPError as e:
-            utils.get_yf_logger().error(str(e))
+            result = self._data.get_raw_json(_QUOTE_SUMMARY_URL_ + f"/{self._symbol}", params=params_dict)
+        except curl_cffi.requests.exceptions.HTTPError as e:
+            if not YfConfig().hide_exceptions:
+                raise
+            utils.get_yf_logger().error(str(e) + e.response.text)
             return None
         return result
 
     def _fetch_additional_info(self):
         params_dict = {"symbols": self._symbol, "formatted": "false"}
         try:
-            result = self._data.get_raw_json(f"{_QUERY1_URL_}/v7/finance/quote?",
-                                             user_agent_headers=self._data.user_agent_headers,
-                                             params=params_dict)
-        except requests.exceptions.HTTPError as e:
-            utils.get_yf_logger().error(str(e))
+            result = self._data.get_raw_json(f"{_QUERY1_URL_}/v7/finance/quote?", params=params_dict)
+        except curl_cffi.requests.exceptions.HTTPError as e:
+            if not YfConfig().hide_exceptions:
+                raise
+            utils.get_yf_logger().error(str(e) + e.response.text)
             return None
         return result
 
@@ -679,7 +686,7 @@ class Quote:
             # p = _re.compile(r'root\.App\.main = (.*);')
             # url = 'https://finance.yahoo.com/quote/{}/key-statistics?p={}'.format(self._ticker.ticker, self._ticker.ticker)
             # try:
-            #     r = session.get(url, headers=utils.user_agent_headers)
+            #     r = session.get(url)
             #     data = _json.loads(p.findall(r.text)[0])
             #     key_stats = data['context']['dispatcher']['stores']['QuoteTimeSeriesStore']["timeSeries"]
             #     for k in keys:
@@ -742,6 +749,8 @@ class Quote:
                 self._calendar['Revenue Low'] = earnings.get('revenueLow', None)
                 self._calendar['Revenue Average'] = earnings.get('revenueAverage', None)
         except (KeyError, IndexError):
+            if not YfConfig().hide_exceptions:
+                raise
             raise YFDataException(f"Failed to parse json response from Yahoo Finance: {result}")
 
 

yfinance/screener/query.py

@@ -165,7 +165,7 @@ class EquityQuery(QueryBase):
         """
         return EQUITY_SCREENER_FIELDS
     
-    @dynamic_docstring({"valid_values_table": generate_list_table_from_dict_universal(EQUITY_SCREENER_EQ_MAP, concat_keys=['exchange'])})
+    @dynamic_docstring({"valid_values_table": generate_list_table_from_dict_universal(EQUITY_SCREENER_EQ_MAP, concat_keys=['exchange', 'industry'])})
     @property
     def valid_values(self) -> Dict:
         """

yfinance/screener/screener.py

@@ -1,14 +1,14 @@
-from .query import EquityQuery as EqyQy
-from .query import FundQuery as FndQy
-from .query import QueryBase, EquityQuery, FundQuery
+import curl_cffi
+from typing import Union
+import warnings
 
 from yfinance.const import _QUERY1_URL_, _SENTINEL_
 from yfinance.data import YfData
+from ..utils import dynamic_docstring, generate_list_table_from_dict_universal
 
-from ..utils import dynamic_docstring, generate_list_table_from_dict_universal, print_once
-
-from typing import Union
-import requests
+from .query import EquityQuery as EqyQy
+from .query import FundQuery as FndQy
+from .query import QueryBase, EquityQuery, FundQuery
 
 _SCREENER_URL_ = f"{_QUERY1_URL_}/v1/finance/screener"
 _PREDEFINED_URL_ = f"{_SCREENER_URL_}/predefined/saved"
@@ -112,7 +112,7 @@ def screen(query: Union[str, EquityQuery, FundQuery],
     """
 
     if proxy is not _SENTINEL_:
-        print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+        warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
         _data = YfData(session=session, proxy=proxy)
     else:
         _data = YfData(session=session)
@@ -135,6 +135,18 @@ def screen(query: Union[str, EquityQuery, FundQuery],
     if size is not None and size > 250:
         raise ValueError("Yahoo limits query size to 250, reduce size.")
 
+    if offset is not None and isinstance(query, str):
+        # offset ignored by predefined API so switch to other API
+        post_query = PREDEFINED_SCREENER_QUERIES[query]
+        query = post_query['query']
+        # use predefined's attributes if user not specified
+        if sortField is None:
+            sortField = post_query['sortField']
+        if sortAsc is None:
+            sortAsc = post_query['sortType'].lower() == 'asc'
+        # and don't use defaults
+        defaults = {}
+
     fields = {'offset': offset, 'count': count, "size": size, 'sortField': sortField, 'sortAsc': sortAsc, 'userId': userId, 'userIdType': userIdType}
 
     params_dict = {"corsDomain": "finance.yahoo.com", "formatted": "false", "lang": "en-US", "region": "US"}
@@ -145,7 +157,7 @@ def screen(query: Union[str, EquityQuery, FundQuery],
         # Switch to Yahoo's predefined endpoint
 
         if size is not None:
-            print_once("YF deprecation warning: 'size' argument is deprecated for predefined screens, set 'count' instead.")
+            warnings.warn("Screen 'size' argument is deprecated for predefined screens, set 'count' instead.", DeprecationWarning, stacklevel=2)
             count = size
             size = None
             fields['count'] = fields['size']
@@ -158,7 +170,7 @@ def screen(query: Union[str, EquityQuery, FundQuery],
         resp = _data.get(url=_PREDEFINED_URL_, params=params_dict)
         try:
             resp.raise_for_status()
-        except requests.exceptions.HTTPError:
+        except curl_cffi.requests.exceptions.HTTPError:
             if query not in PREDEFINED_SCREENER_QUERIES:
                 print(f"yfinance.screen: '{query}' is probably not a predefined query.")
             raise
@@ -190,7 +202,6 @@ def screen(query: Union[str, EquityQuery, FundQuery],
     # Fetch
     response = _data.post(_SCREENER_URL_, 
                             body=post_query, 
-                            user_agent_headers=_data.user_agent_headers, 
                             params=params_dict)
     response.raise_for_status()
     return response.json()['finance']['result'][0]

yfinance/search.py

@@ -20,10 +20,13 @@
 #
 
 import json as _json
+import warnings
 
 from . import utils
+from .config import YfConfig
 from .const import _BASE_URL_, _SENTINEL_
 from .data import YfData
+from .exceptions import YFDataException
 
 
 class Search:
@@ -52,7 +55,7 @@ class Search:
         self._data = YfData(session=self.session)
         
         if proxy is not _SENTINEL_:
-            utils.print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
 
         self.query = query
@@ -103,13 +106,13 @@ class Search:
 
         data = self._data.cache_get(url=url, params=params, timeout=self.timeout)
         if data is None or "Will be right back" in data.text:
-            raise RuntimeError("*** YAHOO! FINANCE IS CURRENTLY DOWN! ***\n"
-                               "Our engineers are working quickly to resolve "
-                               "the issue. Thank you for your patience.")
+            raise YFDataException("*** YAHOO! FINANCE IS CURRENTLY DOWN! ***")
         try:
             data = data.json()
         except _json.JSONDecodeError:
-            self._logger.error(f"{self.query}: Failed to retrieve search results and received faulty response instead.")
+            if not YfConfig().hide_exceptions:
+                raise
+            self._logger.error(f"{self.query}: 'search' fetch received faulty data")
             data = {}
 
         self._response = data

yfinance/ticker.py

@@ -22,17 +22,21 @@
 from __future__ import print_function
 
 from collections import namedtuple as _namedtuple
-from .scrapers.funds import FundsData
+import warnings
 
 import pandas as _pd
 
 from .base import TickerBase
 from .const import _BASE_URL_, _SENTINEL_
+from .scrapers.funds import FundsData
 
 
 class Ticker(TickerBase):
     def __init__(self, ticker, session=None, proxy=_SENTINEL_):
-        super(Ticker, self).__init__(ticker, session=session, proxy=proxy)
+        if proxy is not _SENTINEL_:
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
+            self._data._set_proxy(proxy)
+        super(Ticker, self).__init__(ticker, session=session)
         self._expirations = {}
         self._underlying  = {}
 

yfinance/tickers.py

@@ -21,8 +21,10 @@
 
 from __future__ import print_function
 
+import warnings
+
 from . import Ticker, multi
-from .utils import print_once
+from .live import WebSocket
 from .data import YfData
 from .const import _SENTINEL_
 
@@ -40,6 +42,9 @@ class Tickers:
 
         self._data = YfData(session=session)
 
+        self._message_handler = None
+        self.ws = None
+
         # self.tickers = _namedtuple(
         #     "Tickers", ticker_objects.keys(), rename=True
         # )(*ticker_objects.values())
@@ -52,8 +57,9 @@ class Tickers:
                 timeout=10, **kwargs):
 
         if proxy is not _SENTINEL_:
-            print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
+            proxy = _SENTINEL_
 
         return self.download(
             period, interval,
@@ -71,8 +77,9 @@ class Tickers:
                  timeout=10, **kwargs):
 
         if proxy is not _SENTINEL_:
-            print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
+            warnings.warn("Set proxy via new config function: yf.set_config(proxy=proxy)", DeprecationWarning, stacklevel=2)
             self._data._set_proxy(proxy)
+            proxy = _SENTINEL_
 
         data = multi.download(self.symbols,
                               start=start, end=end,
@@ -99,3 +106,10 @@ class Tickers:
 
     def news(self):
         return {ticker: [item for item in Ticker(ticker).news] for ticker in self.symbols}
+
+    def live(self, message_handler=None, verbose=True):
+        self._message_handler = message_handler
+
+        self.ws = WebSocket(verbose=verbose)
+        self.ws.subscribe(self.symbols)
+        self.ws.listen(self._message_handler)

yfinance/utils.py

@@ -27,7 +27,7 @@ import re
 import re as _re
 import sys as _sys
 import threading
-from functools import lru_cache, wraps
+from functools import wraps
 from inspect import getmembers
 from types import FunctionType
 from typing import List, Optional
@@ -39,9 +39,7 @@ from dateutil.relativedelta import relativedelta
 from pytz import UnknownTimeZoneError
 
 from yfinance import const
-
-user_agent_headers = {
-    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
+from yfinance.exceptions import YFException
 
 # From https://stackoverflow.com/a/59128615
 def attributes(obj):
@@ -53,13 +51,6 @@ def attributes(obj):
         if name[0] != '_' and name not in disallowed_names and hasattr(obj, name)}
 
 
-@lru_cache(maxsize=20)
-def print_once(msg):
-    # 'warnings' module suppression of repeat messages does not work.
-    # This function replicates correct behaviour
-    print(msg)
-
-
 # Logging
 # Note: most of this logic is adding indentation with function depth,
 #       so that DEBUG log is readable.
@@ -184,18 +175,14 @@ def is_isin(string):
     return bool(_re.match("^([A-Z]{2})([A-Z0-9]{9})([0-9])$", string))
 
 
-def get_all_by_isin(isin, proxy=const._SENTINEL_, session=None):
+def get_all_by_isin(isin):
     if not (is_isin(isin)):
         raise ValueError("Invalid ISIN number")
 
-    if proxy is not const._SENTINEL_:
-        print_once("YF deprecation warning: set proxy via new config function: yf.set_config(proxy=proxy)")
-        proxy = None
-
     # Deferred this to prevent circular imports
     from .search import Search
 
-    search = Search(query=isin, max_results=1, session=session, proxy=proxy)
+    search = Search(query=isin, max_results=1)
 
     # Extract the first quote and news
     ticker = search.quotes[0] if search.quotes else {}
@@ -213,18 +200,18 @@ def get_all_by_isin(isin, proxy=const._SENTINEL_, session=None):
     }
 
 
-def get_ticker_by_isin(isin, proxy=const._SENTINEL_, session=None):
-    data = get_all_by_isin(isin, proxy, session)
+def get_ticker_by_isin(isin):
+    data = get_all_by_isin(isin)
     return data.get('ticker', {}).get('symbol', '')
 
 
-def get_info_by_isin(isin, proxy=const._SENTINEL_, session=None):
-    data = get_all_by_isin(isin, proxy, session)
+def get_info_by_isin(isin):
+    data = get_all_by_isin(isin)
     return data.get('ticker', {})
 
 
-def get_news_by_isin(isin, proxy=const._SENTINEL_, session=None):
-    data = get_all_by_isin(isin, proxy, session)
+def get_news_by_isin(isin):
+    data = get_all_by_isin(isin)
     return data.get('news', {})
 
 
@@ -415,18 +402,21 @@ def snake_case_2_camelCase(s):
 
 def _parse_user_dt(dt, exchange_tz):
     if isinstance(dt, int):
-        # Should already be epoch, test with conversion:
-        _datetime.datetime.fromtimestamp(dt)
+        dt = _pd.Timestamp(dt, unit="s", tz=exchange_tz)
     else:
         # Convert str/date -> datetime, set tzinfo=exchange, get timestamp:
         if isinstance(dt, str):
             dt = _datetime.datetime.strptime(str(dt), '%Y-%m-%d')
         if isinstance(dt, _datetime.date) and not isinstance(dt, _datetime.datetime):
             dt = _datetime.datetime.combine(dt, _datetime.time(0))
-        if isinstance(dt, _datetime.datetime) and dt.tzinfo is None:
-            # Assume user is referring to exchange's timezone
-            dt = _tz.timezone(exchange_tz).localize(dt)
-        dt = int(dt.timestamp())
+        if isinstance(dt, _datetime.datetime):
+            if dt.tzinfo is None:
+                # Assume user is referring to exchange's timezone
+                dt = _pd.Timestamp(dt).tz_localize(exchange_tz)
+            else:
+                dt = _pd.Timestamp(dt).tz_convert(exchange_tz)
+        else: # if we reached here, then it hasn't been any known type
+            raise ValueError(f"Unable to parse input dt {dt} of type {type(dt)}")
     return dt
 
 
@@ -533,7 +523,10 @@ def parse_actions(data):
             dividends.set_index("date", inplace=True)
             dividends.index = _pd.to_datetime(dividends.index, unit="s")
             dividends.sort_index(inplace=True)
-            dividends.columns = ["Dividends"]
+            if 'currency' in dividends.columns and (dividends['currency'] == '').all():
+                # Currency column useless, drop it.
+                dividends = dividends.drop('currency', axis=1)
+            dividends = dividends.rename(columns={'amount': 'Dividends'})
 
         if "capitalGains" in data["events"] and len(data["events"]['capitalGains']) > 0:
             capital_gains = _pd.DataFrame(
@@ -710,14 +703,10 @@ def fix_Yahoo_returning_live_separate(quotes, interval, tz_exchange, prepost, re
 
 
 def safe_merge_dfs(df_main, df_sub, interval):
-    if df_sub.empty:
-        raise Exception("No data to merge")
     if df_main.empty:
         return df_main
 
     data_cols = [c for c in df_sub.columns if c not in df_main]
-    if len(data_cols) > 1:
-        raise Exception("Expected 1 data col")
     data_col = data_cols[0]
 
     df_main = df_main.sort_index()
@@ -758,6 +747,13 @@ def safe_merge_dfs(df_main, df_sub, interval):
             if df_sub.empty:
                 df_main['Dividends'] = 0.0
                 return df_main
+
+            # df_sub changed so recalc indices:
+            df_main['_date'] = df_main.index.date
+            df_sub['_date'] = df_sub.index.date
+            indices = _np.searchsorted(_np.append(df_main['_date'], [df_main['_date'].iloc[-1]+td]), df_sub['_date'], side='left')
+            df_main = df_main.drop('_date', axis=1)
+            df_sub = df_sub.drop('_date', axis=1)
         else:
             empty_row_data = {**{c:[_np.nan] for c in const._PRICE_COLNAMES_}, 'Volume':[0]}
             if interval == '1d':
@@ -794,7 +790,7 @@ def safe_merge_dfs(df_main, df_sub, interval):
     f_outOfRange = indices == -1
     if f_outOfRange.any():
         if intraday or interval in ['1d', '1wk']:
-            raise Exception(f"The following '{data_col}' events are out-of-range, did not expect with interval {interval}: {df_sub.index[f_outOfRange]}")
+            raise YFException(f"The following '{data_col}' events are out-of-range, did not expect with interval {interval}: {df_sub.index[f_outOfRange]}")
         get_yf_logger().debug(f'Discarding these {data_col} events:' + '\n' + str(df_sub[f_outOfRange]))
         df_sub = df_sub[~f_outOfRange].copy()
         indices = indices[~f_outOfRange]
@@ -816,7 +812,7 @@ def safe_merge_dfs(df_main, df_sub, interval):
             df = df.groupby("_NewIndex").prod()
             df.index.name = None
         else:
-            raise Exception(f"New index contains duplicates but unsure how to aggregate for '{data_col_name}'")
+            raise YFException(f"New index contains duplicates but unsure how to aggregate for '{data_col_name}'")
         if "_NewIndex" in df.columns:
             df = df.drop("_NewIndex", axis=1)
         return df
@@ -828,7 +824,7 @@ def safe_merge_dfs(df_main, df_sub, interval):
     f_na = df[data_col].isna()
     data_lost = sum(~f_na) < df_sub.shape[0]
     if data_lost:
-        raise Exception('Data was lost in merge, investigate')
+        raise YFException('Data was lost in merge, investigate')
 
     return df
 

yfinance/version.py

@@ -1 +1 @@
-version = "0.2.58"
+version = "0.2.65"