better idea: check if table exists after init

sync main -> dev

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -55,7 +55,7 @@ body:
     id: debug-log
     attributes:
       label: "Debug log"
-      description: "Run code with debug logging enabled and post the full output. IMPORTANT INSTRUCTIONS: https://github.com/ranaroussi/yfinance/tree/main#logging"
+      description: "Run code with debug logging enabled - `yf.enable_debug_mode()` - and post the full output. Context: https://github.com/ranaroussi/yfinance/tree/main#logging"
     validations:
       required: true
 

.github/workflows/python-publish.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python
       uses: actions/setup-python@v4
       with:

CHANGELOG.rst

@@ -1,6 +1,24 @@
 Change Log
 ===========
 
+0.2.43
+------
+Fix price-repair bug introduced in 0.2.42 #2036
+
+0.2.42
+------
+Features:
+- fetch SEC filings #2009
+- fetch analysis #2023 @Fidasek009
+- price repair extended to dividends & adjust #2031
+Fixes:
+- fix error on empty options chain #1995 @stevenbischoff
+- use dict.get() to safely access key in Holders #2013 @ericpien
+- fix datetime conversion with mixed timezones when ignore_tz is False #2016 @mreiche
+- handle faulty response object when getting news. #2021 @ericpien
+Maintenance:
+- prices: improve exceptions and logging #2000
+
 0.2.41
 ------
 Improvements:

README.md

@@ -121,7 +121,7 @@ msft.insider_transactions
 msft.insider_purchases
 msft.insider_roster_holders
 
-msfs.sustainability
+msft.sustainability
 
 # show recommendations
 msft.recommendations

meta.yaml

@@ -1,5 +1,5 @@
 {% set name = "yfinance" %}
-{% set version = "0.2.41" %}
+{% set version = "0.2.43" %}
 
 package:
   name: "{{ name|lower }}"

tests/test_prices.py

@@ -31,17 +31,23 @@ class TestPriceHistory(unittest.TestCase):
                 f = df.index.time == _dt.time(0)
                 self.assertTrue(f.all())
 
-    def test_download(self):
+    def test_download_multi_large_interval(self):
         tkrs = ["BHP.AX", "IMP.JO", "BP.L", "PNL.L", "INTC"]
         intervals = ["1d", "1wk", "1mo"]
         for interval in intervals:
-            df = yf.download(tkrs, period="5y", interval=interval)
+            with self.subTest(interval):
+                df = yf.download(tkrs, period="5y", interval=interval)
 
-            f = df.index.time == _dt.time(0)
-            self.assertTrue(f.all())
+                f = df.index.time == _dt.time(0)
+                self.assertTrue(f.all())
 
-            df_tkrs = df.columns.levels[1]
-            self.assertEqual(sorted(tkrs), sorted(df_tkrs))
+                df_tkrs = df.columns.levels[1]
+                self.assertEqual(sorted(tkrs), sorted(df_tkrs))
+
+    def test_download_multi_small_interval(self):
+        use_tkrs = ["AAPL", "0Q3.DE", "ATVI"]
+        df = yf.download(use_tkrs, period="1d", interval="5m")
+        self.assertEqual(df.index.tz, _dt.timezone.utc)
 
     def test_download_with_invalid_ticker(self):
         #Checks if using an invalid symbol gives the same output as not using an invalid symbol in combination with a valid symbol (AAPL)
@@ -172,9 +178,8 @@ class TestPriceHistory(unittest.TestCase):
             if df_daily_divs.shape[0] == 0:
                 continue
 
-            last_div_date = df_daily_divs.index[-1]
-            start_d = last_div_date.date()
-            end_d = last_div_date.date() + _dt.timedelta(days=1)
+            start_d = df_daily_divs.index[0].date()
+            end_d = df_daily_divs.index[-1].date() + _dt.timedelta(days=1)
             df_intraday = yf.Ticker(tkr, session=self.session).history(start=start_d, end=end_d, interval="15m", actions=True)
             self.assertTrue((df_intraday["Dividends"] != 0.0).any())
 
@@ -200,9 +205,8 @@ class TestPriceHistory(unittest.TestCase):
             if df_daily_divs.shape[0] == 0:
                 continue
 
-            last_div_date = df_daily_divs.index[-1]
-            start_d = last_div_date.date()
-            end_d = last_div_date.date() + _dt.timedelta(days=1)
+            start_d = df_daily_divs.index[0].date()
+            end_d = df_daily_divs.index[-1].date() + _dt.timedelta(days=1)
             df_intraday = yf.Ticker(tkr, session=self.session).history(start=start_d, end=end_d, interval="15m", actions=True)
             self.assertTrue((df_intraday["Dividends"] != 0.0).any())
 

tests/test_utils.py

@@ -8,9 +8,10 @@ Specific test class:
    python -m unittest tests.utils.TestTicker
 
 """
+from datetime import datetime
 from unittest import TestSuite
 
-# import pandas as pd
+import pandas as pd
 # import numpy as np
 
 from .context import yfinance as yf
@@ -81,10 +82,34 @@ class TestCacheNoPermission(unittest.TestCase):
         cache.lookup(tkr)
 
 
+class TestPandas(unittest.TestCase):
+    date_strings = ["2024-08-07 09:05:00+02:00", "2024-08-07 09:05:00-04:00"]
+
+    @unittest.expectedFailure
+    def test_mixed_timezones_to_datetime_fails(self):
+        series = pd.Series(self.date_strings)
+        series = series.map(pd.Timestamp)
+        converted = pd.to_datetime(series)
+        self.assertIsNotNone(converted[0].tz)
+
+    def test_mixed_timezones_to_datetime(self):
+        series = pd.Series(self.date_strings)
+        series = series.map(pd.Timestamp)
+        converted = pd.to_datetime(series, utc=True)
+        self.assertIsNotNone(converted[0].tz)
+        i = 0
+        for dt in converted:
+            dt: datetime
+            ts: pd.Timestamp = series[i]
+            self.assertEqual(dt.isoformat(), ts.tz_convert(tz="UTC").isoformat())
+            i += 1
+
+
 def suite():
     ts: TestSuite = unittest.TestSuite()
     ts.addTest(TestCache('Test cache'))
     ts.addTest(TestCacheNoPermission('Test cache no permission'))
+    ts.addTest(TestPandas("Test pandas"))
     return ts
 
 

yfinance/cache.py

@@ -153,6 +153,18 @@ class _TzCache:
                 db.create_tables([_KV])
             else:
                 raise
+
+        # # Verify that the database file actually exists.
+        # # Maybe peewee silently failed to create file and 
+        # # fellback to in-memory database:
+        # if not _os.path.exists(db.database):
+        #     self.initialised = 0  # failure
+        #     return
+        # Verify that the table was actually created
+        if not db.table_exists('_kv'):
+            self.initialised = 0  # failure
+            return
+
         self.initialised = 1  # success
 
     def lookup(self, key):

yfinance/multi.py

@@ -211,7 +211,7 @@ def download(tickers, start=None, end=None, actions=False, threads=True, ignore_
         _realign_dfs()
         data = _pd.concat(shared._DFS.values(), axis=1, sort=True,
                           keys=shared._DFS.keys(), names=['Ticker', 'Price'])
-    data.index = _pd.to_datetime(data.index)
+    data.index = _pd.to_datetime(data.index, utc=True)
     # switch names back to isins if applicable
     data.rename(columns=shared._ISINS, inplace=True)
 

yfinance/scrapers/history.py

@@ -258,9 +258,7 @@ class PriceHistory:
         # 2) fix weired bug with Yahoo! - returning 60m for 30m bars
         if interval.lower() == "30m":
             logger.debug(f'{self.ticker}: resampling 30m OHLC from 15m')
-            exchangeStartTime = pd.Timestamp(self._history_metadata["tradingPeriods"][0][0]["start"], unit='s')
-            offset = str(exchangeStartTime.minute % 30)+"min"
-            quotes2 = quotes.resample('30min', offset=offset)
+            quotes2 = quotes.resample('30min')
             quotes = pd.DataFrame(index=quotes2.last().index, data={
                 'Open': quotes2['Open'].first(),
                 'High': quotes2['High'].max(),
@@ -370,48 +368,7 @@ class PriceHistory:
             # Must fix bad 'Adj Close' & dividends before 100x/split errors.
             # First make currency consistent. On some exchanges, dividends often in different currency
             # to prices, e.g. £ vs pence.
-            if currency in ["GBp", "ZAc", "ILA"]:
-                if currency == 'GBp':
-                    # UK £/pence
-                    currency = 'GBP'
-                    m = 0.01
-                elif currency == 'ZAc':
-                    # South Africa Rand/cents
-                    currency = 'ZAR'
-                    m = 0.01
-                elif currency == 'ILA':
-                    # Israel Shekels/Agora
-                    currency = 'ILS'
-                    m = 0.01
-
-                prices_in_subunits = True  # usually is true
-                if df.index[-1] > (pd.Timestamp.utcnow() - _datetime.timedelta(days=30)):
-                    try:
-                        ratio = self._history_metadata['regularMarketPrice'] / self._history_metadata['chartPreviousClose']
-                        if abs((ratio*m)-1) < 0.1:
-                            # within 10% of 100x
-                            prices_in_subunits = True
-                    except Exception:
-                        pass
-                if prices_in_subunits:
-                    for c in _PRICE_COLNAMES_:
-                        df[c] *= m
-                self._history_metadata["currency"] = currency
-
-                f_div = df['Dividends']!=0.0
-                if f_div.any():
-                    # But sometimes the dividend was in pence.
-                    # Heuristic is: if dividend yield is ridiculous high vs converted prices, then
-                    # assume dividend was also in pence and convert to GBP.
-                    # Threshold for "ridiculous" based on largest yield I've seen anywhere - 63.4%
-                    # If this simple heuritsic generates a false positive, then _fix_bad_div_adjust()
-                    # will detect and repair.
-                    divs = df[['Close','Dividends']].copy()
-                    divs['Close'] = divs['Close'].ffill().shift(1, fill_value=divs['Close'].iloc[0])
-                    divs = divs[f_div]
-                    div_pcts = (divs['Dividends'] / divs['Close']).to_numpy()
-                    if len(div_pcts) > 0 and np.average(div_pcts) > 1:
-                        df['Dividends'] *= m
+            df, currency = self._standardise_currency(df, currency)
 
             df = self._fix_bad_div_adjust(df, interval, currency)
 
@@ -935,6 +892,61 @@ class PriceHistory:
 
         return df_v2
 
+    def _standardise_currency(self, df, currency):
+        if currency not in ["GBp", "ZAc", "ILA"]:
+            return df, currency
+        currency2 = currency
+        if currency == 'GBp':
+            # UK £/pence
+            currency2 = 'GBP'
+            m = 0.01
+        elif currency == 'ZAc':
+            # South Africa Rand/cents
+            currency2 = 'ZAR'
+            m = 0.01
+        elif currency == 'ILA':
+            # Israel Shekels/Agora
+            currency2 = 'ILS'
+            m = 0.01
+
+        # Use latest row with actual volume, because volume=0 rows can be 0.01x the other rows.
+        # _fix_unit_switch() will ensure all rows are on same scale.
+        f_volume = df['Volume']>0
+        if not f_volume.any():
+            return df, currency
+        last_row = df.iloc[np.where(f_volume)[0][-1]]
+        prices_in_subunits = True  # usually is true
+        if last_row.name > (pd.Timestamp.utcnow() - _datetime.timedelta(days=30)):
+            try:
+                ratio = self._history_metadata['regularMarketPrice'] / last_row['Close']
+                if abs((ratio*m)-1) < 0.1:
+                    # within 10% of 100x
+                    prices_in_subunits = False
+            except Exception:
+                # Should never happen but just-in-case
+                pass
+        if prices_in_subunits:
+            for c in _PRICE_COLNAMES_:
+                df[c] *= m
+        self._history_metadata["currency"] = currency
+
+        f_div = df['Dividends']!=0.0
+        if f_div.any():
+            # But sometimes the dividend was in pence.
+            # Heuristic is: if dividend yield is ridiculous high vs converted prices, then
+            # assume dividend was also in pence and convert to GBP.
+            # Threshold for "ridiculous" based on largest yield I've seen anywhere - 63.4%
+            # If this simple heuristic generates a false positive, then _fix_bad_div_adjust()
+            # will detect and repair.
+            divs = df[['Close','Dividends']].copy()
+            divs['Close'] = divs['Close'].ffill().shift(1, fill_value=divs['Close'].iloc[0])
+            divs = divs[f_div]
+            div_pcts = (divs['Dividends'] / divs['Close']).to_numpy()
+            if len(div_pcts) > 0 and np.average(div_pcts) > 1:
+                df['Dividends'] *= m
+
+        return df, currency2
+
     @utils.log_indent_decorator
     def _fix_unit_mixups(self, df, interval, tz_exchange, prepost):
         if df.empty:

yfinance/version.py

@@ -1 +1 @@
-version = "0.2.41"
+version = "0.2.43"