Fix typo and add tests

- Internal subscriptions start from StartDate during warmup
- Do not allow plotting during warmup and live trading.

Algorithm.CSharp/BacktestingBrokerageRegressionAlgorithm.cs

@@ -296,7 +296,7 @@ namespace QuantConnect.Algorithm.CSharp
         /// <summary>
         /// Data Points count of all timeslices of algorithm
         /// </summary>
-        public long DataPoints => 1753491;
+        public long DataPoints => 1748811;
 
         /// <summary>
         /// Data Points count of the algorithm history

Algorithm.CSharp/BasicTemplateFuturesDailyAlgorithm.cs

@@ -111,7 +111,7 @@ namespace QuantConnect.Algorithm.CSharp
         /// <summary>
         /// Data Points count of all timeslices of algorithm
         /// </summary>
-        public virtual long DataPoints => 17041;
+        public virtual long DataPoints => 13088;
 
         /// <summary>
         /// Data Points count of the algorithm history

Algorithm.CSharp/BasicTemplateFuturesHourlyAlgorithm.cs

@@ -47,7 +47,7 @@ namespace QuantConnect.Algorithm.CSharp
         /// <summary>
         /// Data Points count of all timeslices of algorithm
         /// </summary>
-        public override long DataPoints => 249038;
+        public override long DataPoints => 205553;
 
         /// <summary>
         /// Data Points count of the algorithm history

Algorithm.CSharp/BasicTemplateOptionsDailyAlgorithm.cs

@@ -122,7 +122,7 @@ namespace QuantConnect.Algorithm.CSharp
         /// <summary>
         /// Data Points count of all timeslices of algorithm
         /// </summary>
-        public long DataPoints => 41730;
+        public long DataPoints => 39654;
 
         /// <summary>
         /// Data Points count of the algorithm history

Algorithm.CSharp/BasicTemplateOptionsHourlyAlgorithm.cs

@@ -109,7 +109,7 @@ namespace QuantConnect.Algorithm.CSharp
         /// <summary>
         /// Data Points count of all timeslices of algorithm
         /// </summary>
-        public long DataPoints => 64216;
+        public long DataPoints => 32492;
 
         /// <summary>
         /// Data Points count of the algorithm history

Algorithm.CSharp/FuturesExpiredContractRegression.cs

@@ -81,7 +81,7 @@ namespace QuantConnect.Algorithm.CSharp
         /// <summary>
         /// Data Points count of all timeslices of algorithm
         /// </summary>
-        public long DataPoints => 588056;
+        public long DataPoints => 967006;
 
         /// <summary>
         /// Data Points count of the algorithm history

Algorithm.CSharp/OpenInterestFuturesRegressionAlgorithm.cs

@@ -92,12 +92,12 @@ namespace QuantConnect.Algorithm.CSharp
         /// <summary>
         /// Data Points count of all timeslices of algorithm
         /// </summary>
-        public long DataPoints => 579269;
+        public long DataPoints => 7080848;
 
         /// <summary>
         /// Data Points count of the algorithm history
         /// </summary>
-        public int AlgorithmHistoryDataPoints => 152;
+        public int AlgorithmHistoryDataPoints => 232;
 
         /// <summary>
         /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm

Algorithm.CSharp/OptionExerciseAssignRegressionAlgorithm.cs

@@ -120,7 +120,7 @@ namespace QuantConnect.Algorithm.CSharp
         /// <summary>
         /// Data Points count of all timeslices of algorithm
         /// </summary>
-        public long DataPoints => 1752483;
+        public long DataPoints => 1747803;
 
         /// <summary>
         /// Data Points count of the algorithm history

Algorithm.CSharp/OptionTimeSliceRegressionAlgorithm.cs

@@ -86,7 +86,7 @@ namespace QuantConnect.Algorithm.CSharp
         /// <summary>
         /// Data Points count of all timeslices of algorithm
         /// </summary>
-        public long DataPoints => 6266;
+        public long DataPoints => 7044;
 
         /// <summary>
         /// Data Points count of the algorithm history

Algorithm/QCAlgorithm.Plotting.cs

@@ -25,6 +25,7 @@ namespace QuantConnect.Algorithm
 {
     public partial class QCAlgorithm
     {
+        private bool _isEmitWarmupPlotWarningSet;
         private readonly ConcurrentDictionary<string, Chart> _charts = new ConcurrentDictionary<string, Chart>();
 
         private static readonly Dictionary<string, List<string>> ReservedChartSeriesNames = new Dictionary<string, List<string>>
@@ -210,6 +211,15 @@ namespace QuantConnect.Algorithm
                 thisChart.AddSeries(new Series(series, SeriesType.Line, 0, "$"));
             }
 
+            if (LiveMode && IsWarmingUp)
+            {
+                if (!_isEmitWarmupPlotWarningSet)
+                {
+                    _isEmitWarmupPlotWarningSet = true;
+                    Error("Warning: plotting is disabled during algorithm warmup in live trading.");
+                }
+                return;
+            }
             thisChart.Series[series].AddPoint(UtcTime, value);
         }
 

Algorithm/QCAlgorithm.cs

@@ -2094,8 +2094,11 @@ namespace QuantConnect.Algorithm
                 return false;
             }
 
-            // cancel open orders
-            Transactions.CancelOpenOrders(security.Symbol);
+            if (!IsWarmingUp)
+            {
+                // cancel open orders
+                Transactions.CancelOpenOrders(security.Symbol);
+            }
 
             // liquidate if invested
             if (security.Invested)

Common/Data/Slice.cs

@@ -410,20 +410,35 @@ namespace QuantConnect.Data
                     var dataDictionaryCache = GenericDataDictionary.Get(type, isPythonData);
                     dictionary = Activator.CreateInstance(dataDictionaryCache.GenericType);
 
-                    foreach (var data in instance._data.Value.Values.Select(x => x.Custom).Where(o =>
-                             {
-                                 if (o == null)
-                                 {
-                                     return false;
-                                 }
-                                 if (isPythonData && o is PythonData data)
-                                 {
-                                     return data.IsOfType(type);
-                                 }
-                                 return o.GetType() == type;
-                             }))
+                    var selector = (BaseData value) => {
+                        if (value == null)
+                        {
+                            return false;
+                        }
+                        if (isPythonData && value is PythonData data)
+                        {
+                            return data.IsOfType(type);
+                        }
+                        return value.GetType() == type;
+                    };
+
+                    foreach (var data in instance._data.Value.Values)
                     {
-                        dataDictionaryCache.MethodInfo.Invoke(dictionary, new object[] { data.Symbol, data });
+                        // search both in auxiliary and custom data of the slice for the requested type
+                        if(selector(data.Custom))
+                        {
+                            dataDictionaryCache.MethodInfo.Invoke(dictionary, new object[] { data.Symbol, data.Custom });
+                        }
+                        else
+                        {
+                            for (int i = 0; i < data.AuxilliaryData.Count; i++)
+                            {
+                                if (selector(data.AuxilliaryData[i]))
+                                {
+                                    dataDictionaryCache.MethodInfo.Invoke(dictionary, new object[] { data.Symbol, data.AuxilliaryData[i] });
+                                }
+                            }
+                        }
                     }
                 }
 

Common/Data/UniverseSelection/BaseDataCollection.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -28,6 +28,16 @@ namespace QuantConnect.Data.UniverseSelection
     {
         private DateTime _endTime;
 
+        /// <summary>
+        /// The associated underlying price data if any
+        /// </summary>
+        public BaseData Underlying { get; set; }
+
+        /// <summary>
+        /// Gets or sets the contracts selected by the universe
+        /// </summary>
+        public IReadOnlyCollection<Symbol> FilteredContracts { get; set; }
+
         /// <summary>
         /// Gets the data list
         /// </summary>
@@ -68,8 +78,9 @@ namespace QuantConnect.Data.UniverseSelection
         /// <param name="endTime">The end time of this data</param>
         /// <param name="symbol">A common identifier for all data in this packet</param>
         /// <param name="data">The data to add to this collection</param>
-        public BaseDataCollection(DateTime time, DateTime endTime, Symbol symbol, IEnumerable<BaseData> data = null)
-            : this(time, endTime, symbol, data != null ? data.ToList() : new List<BaseData>())
+        /// <param name="underlying">The associated underlying price data if any</param>
+        public BaseDataCollection(DateTime time, DateTime endTime, Symbol symbol, IEnumerable<BaseData> data = null, BaseData underlying = null)
+            : this(time, endTime, symbol, data != null ? data.ToList() : new List<BaseData>(), underlying, null)
         {
         }
 
@@ -80,11 +91,15 @@ namespace QuantConnect.Data.UniverseSelection
         /// <param name="endTime">The end time of this data</param>
         /// <param name="symbol">A common identifier for all data in this packet</param>
         /// <param name="data">The data to add to this collection</param>
-        public BaseDataCollection(DateTime time, DateTime endTime, Symbol symbol, List<BaseData> data)
+        /// <param name="underlying">The associated underlying price data if any</param>
+        /// <param name="filteredContracts">The contracts selected by the universe</param>
+        public BaseDataCollection(DateTime time, DateTime endTime, Symbol symbol, List<BaseData> data, BaseData underlying, IReadOnlyCollection<Symbol> filteredContracts)
         {
             Symbol = symbol;
             Time = time;
             _endTime = endTime;
+            Underlying = underlying;
+            FilteredContracts = filteredContracts;
             Data = data ?? new List<BaseData>();
         }
 
@@ -115,7 +130,7 @@ namespace QuantConnect.Data.UniverseSelection
         /// <returns>A clone of the current object</returns>
         public override BaseData Clone()
         {
-            return new BaseDataCollection(Time, EndTime, Symbol, Data);
+            return new BaseDataCollection(Time, EndTime, Symbol, Data, Underlying, FilteredContracts);
         }
 
         /// <summary>

Common/Data/UniverseSelection/FuturesChainUniverse.cs

@@ -66,12 +66,6 @@ namespace QuantConnect.Data.UniverseSelection
         /// <returns>The data that passes the filter</returns>
         public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
         {
-            var futuresUniverseDataCollection = data as FuturesChainUniverseDataCollection;
-            if (futuresUniverseDataCollection == null)
-            {
-                throw new ArgumentException($"Expected data of type '{typeof(FuturesChainUniverseDataCollection).Name}'");
-            }
-
             var underlying = new Tick { Time = utcTime };
 
             // date change detection needs to be done in exchange time zone
@@ -80,7 +74,7 @@ namespace QuantConnect.Data.UniverseSelection
                 return Unchanged;
             }
 
-            var availableContracts = futuresUniverseDataCollection.Data.Select(x => x.Symbol);
+            var availableContracts = data.Data.Select(x => x.Symbol);
             var results = Future.ContractFilter.Filter(new FutureFilterUniverse(availableContracts, underlying));
 
             // if results are not dynamic, we cache them and won't call filtering till the end of the day
@@ -89,11 +83,7 @@ namespace QuantConnect.Data.UniverseSelection
                 _cacheDate = data.Time.ConvertFromUtc(Future.Exchange.TimeZone).Date;
             }
 
-            var resultingSymbols = results.ToHashSet();
-
-            futuresUniverseDataCollection.FilteredContracts = resultingSymbols;
-
-            return resultingSymbols;
+            return results;
         }
 
         /// <summary>

Common/Data/UniverseSelection/FuturesChainUniverseDataCollection.cs

@@ -1,84 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- * 
- * Licensed under the Apache License, Version 2.0 (the "License"); 
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
-*/
-
-using System;
-using System.Collections.Generic;
-
-namespace QuantConnect.Data.UniverseSelection
-{
-    /// <summary>
-    /// Defines the universe selection data type for <see cref="FuturesChainUniverse"/>
-    /// </summary>
-    public class FuturesChainUniverseDataCollection : BaseDataCollection
-    {
-        /// <summary>
-        /// Gets or sets the contracts selected by the universe
-        /// </summary>
-        public HashSet<Symbol> FilteredContracts { get; set; }
-        
-        /// <summary>
-        /// Initializes a new default instance of the <see cref="FuturesChainUniverseDataCollection"/> c;ass
-        /// </summary>
-        public FuturesChainUniverseDataCollection()
-            : this(DateTime.MinValue, Symbol.Empty)
-        {
-            FilteredContracts = new HashSet<Symbol>();
-        }
-
-        /// <summary>
-        /// Initializes a new instance of the <see cref="FuturesChainUniverseDataCollection"/> class
-        /// </summary>
-        /// <param name="time">The time of this data</param>
-        /// <param name="symbol">A common identifier for all data in this packet</param>
-        /// <param name="data">The data to add to this collection</param>
-        public FuturesChainUniverseDataCollection(DateTime time, Symbol symbol, List<BaseData> data = null)
-            : this(time, time, symbol, data)
-        {
-        }
-
-        /// <summary>
-        /// Initializes a new instance of the <see cref="FuturesChainUniverseDataCollection"/> class
-        /// </summary>
-        /// <param name="time">The start time of this data</param>
-        /// <param name="endTime">The end time of this data</param>
-        /// <param name="symbol">A common identifier for all data in this packet</param>
-        /// <param name="data">The data to add to this collection</param>
-        public FuturesChainUniverseDataCollection(DateTime time, DateTime endTime, Symbol symbol, List<BaseData> data = null)
-            : base(time, endTime, symbol, data)
-        {
-        }
-
-        /// <summary>
-        /// Return a new instance clone of this object, used in fill forward
-        /// </summary>
-        /// <remarks>
-        /// This base implementation uses reflection to copy all public fields and properties
-        /// </remarks>
-        /// <returns>A clone of the current object</returns>
-        public override BaseData Clone()
-        {
-            return new FuturesChainUniverseDataCollection
-            {
-                Symbol = Symbol,
-                Time = Time,
-                EndTime = EndTime,
-                Data = Data,
-                DataType = DataType,
-                FilteredContracts = FilteredContracts
-            };
-        }
-    }
-}

Common/Data/UniverseSelection/OptionChainUniverse.cs

@@ -79,12 +79,6 @@ namespace QuantConnect.Data.UniverseSelection
         /// <returns>The data that passes the filter</returns>
         public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
         {
-            var optionsUniverseDataCollection = data as OptionChainUniverseDataCollection;
-            if (optionsUniverseDataCollection == null)
-            {
-                throw new ArgumentException($"Expected data of type '{typeof(OptionChainUniverseDataCollection).Name}'");
-            }
-
             // date change detection needs to be done in exchange time zone
             var exchangeDate = data.Time.ConvertFromUtc(Option.Exchange.TimeZone).Date;
             if (_cacheDate == exchangeDate)
@@ -92,9 +86,9 @@ namespace QuantConnect.Data.UniverseSelection
                 return Unchanged;
             }
 
-            var availableContracts = optionsUniverseDataCollection.Data.Select(x => x.Symbol);
+            var availableContracts = data.Data.Select(x => x.Symbol);
             // we will only update unique strikes when there is an exchange date change
-            _optionFilterUniverse.Refresh(availableContracts, optionsUniverseDataCollection.Underlying, _lastExchangeDate != exchangeDate);
+            _optionFilterUniverse.Refresh(availableContracts, data.Underlying, _lastExchangeDate != exchangeDate);
             _lastExchangeDate = exchangeDate;
 
             var results = Option.ContractFilter.Filter(_optionFilterUniverse);
@@ -106,15 +100,7 @@ namespace QuantConnect.Data.UniverseSelection
             }
 
             // always prepend the underlying symbol
-            var resultingSymbols = _underlyingSymbol.Concat(results).ToHashSet();
-
-            // we save off the filtered results to the universe data collection for later
-            // population into the OptionChain. This is non-ideal and could be remedied by
-            // the universe subscription emitting a special type after selection that could
-            // be checked for in TimeSlice.Create, but for now this will do
-            optionsUniverseDataCollection.FilteredContracts = resultingSymbols;
-
-            return resultingSymbols;
+            return _underlyingSymbol.Concat(results);
         }
 
         /// <summary>

Common/Data/UniverseSelection/OptionChainUniverseDataCollection.cs

@@ -1,93 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- * 
- * Licensed under the Apache License, Version 2.0 (the "License"); 
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
-*/
-
-using System;
-using System.Collections.Generic;
-
-namespace QuantConnect.Data.UniverseSelection
-{
-    /// <summary>
-    /// Defines the universe selection data type for <see cref="OptionChainUniverse"/>
-    /// </summary>
-    public class OptionChainUniverseDataCollection : BaseDataCollection
-    {
-        /// <summary>
-        /// The option chain's underlying price data
-        /// </summary>
-        public BaseData Underlying { get; set; }
-
-        /// <summary>
-        /// Gets or sets the contracts selected by the universe
-        /// </summary>
-        public HashSet<Symbol> FilteredContracts { get; set; }
-        
-        /// <summary>
-        /// Initializes a new default instance of the <see cref="OptionChainUniverseDataCollection"/> c;ass
-        /// </summary>
-        public OptionChainUniverseDataCollection()
-            : this(DateTime.MinValue, Symbol.Empty)
-        {
-            FilteredContracts = new HashSet<Symbol>();
-        }
-
-        /// <summary>
-        /// Initializes a new instance of the <see cref="OptionChainUniverseDataCollection"/> class
-        /// </summary>
-        /// <param name="time">The time of this data</param>
-        /// <param name="symbol">A common identifier for all data in this packet</param>
-        /// <param name="data">The data to add to this collection</param>
-        /// <param name="underlying">The option chain's underlying price data</param>
-        public OptionChainUniverseDataCollection(DateTime time, Symbol symbol, List<BaseData> data = null, BaseData underlying = null)
-            : this(time, time, symbol, data, underlying)
-        {
-        }
-
-        /// <summary>
-        /// Initializes a new instance of the <see cref="OptionChainUniverseDataCollection"/> class
-        /// </summary>
-        /// <param name="time">The start time of this data</param>
-        /// <param name="endTime">The end time of this data</param>
-        /// <param name="symbol">A common identifier for all data in this packet</param>
-        /// <param name="data">The data to add to this collection</param>
-        /// <param name="underlying">The option chain's underlying price data</param>
-        public OptionChainUniverseDataCollection(DateTime time, DateTime endTime, Symbol symbol, List<BaseData> data = null, BaseData underlying = null)
-            : base(time, endTime, symbol, data)
-        {
-            Underlying = underlying;
-        }
-
-        /// <summary>
-        /// Return a new instance clone of this object, used in fill forward
-        /// </summary>
-        /// <remarks>
-        /// This base implementation uses reflection to copy all public fields and properties
-        /// </remarks>
-        /// <returns>A clone of the current object</returns>
-        public override BaseData Clone()
-        {
-            return new OptionChainUniverseDataCollection
-            {
-                Underlying = Underlying,
-                Symbol = Symbol,
-                Time = Time,
-                EndTime = EndTime,
-                Data = Data,
-                DataType = DataType,
-                FilteredContracts = FilteredContracts
-            };
-        }
-    }
-}

Common/Data/UniverseSelection/Universe.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -179,10 +179,12 @@ namespace QuantConnect.Data.UniverseSelection
             var result = SelectSymbols(utcTime, data);
             if (ReferenceEquals(result, Unchanged))
             {
+                data.FilteredContracts = _previousSelections;
                 return Unchanged;
             }
 
             var selections = result.ToHashSet();
+            data.FilteredContracts = selections;
             var hasDiffs = _previousSelections.AreDifferent(selections);
             _previousSelections = selections;
             if (!hasDiffs)

Engine/DataFeeds/BacktestingCacheProvider.cs

@@ -0,0 +1,104 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Util;
+using QuantConnect.Logging;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds
+{
+    /// <summary>
+    /// Base backtesting cache provider which will source symbols from local zip files
+    /// </summary>
+    public abstract class BacktestingCacheProvider
+    {
+        // see https://github.com/QuantConnect/Lean/issues/6384
+        private static readonly TickType[] DataTypes = new[] { TickType.Quote, TickType.OpenInterest, TickType.Trade };
+        private bool _loggedPreviousTradableDate;
+
+        /// <summary>
+        /// The data cache instance to use
+        /// </summary>
+        protected IDataCacheProvider DataCacheProvider { get; }
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        protected BacktestingCacheProvider(IDataCacheProvider dataCacheProvider)
+        {
+            DataCacheProvider = dataCacheProvider;
+        }
+
+        /// <summary>
+        /// Get the contract symbols associated with the given canonical symbol and date
+        /// </summary>
+        /// <param name="canonicalSymbol">The canonical symbol</param>
+        /// <param name="date">The date to search for</param>
+        protected IEnumerable<Symbol> GetSymbols(Symbol canonicalSymbol, DateTime date)
+        {
+            IEnumerable<string> entries = null;
+            foreach (var tickType in DataTypes)
+            {
+                // build the zip file name and fetch it with our provider
+                var zipFileName = LeanData.GenerateZipFilePath(Globals.DataFolder, canonicalSymbol, date, Resolution.Minute, tickType);
+                try
+                {
+                    entries = DataCacheProvider.GetZipEntries(zipFileName);
+                }
+                catch
+                {
+                }
+
+                if (entries != null)
+                {
+                    break;
+                }
+            }
+
+            if (entries == null)
+            {
+                var mhdb = MarketHoursDatabase.FromDataFolder();
+                if (mhdb.TryGetEntry(canonicalSymbol.ID.Market, canonicalSymbol, canonicalSymbol.SecurityType, out var entry) && !entry.ExchangeHours.IsDateOpen(date))
+                {
+                    if (!_loggedPreviousTradableDate)
+                    {
+                        _loggedPreviousTradableDate = true;
+                        Log.Error($"BacktestingCacheProvider.GetSymbols(): {date} is not a tradable date for {canonicalSymbol}. When requesting contracts " +
+                            $" for non tradable dates, will return contracts of previous tradable date.");
+                    }
+
+                    // be user friendly, will return contracts from the previous tradable date
+                    foreach (var symbols in GetSymbols(canonicalSymbol, Time.GetStartTimeForTradeBars(entry.ExchangeHours, date, Time.OneDay, 1, false, entry.DataTimeZone)))
+                    {
+                        yield return symbols;
+                    }
+                    yield break;
+                }
+
+                Log.Trace($"BacktestingCacheProvider.GetSymbols(): found no source of contracts for {canonicalSymbol} for date {date.ToString(DateFormat.EightCharacter)} for any tick type");
+                yield break;
+            }
+
+            // generate and return the contract symbol for each zip entry
+            foreach (var zipEntryName in entries)
+            {
+                yield return LeanData.ReadSymbolFromZipEntry(canonicalSymbol, Resolution.Minute, zipEntryName);
+            }
+        }
+    }
+}

Engine/DataFeeds/BacktestingFutureChainProvider.cs

@@ -14,27 +14,23 @@
 */
 
 using System;
-using System.Collections.Generic;
 using QuantConnect.Interfaces;
-using QuantConnect.Logging;
-using QuantConnect.Util;
+using System.Collections.Generic;
 
 namespace QuantConnect.Lean.Engine.DataFeeds
 {
     /// <summary>
     /// An implementation of <see cref="IFutureChainProvider"/> that reads the list of contracts from open interest zip data files
     /// </summary>
-    public class BacktestingFutureChainProvider : IFutureChainProvider
+    public class BacktestingFutureChainProvider : BacktestingCacheProvider, IFutureChainProvider
     {
-        private IDataProvider _dataProvider;
-
         /// <summary>
         /// Creates a new instance
         /// </summary>
-        /// <param name="dataProvider">The data provider instance to use</param>
-        public BacktestingFutureChainProvider(IDataProvider dataProvider)
+        /// <param name="dataCacheProvider">The data cache provider instance to use</param>
+        public BacktestingFutureChainProvider(IDataCacheProvider dataCacheProvider)
+            : base(dataCacheProvider)
         {
-            _dataProvider = dataProvider;
         }
 
         /// <summary>
@@ -43,40 +39,31 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         /// <param name="symbol">The underlying symbol</param>
         /// <param name="date">The date for which to request the future chain (only used in backtesting)</param>
         /// <returns>The list of future contracts</returns>
-        public IEnumerable<Symbol> GetFutureContractList(Symbol symbol, DateTime date)
+        public virtual IEnumerable<Symbol> GetFutureContractList(Symbol symbol, DateTime date)
+        {
+            return GetSymbols(GetSymbol(symbol), date);
+        }
+
+        /// <summary>
+        /// Helper method to get the symbol to use
+        /// </summary>
+        protected static Symbol GetSymbol(Symbol symbol)
         {
             if (symbol.SecurityType != SecurityType.Future)
             {
-                throw new NotSupportedException($"BacktestingFutureChainProvider.GetFutureContractList(): SecurityType.Future is expected but was {symbol.SecurityType}");
-            }
-
-            // build the future contract list from the open interest zip file entry names
-
-            // build the zip file name for open interest data
-            var zipFileName = LeanData.GenerateZipFilePath(Globals.DataFolder, symbol, date, Resolution.Minute, TickType.OpenInterest);
-            var stream = _dataProvider.Fetch(zipFileName);
-
-            // If the file isn't found lets give quote a chance - some futures do not have an open interest file
-            if (stream == null)
-            {
-                var zipFileNameQuote = LeanData.GenerateZipFilePath(Globals.DataFolder, symbol, date, Resolution.Minute, TickType.Quote);
-                stream = _dataProvider.Fetch(zipFileNameQuote);
-
-                if (stream == null) 
+                if (symbol.SecurityType == SecurityType.FutureOption && symbol.Underlying != null)
                 {
-                    Log.Error($"BacktestingFutureChainProvider.GetFutureContractList(): Failed, files not found: {zipFileName} {zipFileNameQuote}");
-                    yield break;
+                    // be user friendly and take the underlying
+                    symbol = symbol.Underlying;
+                }
+                else
+                {
+                    throw new NotSupportedException($"BacktestingFutureChainProvider.GetFutureContractList():" +
+                        $" {nameof(SecurityType.Future)} or {nameof(SecurityType.FutureOption)} is expected but was {symbol.SecurityType}");
                 }
             }
 
-            // generate and return the contract symbol for each zip entry
-            var zipEntryNames = Compression.GetZipEntryFileNames(stream);
-            foreach (var zipEntryName in zipEntryNames)
-            {
-                yield return LeanData.ReadSymbolFromZipEntry(symbol, Resolution.Minute, zipEntryName);
-            }
-
-            stream.DisposeSafely();
+            return symbol.Canonical;
         }
     }
 }

Engine/DataFeeds/BacktestingOptionChainProvider.cs

@@ -14,67 +14,67 @@
 */
 
 using System;
-using System.Collections.Generic;
-using System.IO;
-using QuantConnect.Configuration;
-using QuantConnect.Data.Auxiliary;
 using QuantConnect.Interfaces;
-using QuantConnect.Logging;
-using QuantConnect.Util;
+using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
 
 namespace QuantConnect.Lean.Engine.DataFeeds
 {
     /// <summary>
     /// An implementation of <see cref="IOptionChainProvider"/> that reads the list of contracts from open interest zip data files
     /// </summary>
-    public class BacktestingOptionChainProvider : IOptionChainProvider
+    public class BacktestingOptionChainProvider : BacktestingCacheProvider, IOptionChainProvider
     {
-        private IDataProvider _dataProvider;
         private IMapFileProvider _mapFileProvider;
 
         /// <summary>
         /// Creates a new instance
         /// </summary>
-        /// <param name="dataProvider">The data provider instance to use</param>
-        public BacktestingOptionChainProvider(IDataProvider dataProvider)
+        /// <param name="dataCacheProvider">The data cache provider instance to use</param>
+        /// <param name="mapFileProvider">The map file provider instance to use</param>
+        public BacktestingOptionChainProvider(IDataCacheProvider dataCacheProvider, IMapFileProvider mapFileProvider)
+            : base(dataCacheProvider)
         {
-            _dataProvider = dataProvider;
-            _mapFileProvider =
-                Composer.Instance.GetExportedValueByTypeName<IMapFileProvider>(Config.Get("map-file-provider",
-                    "LocalDiskMapFileProvider"));
+            _mapFileProvider = mapFileProvider;
         }
 
         /// <summary>
         /// Gets the list of option contracts for a given underlying symbol
         /// </summary>
-        /// <param name="underlyingSymbol">The underlying symbol</param>
+        /// <param name="symbol">The underlying symbol</param>
         /// <param name="date">The date for which to request the option chain (only used in backtesting)</param>
         /// <returns>The list of option contracts</returns>
-        public IEnumerable<Symbol> GetOptionContractList(Symbol underlyingSymbol, DateTime date)
+        public virtual IEnumerable<Symbol> GetOptionContractList(Symbol symbol, DateTime date)
         {
-            if (!underlyingSymbol.SecurityType.HasOptions())
+            if (!symbol.SecurityType.HasOptions())
             {
-                throw new NotSupportedException($"BacktestingOptionChainProvider.GetOptionContractList(): SecurityType.Equity, SecurityType.Future, or SecurityType.Index is expected but was {underlyingSymbol.SecurityType}");
+                if (symbol.SecurityType.IsOption() && symbol.Underlying != null)
+                {
+                    // be user friendly and take the underlying
+                    symbol = symbol.Underlying;
+                }
+                else
+                {
+                    throw new NotSupportedException($"BacktestingOptionChainProvider.GetOptionContractList(): " +
+                        $"{nameof(SecurityType.Equity)}, {nameof(SecurityType.Future)}, or {nameof(SecurityType.Index)} is expected but was {symbol.SecurityType}");
+                }
             }
 
             // Resolve any mapping before requesting option contract list for equities
             // Needs to be done in order for the data file key to be accurate
             Symbol mappedSymbol;
-            if (underlyingSymbol.RequiresMapping())
+            if (symbol.RequiresMapping())
             {
-                var mapFileResolver = _mapFileProvider.Get(AuxiliaryDataKey.Create(underlyingSymbol));
-                var mapFile = mapFileResolver.ResolveMapFile(underlyingSymbol);
-                var ticker = mapFile.GetMappedSymbol(date, underlyingSymbol.Value);
-                mappedSymbol = underlyingSymbol.UpdateMappedSymbol(ticker);
+                var mapFileResolver = _mapFileProvider.Get(AuxiliaryDataKey.Create(symbol));
+                var mapFile = mapFileResolver.ResolveMapFile(symbol);
+                var ticker = mapFile.GetMappedSymbol(date, symbol.Value);
+                mappedSymbol = symbol.UpdateMappedSymbol(ticker);
             }
             else
             {
-                mappedSymbol = underlyingSymbol;
+                mappedSymbol = symbol;
             }
 
-
-            // build the option contract list from the open interest zip file entry names
-
             // create a canonical option symbol for the given underlying
             var canonicalSymbol = Symbol.CreateOption(
                 mappedSymbol,
@@ -84,39 +84,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                 0,
                 SecurityIdentifier.DefaultDate);
 
-            var zipFileName = string.Empty;
-            Stream stream = null;
-
-            // In order of trust-worthiness of containing the complete option chain, OpenInterest is guaranteed
-            // to have the complete option chain. Quotes come after open-interest
-            // because it's also likely to contain the option chain. Trades may be
-            // missing portions of the option chain, so we resort to it last.
-            foreach (var tickType in new[] { TickType.OpenInterest, TickType.Quote, TickType.Trade })
-            {
-                // build the zip file name and fetch it with our provider
-                zipFileName = LeanData.GenerateZipFilePath(Globals.DataFolder, canonicalSymbol, date, Resolution.Minute, tickType);
-                stream = _dataProvider.Fetch(zipFileName);
-
-                if (stream != null)
-                {
-                    break;
-                }
-            }
-
-            if (stream == null)
-            {
-                Log.Trace($"BacktestingOptionChainProvider.GetOptionContractList(): File not found: {zipFileName}");
-                yield break;
-            }
-
-            // generate and return the contract symbol for each zip entry
-            var zipEntryNames = Compression.GetZipEntryFileNames(stream);
-            foreach (var zipEntryName in zipEntryNames)
-            {
-                yield return LeanData.ReadSymbolFromZipEntry(canonicalSymbol, Resolution.Minute, zipEntryName);
-            }
-
-            stream.DisposeSafely();
+            return GetSymbols(canonicalSymbol, date);
         }
     }
 }

Engine/DataFeeds/Enumerators/BaseDataCollectionAggregatorEnumerator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -26,8 +26,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
     /// that aggregates an underlying <see cref="IEnumerator{BaseData}"/> into a single
     /// data packet
     /// </summary>
-    public class BaseDataCollectionAggregatorEnumerator<T> : IEnumerator<T>
-        where T : BaseDataCollection, new()
+    public class BaseDataCollectionAggregatorEnumerator : IEnumerator<BaseDataCollection>
     {
         private bool _endOfStream;
         private bool _needsMoveNext;
@@ -65,7 +64,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
                 return false;
             }
 
-            T collection = null;
+            BaseDataCollection collection = null;
             while (true)
             {
                 if (_needsMoveNext)
@@ -135,7 +134,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// <returns>
         /// The element in the collection at the current position of the enumerator.
         /// </returns>
-        public T Current
+        public BaseDataCollection Current
         {
             get; private set;
         }
@@ -168,9 +167,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// <param name="time">The start time of the collection</param>
         /// <param name="endTime">The end time of the collection</param>
         /// <returns>A new, empty <see cref="BaseDataCollection"/></returns>
-        protected virtual T CreateCollection(Symbol symbol, DateTime time, DateTime endTime)
+        private BaseDataCollection CreateCollection(Symbol symbol, DateTime time, DateTime endTime)
         {
-            return new T
+            return new BaseDataCollection
             {
                 Symbol = symbol,
                 Time = time,
@@ -183,19 +182,39 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// </summary>
         /// <param name="collection">The collection to be added to</param>
         /// <param name="current">The data to be added</param>
-        protected virtual void Add(T collection, BaseData current)
+        private void Add(BaseDataCollection collection, BaseData current)
         {
-            collection.Add(current);
-        }
-
-        /// <summary>
-        /// Adds all specified instances of <see cref="BaseData"/> to the current collection
-        /// </summary>
-        /// <param name="collection">The collection to be added to</param>
-        /// <param name="current">The data collection to be added</param>
-        protected virtual void SetData(T collection, List<BaseData> current)
-        {
-            collection.Data = current;
+            if (_symbol.HasUnderlying && _symbol.Underlying == current.Symbol)
+            {
+                var baseDataCollection = current as BaseDataCollection;
+                if (baseDataCollection != null)
+                {
+                    collection.Underlying = baseDataCollection.Data[0];
+                }
+                else
+                {
+                    collection.Underlying = current;
+                }
+            }
+            else
+            {
+                var baseDataCollection = current as BaseDataCollection;
+                if (baseDataCollection != null)
+                {
+                    if(baseDataCollection.Data.Count > 1)
+                    {
+                        collection.Data = baseDataCollection.Data;
+                    }
+                    else
+                    {
+                        collection.Data.Add(baseDataCollection.Data[0]);
+                    }
+                }
+                else
+                {
+                    collection.Data.Add(current);
+                }
+            }
         }
 
         /// <summary>
@@ -203,26 +222,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// </summary>
         /// <param name="collection">The collection to be emitted</param>
         /// <returns>True if its a valid data point</returns>
-        protected virtual bool IsValid(T collection)
-        {
-            return true;
-        }
-    }
-
-    /// <summary>
-    /// Provides a non-generic implementation of <see cref="BaseDataCollectionAggregatorEnumerator{T}"/>
-    /// </summary>
-    public class BaseDataCollectionAggregatorEnumerator : BaseDataCollectionAggregatorEnumerator<BaseDataCollection>
-    {
-        /// <summary>
-        /// Initializes a new instance of the <see cref="BaseDataCollectionAggregatorEnumerator"/> class
-        /// </summary>
-        /// <param name="enumerator">The enumerator to aggregate</param>
-        /// <param name="symbol">The output data's symbol</param>
-        /// <param name="liveMode">True if running in live mode</param>
-        public BaseDataCollectionAggregatorEnumerator(IEnumerator<BaseData> enumerator, Symbol symbol, bool liveMode = false)
-            : base(enumerator, symbol, liveMode)
+        private bool IsValid(BaseDataCollection collection)
         {
+            return collection != null && collection.Data?.Count > 0;
         }
     }
 }

Engine/DataFeeds/Enumerators/ConcatEnumerator.cs

@@ -38,6 +38,14 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// The current BaseData object
         /// </summary>
         public BaseData Current { get; set; }
+
+        /// <summary>
+        /// True if emitting a null data point is expected
+        /// </summary>
+        /// <remarks>Warmup enumerators are not allowed to return true setting current to Null, this is because it's not a valid behavior for backtesting enumerators,
+        /// for example <see cref="FillForwardEnumerator"/></remarks>
+        public bool CanEmitNull { get; set; }
+
         object IEnumerator.Current => Current;
 
         /// <summary>
@@ -50,8 +58,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
             params IEnumerator<BaseData>[] enumerators
             )
         {
-            _enumerators = enumerators.Where(enumerator => enumerator != null).ToList();
+            CanEmitNull = true;
             _skipDuplicateEndTimes = skipDuplicateEndTimes;
+            _enumerators = enumerators.Where(enumerator => enumerator != null).ToList();
         }
 
         /// <summary>
@@ -65,11 +74,12 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
                 var enumerator = _enumerators[_currentIndex];
                 while (enumerator.MoveNext())
                 {
-                    if (enumerator.Current == null && _currentIndex < _enumerators.Count - 1)
+                    if (enumerator.Current == null && (_currentIndex < _enumerators.Count - 1 || !CanEmitNull))
                     {
                         // if there are more enumerators and the current stopped providing data drop it
                         // in live trading, some enumerators will always return true (see TimeTriggeredUniverseSubscriptionEnumeratorFactory & InjectionEnumerator)
                         // but unless it's the last enumerator we drop it, because these first are the warmup enumerators
+                        // or we are not allowed to return null
                         break;
                     }
 

Engine/DataFeeds/Enumerators/DataQueueFuturesChainUniverseDataCollectionEnumerator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -22,13 +22,14 @@ using System.Collections;
 using System.Linq;
 using QuantConnect.Data.Auxiliary;
 using QuantConnect.Interfaces;
+using QuantConnect.Logging;
 
 namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
 {
     /// <summary>
-    /// Enumerates live futures symbol universe data into <see cref="FuturesChainUniverseDataCollection"/> instances
+    /// Enumerates live futures symbol universe data into <see cref="BaseDataCollection"/> instances
     /// </summary>
-    public class DataQueueFuturesChainUniverseDataCollectionEnumerator : IEnumerator<FuturesChainUniverseDataCollection>
+    public class DataQueueFuturesChainUniverseDataCollectionEnumerator : IEnumerator<BaseDataCollection>
     {
         private readonly SubscriptionRequest _subscriptionRequest;
         private readonly IDataQueueUniverseProvider _universeProvider;
@@ -58,7 +59,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// <summary>
         /// Returns current futures chain enumerator position
         /// </summary>
-        public FuturesChainUniverseDataCollection Current { get; private set; }
+        public BaseDataCollection Current { get; private set; }
 
         /// <summary>
         /// Returns current futures chain enumerator position
@@ -101,7 +102,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
                 // loading the list of futures contracts and converting them into zip entries
                 var symbols = _universeProvider.LookupSymbols(_subscriptionRequest.Security.Symbol, false);
                 var zipEntries = symbols.Select(x => new ZipEntryName { Time = localTime, Symbol = x } as BaseData).ToList();
-                var current = new FuturesChainUniverseDataCollection
+                var current = new BaseDataCollection
                 {
                     Symbol = _subscriptionRequest.Security.Symbol,
                     Data = zipEntries,
@@ -111,6 +112,8 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
 
                 _lastEmitTime = localTime;
 
+                Log.Trace($"DataQueueFuturesChainUniverseDataCollectionEnumerator({current.Symbol}): Emitting data point: {current.EndTime}. Count: {current.Data.Count}");
+
                 Current = current;
                 _needNewCurrent = false;
             }

Engine/DataFeeds/Enumerators/DataQueueOptionChainUniverseDataCollectionEnumerator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -22,20 +22,21 @@ using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Interfaces;
 using QuantConnect.Data.Auxiliary;
 using System.Collections;
+using QuantConnect.Logging;
 
 namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
 {
     /// <summary>
     /// Enumerates live options symbol universe data into <see cref="OptionChainUniverseDataCollection"/> instances
     /// </summary>
-    public class DataQueueOptionChainUniverseDataCollectionEnumerator : IEnumerator<OptionChainUniverseDataCollection>
+    public class DataQueueOptionChainUniverseDataCollectionEnumerator : IEnumerator<BaseDataCollection>
     {
         private readonly SubscriptionRequest _subscriptionRequest;
         private readonly IDataQueueUniverseProvider _universeProvider;
         private readonly ITimeProvider _timeProvider;
         private bool _needNewCurrent;
         private DateTime _lastEmitTime;
-        private OptionChainUniverseDataCollection _currentData;
+        private BaseDataCollection _currentData;
 
         /// <summary>
         /// Gets the enumerator for the underlying asset
@@ -66,7 +67,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// <summary>
         /// Returns current option chain enumerator position
         /// </summary>
-        public OptionChainUniverseDataCollection Current { get; private set; }
+        public BaseDataCollection Current { get; private set; }
 
         /// <summary>
         /// Returns current option chain enumerator position
@@ -118,7 +119,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
                 // loading the list of futures contracts and converting them into zip entries
                 var symbols = _universeProvider.LookupSymbols(_subscriptionRequest.Security.Symbol, false);
                 var zipEntries = symbols.Select(x => new ZipEntryName { Time = localTime, Symbol = x } as BaseData).ToList();
-                _currentData = new OptionChainUniverseDataCollection
+                _currentData = new BaseDataCollection
                 {
                     Symbol = _subscriptionRequest.Security.Symbol,
                     Underlying = Underlying.Current,
@@ -127,6 +128,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
                     EndTime = localTime
                 };
 
+                Log.Trace($"DataQueueOptionChainUniverseDataCollectionEnumerator({_currentData.Symbol}): Emitting data point: {_currentData.EndTime}. " +
+                    $"Count: {_currentData.Data.Count}. Underlying: {_currentData.Underlying}");
+
                 _lastEmitTime = localTime;
 
                 Current = _currentData;

Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactory.cs

@@ -15,11 +15,11 @@
 */
 
 using System;
-using System.Collections.Generic;
 using QuantConnect.Data;
-using QuantConnect.Data.UniverseSelection;
-using QuantConnect.Data.Auxiliary;
 using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
+using QuantConnect.Data.UniverseSelection;
 
 namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
 {
@@ -29,32 +29,19 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
     /// </summary>
     public class BaseDataSubscriptionEnumeratorFactory : ISubscriptionEnumeratorFactory
     {
+        private readonly IOptionChainProvider _optionChainProvider;
+        private readonly IFutureChainProvider _futureChainProvider;
         private readonly Func<SubscriptionRequest, IEnumerable<DateTime>> _tradableDaysProvider;
-        private readonly IMapFileProvider _mapFileProvider;
-        private readonly IDataCacheProvider _dataCacheProvider;
-        private readonly bool _isLiveMode;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="BaseDataSubscriptionEnumeratorFactory"/> class
         /// </summary>
-        /// <param name="isLiveMode">True for live mode, false otherwise</param>
-        /// <param name="mapFileProvider">Used for resolving the correct map files</param>
-        /// <param name="dataCacheProvider">The cache provider instance to use</param>
-        public BaseDataSubscriptionEnumeratorFactory(bool isLiveMode, IMapFileProvider mapFileProvider, IDataCacheProvider dataCacheProvider)
-            : this(isLiveMode, dataCacheProvider)
+        /// <param name="optionChainProvider">The option chain provider</param>
+        /// <param name="futureChainProvider">The future chain provider</param>
+        public BaseDataSubscriptionEnumeratorFactory(IOptionChainProvider optionChainProvider, IFutureChainProvider futureChainProvider)
         {
-            _mapFileProvider = mapFileProvider;
-        }
-
-        /// <summary>
-        /// Initializes a new instance of the <see cref="BaseDataSubscriptionEnumeratorFactory"/> class
-        /// </summary>
-        /// <param name="isLiveMode">True for live mode, false otherwise</param>
-        /// <param name="dataCacheProvider">The cache provider instance to use</param>
-        public BaseDataSubscriptionEnumeratorFactory(bool isLiveMode, IDataCacheProvider dataCacheProvider)
-        {
-            _isLiveMode = isLiveMode;
-            _dataCacheProvider = dataCacheProvider;
+            _futureChainProvider = futureChainProvider;
+            _optionChainProvider = optionChainProvider;
             _tradableDaysProvider = (request => request.TradableDays);
         }
 
@@ -73,24 +60,25 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
             var sourceFactory = request.Configuration.GetBaseDataInstance();
             foreach (var date in _tradableDaysProvider(request))
             {
-                if (sourceFactory.RequiresMapping() && _mapFileProvider != null)
+                IEnumerable<Symbol> symbols;
+                if (request.Configuration.SecurityType.IsOption())
                 {
-                    request.Configuration.MappedSymbol = GetMappedSymbol(request.Configuration, date);
+                    symbols = _optionChainProvider.GetOptionContractList(request.Configuration.Symbol.Underlying, date);
+                }
+                else if (request.Configuration.SecurityType == SecurityType.Future)
+                {
+                    symbols = _futureChainProvider.GetFutureContractList(request.Configuration.Symbol, date);
+                }
+                else
+                {
+                    throw new NotImplementedException($"{request.Configuration.SecurityType} is not supported");
                 }
 
-                var source = sourceFactory.GetSource(request.Configuration, date, _isLiveMode);
-                var factory = SubscriptionDataSourceReader.ForSource(source, _dataCacheProvider, request.Configuration, date, _isLiveMode, sourceFactory, dataProvider);
-                var entriesForDate = factory.Read(source);
-                foreach (var entry in entriesForDate)
+                foreach (var symbol in symbols)
                 {
-                    yield return entry;
+                    yield return new ZipEntryName { Symbol = symbol, Time = date };
                 }
             }
         }
-
-        private string GetMappedSymbol(SubscriptionDataConfig config, DateTime date)
-        {
-            return _mapFileProvider.ResolveMapFile(config).GetMappedSymbol(date, config.MappedSymbol);
-        }
     }
 }

Engine/DataFeeds/Enumerators/Factories/FuturesChainUniverseSubscriptionEnumeratorFactory.cs

@@ -1,96 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
-*/
-
-using System;
-using System.Collections.Generic;
-using QuantConnect.Data;
-using QuantConnect.Data.UniverseSelection;
-using QuantConnect.Interfaces;
-
-namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
-{
-    /// <summary>
-    /// Provides an implementation of <see cref="ISubscriptionEnumeratorFactory"/> for the <see cref="FuturesChainUniverse"/> in backtesting
-    /// </summary>
-    public class FuturesChainUniverseSubscriptionEnumeratorFactory : ISubscriptionEnumeratorFactory
-    {
-        private readonly Func<SubscriptionRequest, IEnumerator<BaseData>, IEnumerator<BaseData>> _enumeratorConfigurator;
-        private readonly bool _isLiveMode;
-
-        private readonly IDataQueueUniverseProvider _symbolUniverse;
-        private readonly IDataCacheProvider _dataCacheProvider;
-        private readonly ITimeProvider _timeProvider;
-
-        /// <summary>
-        /// Initializes a new instance of the <see cref="FuturesChainUniverseSubscriptionEnumeratorFactory"/> class
-        /// </summary>
-        /// <param name="enumeratorConfigurator">Function used to configure the sub-enumerators before sync (fill-forward/filter/ect...)</param>
-        /// <param name="dataCacheProvider">The cache provider instance to use</param>
-        public FuturesChainUniverseSubscriptionEnumeratorFactory(Func<SubscriptionRequest, IEnumerator<BaseData>, IEnumerator<BaseData>> enumeratorConfigurator, IDataCacheProvider dataCacheProvider)
-        {
-            _isLiveMode = false;
-            _dataCacheProvider = dataCacheProvider;
-            _enumeratorConfigurator = enumeratorConfigurator;
-        }
-
-        /// <summary>
-        /// Initializes a new instance of the <see cref="FuturesChainUniverseSubscriptionEnumeratorFactory"/> class
-        /// </summary>
-        /// <param name="symbolUniverse">Symbol universe provider of the data queue</param>
-        /// <param name="timeProvider">The time provider to be used</param>
-        public FuturesChainUniverseSubscriptionEnumeratorFactory(IDataQueueUniverseProvider symbolUniverse, ITimeProvider timeProvider)
-        {
-            _isLiveMode = true;
-            _symbolUniverse = symbolUniverse;
-            _timeProvider = timeProvider;
-            _enumeratorConfigurator = (sr, input) => input;
-        }
-
-        /// <summary>
-        /// Creates an enumerator to read the specified request
-        /// </summary>
-        /// <param name="request">The subscription request to be read</param>
-        /// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
-        /// <returns>An enumerator reading the subscription request</returns>
-        public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
-        {
-            if (_isLiveMode)
-            {
-                var subscriptionRequest = new SubscriptionRequest(request, configuration: GetSubscriptionConfiguration(request));
-
-                return new DataQueueFuturesChainUniverseDataCollectionEnumerator(subscriptionRequest, _symbolUniverse, _timeProvider);
-            }
-            else
-            {
-                var factory = new BaseDataSubscriptionEnumeratorFactory(_isLiveMode, _dataCacheProvider);
-
-                var newRequest = new SubscriptionRequest(request, configuration: GetSubscriptionConfiguration(request));
-                var enumerator = _enumeratorConfigurator(request, factory.CreateEnumerator(newRequest, dataProvider));
-
-                return new FuturesChainUniverseDataCollectionAggregatorEnumerator(enumerator, request.Security.Symbol);
-            }
-        }
-
-        private static SubscriptionDataConfig GetSubscriptionConfiguration(SubscriptionRequest request)
-        {
-            var config = request.Configuration;
-            var resolution = config.Resolution;
-
-            // rewrite the primary to be fill forward
-            return new SubscriptionDataConfig(config, resolution: resolution, fillForward: true);
-        }
-    }
-}

Engine/DataFeeds/Enumerators/Factories/OptionChainUniverseSubscriptionEnumeratorFactory.cs

@@ -35,17 +35,6 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
         private readonly IDataQueueUniverseProvider _symbolUniverse;
         private readonly ITimeProvider _timeProvider;
 
-        /// <summary>
-        /// Initializes a new instance of the <see cref="OptionChainUniverseSubscriptionEnumeratorFactory"/> class
-        /// </summary>
-        /// <param name="enumeratorConfigurator">Function used to configure the sub-enumerators before sync (fill-forward/filter/ect...)</param>
-        public OptionChainUniverseSubscriptionEnumeratorFactory(Func<SubscriptionRequest, IEnumerator<BaseData>> enumeratorConfigurator)
-        {
-            _isLiveMode = false;
-            _enumeratorConfigurator = enumeratorConfigurator;
-
-        }
-
         /// <summary>
         /// Initializes a new instance of the <see cref="OptionChainUniverseSubscriptionEnumeratorFactory"/> class
         /// </summary>
@@ -80,12 +69,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
             }
             else
             {
-                var enumerators = GetSubscriptionConfigurations(request)
-                    .Select(c => new SubscriptionRequest(request, configuration: c))
-                    .Select(sr => _enumeratorConfigurator(sr));
-
-                var sync = new SynchronizingBaseDataEnumerator(enumerators);
-                return new OptionChainUniverseDataCollectionEnumerator(sync, request.Security.Symbol);
+                throw new InvalidOperationException($"Backtesting is expected to be using {nameof(BaseDataSubscriptionEnumeratorFactory)}");
             }
         }
 

Engine/DataFeeds/Enumerators/FuturesChainUniverseDataCollectionAggregatorEnumerator.cs

@@ -1,68 +0,0 @@
-
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- * 
- * Licensed under the Apache License, Version 2.0 (the "License"); 
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System.Collections.Generic;
-using QuantConnect.Data;
-using QuantConnect.Data.Auxiliary;
-using QuantConnect.Data.UniverseSelection;
-
-namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
-{
-    /// <summary>
-    /// Aggregates an enumerator into <see cref="FuturesChainUniverseDataCollection"/> instances
-    /// </summary>
-    public class FuturesChainUniverseDataCollectionAggregatorEnumerator : BaseDataCollectionAggregatorEnumerator<FuturesChainUniverseDataCollection>
-    {
-        /// <summary>
-        /// Initializes a new instance of the <see cref="FuturesChainUniverseDataCollectionAggregatorEnumerator"/> class
-        /// </summary>
-        /// <param name="enumerator">The enumerator to aggregate</param>
-        /// <param name="symbol">The output data's symbol</param>
-        public FuturesChainUniverseDataCollectionAggregatorEnumerator(IEnumerator<BaseData> enumerator, Symbol symbol)
-            : base(enumerator, symbol)
-        {
-        }
-
-        /// <summary>
-        /// Adds the specified instance of <see cref="BaseData"/> to the current collection
-        /// </summary>
-        /// <param name="collection">The collection to be added to</param>
-        /// <param name="current">The data to be added</param>
-        protected override void Add(FuturesChainUniverseDataCollection collection, BaseData current)
-        {
-            AddSingleItem(collection, current);
-        }
-
-        private static void AddSingleItem(FuturesChainUniverseDataCollection collection, BaseData current)
-        {
-            var baseDataCollection = current as BaseDataCollection;
-            if (baseDataCollection != null)
-            {
-                foreach (var data in baseDataCollection.Data)
-                {
-                    AddSingleItem(collection, data);
-                }
-                return;
-            }
-
-            if (current is ZipEntryName)
-            {
-                collection.Add(current);
-                return;
-            }
-        }
-    }
-}

Engine/DataFeeds/Enumerators/OptionChainUniverseDataCollectionEnumerator.cs

@@ -1,67 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System.Collections.Generic;
-using QuantConnect.Data;
-using QuantConnect.Data.UniverseSelection;
-using QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories;
-
-namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
-{
-    /// <summary>
-    /// Enumerates data into <see cref="OptionChainUniverseDataCollection"/> instances
-    /// </summary>
-    /// <remarks>Used in backtest mode <see cref="OptionChainUniverseSubscriptionEnumeratorFactory"/></remarks>
-    public class OptionChainUniverseDataCollectionEnumerator : BaseDataCollectionAggregatorEnumerator<OptionChainUniverseDataCollection>
-    {
-        /// <summary>
-        /// Initializes a new instance of the <see cref="OptionChainUniverseDataCollectionEnumerator"/> class
-        /// </summary>
-        /// <param name="enumerator">The enumerator to aggregate</param>
-        /// <param name="symbol">The output data's symbol</param>
-        public OptionChainUniverseDataCollectionEnumerator(IEnumerator<BaseData> enumerator, Symbol symbol)
-            : base(enumerator, symbol)
-        {
-        }
-
-        /// <summary>
-        /// Adds the specified instance of <see cref="BaseData"/> to the current collection
-        /// </summary>
-        /// <param name="collection">The collection to be added to</param>
-        /// <param name="current">The data to be added</param>
-        protected override void Add(OptionChainUniverseDataCollection collection, BaseData current)
-        {
-            var baseDataCollection = current as BaseDataCollection;
-            if (baseDataCollection != null)
-            {
-                if (baseDataCollection.Data.Count > 1)
-                    SetData(collection, baseDataCollection.Data);
-
-                else if (baseDataCollection.Data.Count == 1)
-                    collection.Underlying = baseDataCollection.Data[0];
-            }
-        }
-
-        /// <summary>
-        /// Determines if a given data point is valid and can be emitted
-        /// </summary>
-        /// <param name="collection">The collection to be emitted</param>
-        /// <returns>True if its a valid data point</returns>
-        protected override bool IsValid(OptionChainUniverseDataCollection collection)
-        {
-            return collection?.Underlying != null && collection.Data.Count != 0;
-        }
-    }
-}

Engine/DataFeeds/FileSystemDataFeed.cs

@@ -14,10 +14,12 @@
  *
 */
 
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using QuantConnect.Data;
+using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Market;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Interfaces;
@@ -91,7 +93,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         /// <remarks>Protected so it can be used by the <see cref="LiveTradingDataFeed"/> to warmup requests</remarks>
         protected IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request)
         {
-            return request.IsUniverseSubscription ? CreateUniverseEnumerator(request) : CreateDataEnumerator(request);
+            return request.IsUniverseSubscription ? CreateUniverseEnumerator(request, CreateDataEnumerator) : CreateDataEnumerator(request);
         }
 
         private IEnumerator<BaseData> CreateDataEnumerator(SubscriptionRequest request)
@@ -138,7 +140,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         {
         }
 
-        private IEnumerator<BaseData> CreateUniverseEnumerator(SubscriptionRequest request)
+        protected IEnumerator<BaseData> CreateUniverseEnumerator(SubscriptionRequest request, Func<SubscriptionRequest, IEnumerator<BaseData>> createUnderlyingEnumerator)
         {
             ISubscriptionEnumeratorFactory factory = _subscriptionFactory;
             if (request.Universe is ITimeTriggeredUniverse)
@@ -152,27 +154,22 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                     return factory.CreateEnumerator(request, _dataProvider);
                 }
             }
-            if (request.Configuration.Type == typeof(CoarseFundamental))
+            else if (request.Configuration.Type == typeof(CoarseFundamental))
             {
                 factory = new BaseDataCollectionSubscriptionEnumeratorFactory();
             }
-            if (request.Universe is OptionChainUniverse)
+            else if (request.Configuration.Type == typeof(ZipEntryName))
             {
-                factory = new OptionChainUniverseSubscriptionEnumeratorFactory((req) =>
-                {
-                    if (!req.Configuration.SecurityType.IsOption())
-                    {
-                        var enumerator = _subscriptionFactory.CreateEnumerator(req, _dataProvider);
-                        enumerator = new FilterEnumerator<BaseData>(enumerator, data => data.DataType != MarketDataType.Auxiliary);
-                        return ConfigureEnumerator(req, true, enumerator);
-                    }
-                    var underlyingFactory = new BaseDataSubscriptionEnumeratorFactory(false, _mapFileProvider, _cacheProvider);
-                    return ConfigureEnumerator(req, true, underlyingFactory.CreateEnumerator(req, _dataProvider));
-                });
-            }
-            if (request.Universe is FuturesChainUniverse)
-            {
-                factory = new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e), _cacheProvider);
+                // TODO: subscription should already come in correctly built
+                var resolution = request.Configuration.Resolution == Resolution.Tick ? Resolution.Second : request.Configuration.Resolution;
+
+                // TODO: subscription should already come in as fill forward true
+                request = new SubscriptionRequest(request, configuration: new SubscriptionDataConfig(request.Configuration, fillForward: true, resolution: resolution));
+
+                var result = new BaseDataSubscriptionEnumeratorFactory(_algorithm.OptionChainProvider, _algorithm.FutureChainProvider)
+                    .CreateEnumerator(request, _dataProvider);
+                result = ConfigureEnumerator(request, true, result);
+                return TryAppendUnderlyingEnumerator(request, result, createUnderlyingEnumerator);
             }
 
             // define our data enumerator
@@ -180,6 +177,32 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             return enumerator;
         }
 
+        /// <summary>
+        /// If required will add a new enumerator for the underlying symbol
+        /// </summary>
+        protected IEnumerator<BaseData> TryAppendUnderlyingEnumerator(SubscriptionRequest request, IEnumerator<BaseData> parent, Func<SubscriptionRequest, IEnumerator<BaseData>> createEnumerator)
+        {
+            if (request.Configuration.Symbol.SecurityType.IsOption() && request.Configuration.Symbol.HasUnderlying)
+            {
+                // TODO: creating this subscription request/config is bad
+                var underlyingRequests = new SubscriptionRequest(request,
+                    isUniverseSubscription: false,
+                    configuration: new SubscriptionDataConfig(request.Configuration, symbol: request.Configuration.Symbol.Underlying, objectType: typeof(TradeBar), tickType: TickType.Trade));
+
+                var underlying = createEnumerator(underlyingRequests);
+                underlying = new FilterEnumerator<BaseData>(underlying, data => data.DataType != MarketDataType.Auxiliary);
+
+                parent = new SynchronizingBaseDataEnumerator(parent, underlying);
+                // we aggregate both underlying and chain data
+                parent = new BaseDataCollectionAggregatorEnumerator(parent, request.Configuration.Symbol);
+                // only let through if underlying and chain data present
+                parent = new FilterEnumerator<BaseData>(parent, data => (data as BaseDataCollection).Underlying != null);
+                parent = ConfigureEnumerator(request, false, parent);
+            }
+
+            return parent;
+        }
+
         /// <summary>
         /// Send an exit signal to the thread.
         /// </summary>
@@ -199,15 +222,32 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         /// <summary>
         /// Configure the enumerator with aggregation/fill-forward/filter behaviors. Returns new instance if re-configured
         /// </summary>
-        private IEnumerator<BaseData> ConfigureEnumerator(SubscriptionRequest request, bool aggregate, IEnumerator<BaseData> enumerator)
+        protected IEnumerator<BaseData> ConfigureEnumerator(SubscriptionRequest request, bool aggregate, IEnumerator<BaseData> enumerator)
         {
             if (aggregate)
             {
                 enumerator = new BaseDataCollectionAggregatorEnumerator(enumerator, request.Configuration.Symbol);
             }
 
+            enumerator = TryAddFillForwardEnumerator(request, enumerator, request.Configuration.FillDataForward);
+
+            // optionally apply exchange/user filters
+            if (request.Configuration.IsFilteredSubscription)
+            {
+                enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, request.Security,
+                    request.EndTimeLocal, request.Configuration.ExtendedMarketHours, false, request.ExchangeHours);
+            }
+
+            return enumerator;
+        }
+
+        /// <summary>
+        /// Will add a fill forward enumerator if requested
+        /// </summary>
+        protected IEnumerator<BaseData> TryAddFillForwardEnumerator(SubscriptionRequest request, IEnumerator<BaseData> enumerator, bool fillForward)
+        {
             // optionally apply fill forward logic, but never for tick data
-            if (request.Configuration.FillDataForward && request.Configuration.Resolution != Resolution.Tick)
+            if (fillForward && request.Configuration.Resolution != Resolution.Tick)
             {
                 // copy forward Bid/Ask bars for QuoteBars
                 if (request.Configuration.Type == typeof(QuoteBar))
@@ -221,13 +261,6 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                     request.Configuration.ExtendedMarketHours, request.EndTimeLocal, request.Configuration.Resolution.ToTimeSpan(), request.Configuration.DataTimeZone);
             }
 
-            // optionally apply exchange/user filters
-            if (request.Configuration.IsFilteredSubscription)
-            {
-                enumerator = SubscriptionFilterEnumerator.WrapForDataFeed(_resultHandler, enumerator, request.Security,
-                    request.EndTimeLocal, request.Configuration.ExtendedMarketHours, false, request.ExchangeHours);
-            }
-
             return enumerator;
         }
     }

Engine/DataFeeds/InternalSubscriptionManager.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -72,7 +72,14 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                     // low resolution subscriptions we will add internal Resolution.Minute subscriptions
                     // if we don't already have this symbol added
                     var config = new SubscriptionDataConfig(request.Configuration, resolution: _resolution, isInternalFeed: true, extendedHours: true, isFilteredSubscription: false);
-                    var internalRequest = new SubscriptionRequest(false, null, request.Security, config, request.StartTimeUtc, request.EndTimeUtc);
+                    var startTimeUtc = request.StartTimeUtc;
+                    if (_algorithm.IsWarmingUp)
+                    {
+                        // during warmup in live trading do not add these internal subscription until the algorithm starts
+                        // these subscription are only added for realtime price in low resolution subscriptions which isn't required for warmup
+                        startTimeUtc = DateTime.UtcNow;
+                    }
+                    var internalRequest = new SubscriptionRequest(false, null, request.Security, config, startTimeUtc, request.EndTimeUtc);
                     if (existing)
                     {
                         _subscriptionRequests[request.Configuration.Symbol].Add(internalRequest);

Engine/DataFeeds/LiveFutureChainProvider.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -14,8 +14,9 @@
 */
 
 using System;
-using System.Collections.Generic;
+using System.Linq;
 using QuantConnect.Interfaces;
+using System.Collections.Generic;
 
 namespace QuantConnect.Lean.Engine.DataFeeds
 {
@@ -23,17 +24,44 @@ namespace QuantConnect.Lean.Engine.DataFeeds
     /// An implementation of <see cref="IFutureChainProvider"/> that fetches the list of contracts
     /// from an external source
     /// </summary>
-    public class LiveFutureChainProvider : IFutureChainProvider
+    public class LiveFutureChainProvider : BacktestingFutureChainProvider
     {
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="dataCacheProvider">The data cache provider instance to use</param>
+        public LiveFutureChainProvider(IDataCacheProvider dataCacheProvider) : base(dataCacheProvider)
+        {
+        }
+
         /// <summary>
         /// Gets the list of future contracts for a given underlying symbol
         /// </summary>
         /// <param name="symbol">The underlying symbol</param>
         /// <param name="date">The date for which to request the future chain (only used in backtesting)</param>
         /// <returns>The list of future contracts</returns>
-        public IEnumerable<Symbol> GetFutureContractList(Symbol symbol, DateTime date)
+        public override IEnumerable<Symbol> GetFutureContractList(Symbol symbol, DateTime date)
         {
-            throw new NotImplementedException("LiveFutureChainProvider.GetFutureContractList() has not been implemented yet.");
+            var result = Enumerable.Empty<Symbol>();
+            try
+            {
+                result = base.GetFutureContractList(symbol, date);
+            }
+            catch
+            {
+            }
+
+            bool yielded = false;
+            foreach (var symbols in result)
+            {
+                yielded = true;
+                yield return symbols;
+            }
+
+            if (!yielded)
+            {
+                throw new NotImplementedException("LiveFutureChainProvider.GetFutureContractList() has not been implemented yet.");
+            }
         }
     }
-}
+}

Engine/DataFeeds/LiveOptionChainProvider.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -34,7 +34,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
     /// An implementation of <see cref="IOptionChainProvider"/> that fetches the list of contracts
     /// from the Options Clearing Corporation (OCC) website
     /// </summary>
-    public class LiveOptionChainProvider : IOptionChainProvider
+    public class LiveOptionChainProvider : BacktestingOptionChainProvider
     {
         private static readonly HttpClient _client;
         private static readonly DateTime _epoch = new DateTime(1970, 1, 1);
@@ -66,6 +66,16 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             _client.DefaultRequestHeaders.Connection.Add("keep-alive");
         }
 
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="dataCacheProvider">The data cache provider instance to use</param>
+        /// <param name="mapFileProvider">The map file provider instance to use</param>
+        public LiveOptionChainProvider(IDataCacheProvider dataCacheProvider, IMapFileProvider mapFileProvider)
+            : base(dataCacheProvider, mapFileProvider)
+        {
+        }
+
         /// <summary>
         /// Gets the option chain associated with the underlying Symbol
         /// </summary>
@@ -73,20 +83,47 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         /// <param name="date">Unused</param>
         /// <returns>Option chain</returns>
         /// <exception cref="ArgumentException">Option underlying Symbol is not Future or Equity</exception>
-        public IEnumerable<Symbol> GetOptionContractList(Symbol underlyingSymbol, DateTime date)
+        public override IEnumerable<Symbol> GetOptionContractList(Symbol underlyingSymbol, DateTime date)
         {
-            if (underlyingSymbol.SecurityType == SecurityType.Equity || underlyingSymbol.SecurityType == SecurityType.Index)
+            var result = Enumerable.Empty<Symbol>();
+            try
             {
-                // Source data from TheOCC if we're trading equity or index options
-                return GetEquityOptionContractList(underlyingSymbol, date);
+                result = base.GetOptionContractList(underlyingSymbol, date);
             }
-            if (underlyingSymbol.SecurityType == SecurityType.Future)
+            catch
             {
-                // We get our data from CME if we're trading future options
-                return GetFutureOptionContractList(underlyingSymbol, date);
             }
 
-            throw new ArgumentException("Option Underlying SecurityType is not supported. Supported types are: Equity, Index, Future");
+            bool yielded = false;
+            foreach (var symbol in result)
+            {
+                yielded = true;
+                yield return symbol;
+            }
+
+            if (!yielded)
+            {
+                if (underlyingSymbol.SecurityType == SecurityType.Equity || underlyingSymbol.SecurityType == SecurityType.Index)
+                {
+                    // Source data from TheOCC if we're trading equity or index options
+                    foreach (var symbol in GetEquityOptionContractList(underlyingSymbol, date))
+                    {
+                        yield return symbol;
+                    }
+                }
+                else if (underlyingSymbol.SecurityType == SecurityType.Future)
+                {
+                    // We get our data from CME if we're trading future options
+                    foreach (var symbol in GetFutureOptionContractList(underlyingSymbol, date))
+                    {
+                        yield return symbol;
+                    }
+                }
+                else
+                {
+                    throw new ArgumentException("Option Underlying SecurityType is not supported. Supported types are: Equity, Index, Future");
+                }
+            }
         }
 
         private IEnumerable<Symbol> GetFutureOptionContractList(Symbol futureContractSymbol, DateTime date)
@@ -274,52 +311,49 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         {
             var symbols = new List<Symbol>();
 
-            using (var client = new WebClient())
+            // use QC url to bypass TLS issues with Mono pre-4.8 version
+            var url = "https://www.quantconnect.com/api/v2/theocc/series-search?symbolType=U&symbol=" + underlyingSymbol.Value;
+
+            // download the text file
+            var fileContent = url.DownloadData();
+
+            // read the lines, skipping the headers
+            var lines = fileContent.Split(new[] { "\r\n" }, StringSplitOptions.None).Skip(7);
+
+            // Example of a line:
+            // SPY		2021	03	26	190	000	C P 	0	612	360000000
+
+            // parse the lines, creating the Lean option symbols
+            foreach (var line in lines)
             {
-                // use QC url to bypass TLS issues with Mono pre-4.8 version
-                var url = "https://www.quantconnect.com/api/v2/theocc/series-search?symbolType=U&symbol=" + underlyingSymbol.Value;
+                var fields = line.Split('\t');
 
-                // download the text file
-                var fileContent = client.DownloadString(url);
+                var ticker = fields[0].Trim();
+                if (ticker != underlyingSymbol.Value)
+                    continue;
 
-                // read the lines, skipping the headers
-                var lines = fileContent.Split(new[] { "\r\n" }, StringSplitOptions.None).Skip(7);
+                var expiryDate = new DateTime(fields[2].ToInt32(), fields[3].ToInt32(), fields[4].ToInt32());
+                var strike = (fields[5] + "." + fields[6]).ToDecimal();
 
-                // Example of a line:
-                // SPY		2021	03	26	190	000	C P 	0	612	360000000
+                Action<OptionRight> addSymbol = right =>
+                    symbols.Add(Symbol.CreateOption(
+                        underlyingSymbol,
+                        underlyingSymbol.ID.Market,
+                        underlyingSymbol.SecurityType.DefaultOptionStyle(),
+                        right,
+                        strike,
+                        expiryDate));
 
-                // parse the lines, creating the Lean option symbols
-                foreach (var line in lines)
+                foreach (var right in fields[7].Trim().Split(' '))
                 {
-                    var fields = line.Split('\t');
-
-                    var ticker = fields[0].Trim();
-                    if (ticker != underlyingSymbol.Value)
-                        continue;
-
-                    var expiryDate = new DateTime(fields[2].ToInt32(), fields[3].ToInt32(), fields[4].ToInt32());
-                    var strike = (fields[5] + "." + fields[6]).ToDecimal();
-
-                    Action<OptionRight> addSymbol = right =>
-                        symbols.Add(Symbol.CreateOption(
-                            underlyingSymbol,
-                            underlyingSymbol.ID.Market,
-                            underlyingSymbol.SecurityType.DefaultOptionStyle(),
-                            right,
-                            strike,
-                            expiryDate));
-
-                    foreach (var right in fields[7].Trim().Split(' '))
+                    if (right.Contains("C"))
                     {
-                        if (right.Contains("C"))
-                        {
-                            addSymbol(OptionRight.Call);
-                        }
+                        addSymbol(OptionRight.Call);
+                    }
 
-                        if (right.Contains("P"))
-                        {
-                            addSymbol(OptionRight.Put);
-                        }
+                    if (right.Contains("P"))
+                    {
+                        addSymbol(OptionRight.Put);
                     }
                 }
             }

Engine/DataFeeds/LiveTradingDataFeed.cs

@@ -39,7 +39,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
     /// </summary>
     public class LiveTradingDataFeed : FileSystemDataFeed
     {
-        private static readonly int MaximumWarmupHistoryDaysLookBack = Config.GetInt("maximum-warmup-history-days-look-back", 7);
+        private static readonly int MaximumWarmupHistoryDaysLookBack = Config.GetInt("maximum-warmup-history-days-look-back", 5);
 
         private LiveNodePacket _job;
 
@@ -211,7 +211,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                         {
                             enqueable.Enqueue(data);
 
-                            subscription.OnNewDataAvailable();
+                            subscription?.OnNewDataAvailable();
                         });
 
                         enumerator = enqueable;
@@ -259,14 +259,14 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                         enumerator = new LiveFillForwardEnumerator(_frontierTimeProvider, enumerator, request.Security.Exchange, fillForwardResolution, request.Configuration.ExtendedMarketHours, localEndTime, request.Configuration.Increment, request.Configuration.DataTimeZone);
                     }
 
-                    // define market hours and user filters to incoming data
+                    // make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
+                    enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
+
+                    // define market hours and user filters to incoming data after the frontier enumerator so during warmup we avoid any realtime data making it's way into the securities
                     if (request.Configuration.IsFilteredSubscription)
                     {
                         enumerator = new SubscriptionFilterEnumerator(enumerator, request.Security, localEndTime, request.Configuration.ExtendedMarketHours, true, request.ExchangeHours);
                     }
-
-                    // finally, make our subscriptions aware of the frontier of the data feed, prevents future data from spewing into the feed
-                    enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, timeZoneOffsetProvider);
                 }
                 else
                 {
@@ -354,7 +354,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                 _customExchange.AddEnumerator(config.Symbol, aggregator, handleData: data =>
                 {
                     enqueable.Enqueue(data);
-                    subscription.OnNewDataAvailable();
+                    subscription?.OnNewDataAvailable();
                 });
 
                 enumerator = GetConfiguredFrontierAwareEnumerator(enqueable, tzOffsetProvider,
@@ -368,7 +368,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                 Func<SubscriptionRequest, IEnumerator<BaseData>> configure = (subRequest) =>
                 {
                     var fillForwardResolution = _subscriptions.UpdateAndGetFillForwardResolution(subRequest.Configuration);
-                    var input = Subscribe(subRequest.Configuration, (sender, args) => subscription.OnNewDataAvailable());
+                    var input = Subscribe(subRequest.Configuration, (sender, args) => subscription?.OnNewDataAvailable());
                     return new LiveFillForwardEnumerator(_frontierTimeProvider, input, subRequest.Security.Exchange, fillForwardResolution, subRequest.Configuration.ExtendedMarketHours, localEndTime, subRequest.Configuration.Increment, subRequest.Configuration.DataTimeZone);
                 };
 
@@ -383,11 +383,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             {
                 Log.Trace("LiveTradingDataFeed.CreateUniverseSubscription(): Creating futures chain universe: " + config.Symbol.ID);
 
-                var symbolUniverse = GetUniverseProvider(SecurityType.Option);
-
-                var enumeratorFactory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, _timeProvider);
-                enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
+                var symbolUniverse = GetUniverseProvider(SecurityType.Future);
 
+                enumerator = new DataQueueFuturesChainUniverseDataCollectionEnumerator(request, symbolUniverse, _timeProvider);
                 enumerator = new FrontierAwareEnumerator(enumerator, _frontierTimeProvider, tzOffsetProvider);
             }
             else
@@ -412,7 +410,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             // send the subscription for the new symbol through to the data queuehandler
             if (_channelProvider.ShouldStreamSubscription(subscription.Configuration))
             {
-                Subscribe(request.Configuration, (sender, args) => subscription.OnNewDataAvailable());
+                Subscribe(request.Configuration, (sender, args) => subscription?.OnNewDataAvailable());
             }
 
             return subscription;
@@ -425,21 +423,28 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         {
             if (_algorithm.IsWarmingUp)
             {
-                var warmup = new SubscriptionRequest(request, endTimeUtc: _timeProvider.GetUtcNow());
-                if (warmup.TradableDays.Any())
+                var warmupRequest = new SubscriptionRequest(request, endTimeUtc: _timeProvider.GetUtcNow(),
+                    // we will not fill forward each warmup enumerators separately but concatenated bellow
+                    configuration: new SubscriptionDataConfig(request.Configuration, fillForward: false));
+                if (warmupRequest.TradableDays.Any())
                 {
                     // since we will source data locally and from the history provider, let's limit the history request size
                     // by setting a start date respecting the 'MaximumWarmupHistoryDaysLookBack'
-                    var historyWarmup = warmup;
-                    var warmupHistoryStartDate = warmup.EndTimeUtc.AddDays(-MaximumWarmupHistoryDaysLookBack);
-                    if (warmupHistoryStartDate > warmup.StartTimeUtc)
+                    var historyWarmup = warmupRequest;
+                    var warmupHistoryStartDate = warmupRequest.EndTimeUtc.AddDays(-MaximumWarmupHistoryDaysLookBack);
+                    if (warmupHistoryStartDate > warmupRequest.StartTimeUtc)
                     {
-                        historyWarmup = new SubscriptionRequest(warmup, startTimeUtc: warmupHistoryStartDate);
+                        historyWarmup = new SubscriptionRequest(warmupRequest, startTimeUtc: warmupHistoryStartDate);
                     }
 
+                    var synchronizedWarmupEnumerator = TryAddFillForwardEnumerator(warmupRequest,
+                        // we concatenate the file based and history based warmup enumerators, dropping duplicate time stamps
+                        new ConcatEnumerator(true, GetFileBasedWarmupEnumerator(warmupRequest), GetHistoryWarmupEnumerator(historyWarmup)) { CanEmitNull = false },
+                        // if required by the original request, we will fill forward the Synced warmup data
+                        request.Configuration.FillDataForward);
+
                     // the order here is important, concat enumerator will keep the last enumerator given and dispose of the rest
-                    liveEnumerator = new ConcatEnumerator(true, GetFileBasedWarmupEnumerator(warmup),
-                        GetHistoryWarmupEnumerator(historyWarmup), liveEnumerator);
+                    liveEnumerator = new ConcatEnumerator(true, synchronizedWarmupEnumerator, liveEnumerator);
                 }
             }
             return liveEnumerator;
@@ -454,7 +459,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             try
             {
                 result = new FilterEnumerator<BaseData>(CreateEnumerator(warmup),
-                    // don't let future data past, nor fill forward, that will be handled by the history request
+                    // don't let future data past, nor fill forward, that will be handled after merging with the history request response
                     data => data == null || data.EndTime < warmup.EndTimeLocal && !data.IsFillForward);
             }
             catch (Exception e)
@@ -472,22 +477,31 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             IEnumerator<BaseData> result = null;
             try
             {
-                var historyRequest = new Data.HistoryRequest(warmup.Configuration, warmup.ExchangeHours, warmup.StartTimeUtc, warmup.EndTimeUtc);
-                result = _algorithm.HistoryProvider.GetHistory(new[] { historyRequest }, _algorithm.TimeZone).Select(slice =>
+                if (warmup.IsUniverseSubscription)
                 {
-                    try
+                    result = CreateUniverseEnumerator(warmup, createUnderlyingEnumerator: GetHistoryWarmupEnumerator);
+                }
+                else
+                {
+                    var historyRequest = new Data.HistoryRequest(warmup.Configuration, warmup.ExchangeHours, warmup.StartTimeUtc, warmup.EndTimeUtc);
+                    result = _algorithm.HistoryProvider.GetHistory(new[] { historyRequest }, _algorithm.TimeZone).SelectMany(slice =>
                     {
-                        var data = slice.Get(historyRequest.DataType);
-                        return (BaseData)data[warmup.Configuration.Symbol];
-                    }
-                    catch (Exception e)
-                    {
-                        Log.Error(e, $"History warmup: {warmup.Configuration}");
-                    }
-                    return null;
-                }).GetEnumerator();
+                        try
+                        {
+                            var data = slice.Get(historyRequest.DataType);
+                            return (IEnumerable<BaseData>)data.Values;
+                        }
+                        catch (Exception e)
+                        {
+                            Log.Error(e, $"History warmup: {warmup.Configuration}");
+                        }
+                        return null;
+                    }).GetEnumerator();
+                }
 
-                result = new FilterEnumerator<BaseData>(result, data => data == null || data.EndTime < warmup.EndTimeLocal);
+                return new FilterEnumerator<BaseData>(result,
+                    // don't let future data past, nor fill forward, that will be handled after merging with the file based enumerator
+                    data => data == null || data.EndTime < warmup.EndTimeLocal && !data.IsFillForward);
             }
             catch
             {

Engine/DataFeeds/Queues/FakeDataQueue.cs

@@ -23,6 +23,7 @@ using QuantConnect.Securities;
 using QuantConnect.Interfaces;
 using QuantConnect.Data.Market;
 using System.Collections.Generic;
+using QuantConnect.Configuration;
 using Timer = System.Timers.Timer;
 using QuantConnect.Lean.Engine.HistoricalData;
 
@@ -37,6 +38,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Queues
         private readonly Random _random = new Random();
 
         private readonly Timer _timer;
+        private readonly IDataCacheProvider _dataCacheProvider;
         private readonly IOptionChainProvider _optionChainProvider;
         private readonly EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager;
         private readonly IDataAggregator _aggregator;
@@ -63,8 +65,10 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Queues
         public FakeDataQueue(IDataAggregator dataAggregator)
         {
             _aggregator = dataAggregator;
-            _optionChainProvider = new LiveOptionChainProvider();
             _marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
+            _dataCacheProvider = new ZipDataCacheProvider(new DefaultDataProvider(), true);
+            var mapFileProvider = Composer.Instance.GetExportedValueByTypeName<IMapFileProvider>(Config.Get("map-file-provider", "LocalDiskMapFileProvider"), false);
+            _optionChainProvider = new LiveOptionChainProvider(_dataCacheProvider, mapFileProvider);
             _symbolExchangeTimeZones = new Dictionary<Symbol, TimeZoneOffsetProvider>();
             _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
             _subscriptionManager.SubscribeImpl += (s, t) => true;
@@ -149,6 +153,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Queues
         {
             _timer.Stop();
             _timer.DisposeSafely();
+            _dataCacheProvider.DisposeSafely();
         }
 
         /// <summary>

Engine/DataFeeds/SubscriptionSynchronizer.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -180,20 +180,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                                 }
                                 else
                                 {
-                                    if (packetBaseDataCollection is OptionChainUniverseDataCollection)
-                                    {
-                                        var current = packetBaseDataCollection as OptionChainUniverseDataCollection;
-                                        collection = new OptionChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData, current?.Underlying);
-                                    }
-                                    else if (packetBaseDataCollection is FuturesChainUniverseDataCollection)
-                                    {
-                                        collection = new FuturesChainUniverseDataCollection(frontierUtc, subscription.Configuration.Symbol, packetData);
-                                    }
-                                    else
-                                    {
-                                        collection = new BaseDataCollection(frontierUtc, frontierUtc, subscription.Configuration.Symbol, packetData);
-                                    }
-
+                                    collection = new BaseDataCollection(frontierUtc, frontierUtc, subscription.Configuration.Symbol, packetData, packetBaseDataCollection?.Underlying);
                                     if (universeData == null)
                                     {
                                         universeData = new Dictionary<Universe, BaseDataCollection>();

Engine/DataFeeds/TimeSliceFactory.cs

@@ -430,7 +430,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                 chain.Underlying = underlyingData;
             }
 
-            var universeData = baseData as OptionChainUniverseDataCollection;
+            var universeData = baseData as BaseDataCollection;
             if (universeData != null)
             {
                 if (universeData.Underlying != null)
@@ -501,7 +501,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                 futuresChains[canonical] = chain;
             }
 
-            var universeData = baseData as FuturesChainUniverseDataCollection;
+            var universeData = baseData as BaseDataCollection;
             if (universeData != null)
             {
                 foreach (var contractSymbol in universeData.FilteredContracts)

Engine/Engine.cs

@@ -15,7 +15,6 @@
 */
 
 using System;
-using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Runtime.CompilerServices;
@@ -103,7 +102,6 @@ namespace QuantConnect.Lean.Engine
 
                 IBrokerage brokerage = null;
                 DataManager dataManager = null;
-                IDataCacheProvider historyDataCacheProvider = null;
                 var synchronizer = _liveMode ? new LiveSynchronizer() : new Synchronizer();
                 try
                 {
@@ -187,13 +185,12 @@ namespace QuantConnect.Lean.Engine
                     // set the history provider before setting up the algorithm
                     var historyProvider = GetHistoryProvider();
                     historyProvider.SetBrokerage(brokerage);
-                    historyDataCacheProvider = new ZipDataCacheProvider(AlgorithmHandlers.DataProvider, isDataEphemeral:_liveMode);
                     historyProvider.Initialize(
                         new HistoryProviderInitializeParameters(
                             job,
                             SystemHandlers.Api,
                             AlgorithmHandlers.DataProvider,
-                            historyDataCacheProvider,
+                            AlgorithmHandlers.DataCacheProvider,
                             AlgorithmHandlers.MapFileProvider,
                             AlgorithmHandlers.FactorFileProvider,
                             progress =>
@@ -222,7 +219,7 @@ namespace QuantConnect.Lean.Engine
 
                     //Initialize the internal state of algorithm and job: executes the algorithm.Initialize() method.
                     initializeComplete = AlgorithmHandlers.Setup.Setup(new SetupHandlerParameters(dataManager.UniverseSelection, algorithm, brokerage, job, AlgorithmHandlers.Results,
-                        AlgorithmHandlers.Transactions, AlgorithmHandlers.RealTime, AlgorithmHandlers.ObjectStore, AlgorithmHandlers.DataProvider));
+                        AlgorithmHandlers.Transactions, AlgorithmHandlers.RealTime, AlgorithmHandlers.ObjectStore, AlgorithmHandlers.DataCacheProvider, AlgorithmHandlers.MapFileProvider));
 
                     // set this again now that we've actually added securities
                     AlgorithmHandlers.Results.SetAlgorithm(algorithm, AlgorithmHandlers.Setup.StartingPortfolioValue);
@@ -433,7 +430,6 @@ namespace QuantConnect.Lean.Engine
                     AlgorithmHandlers.Setup.Dispose();
                 }
 
-                historyDataCacheProvider.DisposeSafely();
                 Log.Trace("Engine.Main(): Analysis Completed and Results Posted.");
             }
             catch (Exception err)

Engine/HistoricalData/FakeHistoryProvider.cs

@@ -19,6 +19,7 @@ using QuantConnect.Data;
 using QuantConnect.Interfaces;
 using QuantConnect.Data.Market;
 using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
 using QuantConnect.Lean.Engine.DataFeeds.Queues;
 
 namespace QuantConnect.Lean.Engine.HistoricalData
@@ -52,7 +53,7 @@ namespace QuantConnect.Lean.Engine.HistoricalData
         public override IEnumerable<Slice> GetHistory(IEnumerable<HistoryRequest> requests, DateTimeZone sliceTimeZone)
         {
             var single = requests.FirstOrDefault();
-            if(single == null)
+            if (single == null)
             {
                 yield break;
             }
@@ -90,6 +91,34 @@ namespace QuantConnect.Lean.Engine.HistoricalData
                             Period = single.Resolution.ToTimeSpan()
                         };
                     }
+                    else if (single.DataType == typeof(ZipEntryName))
+                    {
+                        if (single.Symbol.SecurityType == SecurityType.Future)
+                        {
+                            data = new ZipEntryName
+                            {
+                                Symbol = Symbol.CreateFuture(single.Symbol.ID.Symbol, single.Symbol.ID.Market, currentLocalTime.AddDays(20)),
+                                Time = currentLocalTime
+                            };
+                        }
+                        else if (single.Symbol.SecurityType.IsOption())
+                        {
+                            data = new ZipEntryName
+                            {
+                                Symbol = Symbol.CreateOption(single.Symbol.Underlying.ID.Symbol,
+                                    single.Symbol.ID.Market,
+                                    single.Symbol.Underlying.SecurityType.DefaultOptionStyle(),
+                                    default(OptionRight),
+                                    0m,
+                                    currentLocalTime.AddDays(20)),
+                                Time = currentLocalTime
+                            };
+                        }
+                        else
+                        {
+                            yield break;
+                        }
+                    }
                     else
                     {
                         yield break;

Engine/HistoricalData/SubscriptionDataReaderHistoryProvider.cs

@@ -158,7 +158,7 @@ namespace QuantConnect.Lean.Engine.HistoricalData
             {
                 // allow all ticks
                 if (config.Resolution == Resolution.Tick) return true;
-                // filter out all aux data
+                // filter out all aux data. TODO: what if we are asking for aux data?
                 if (data.DataType == MarketDataType.Auxiliary) return false;
                 // filter out future data
                 if (data.EndTime > request.EndTimeLocal) return false;

Engine/LeanEngineAlgorithmHandlers.cs

@@ -75,6 +75,11 @@ namespace QuantConnect.Lean.Engine
         /// </summary>
         public IDataProvider DataProvider { get; }
 
+        /// <summary>
+        /// Gets the data file provider used to retrieve security data if it is not on the file system
+        /// </summary>
+        public IDataCacheProvider DataCacheProvider { get; }
+
         /// <summary>
         /// Gets the alpha handler used to process algorithm generated insights
         /// </summary>
@@ -114,7 +119,8 @@ namespace QuantConnect.Lean.Engine
             IDataProvider dataProvider,
             IAlphaHandler alphas,
             IObjectStore objectStore,
-            IDataPermissionManager dataPermissionsManager
+            IDataPermissionManager dataPermissionsManager,
+            bool liveMode
             )
         {
             if (results == null)
@@ -173,6 +179,7 @@ namespace QuantConnect.Lean.Engine
             Alphas = alphas;
             ObjectStore = objectStore;
             DataPermissionsManager = dataPermissionsManager;
+            DataCacheProvider = new ZipDataCacheProvider(DataProvider, isDataEphemeral: liveMode);
         }
 
         /// <summary>
@@ -206,7 +213,8 @@ namespace QuantConnect.Lean.Engine
                 composer.GetExportedValueByTypeName<IDataProvider>(dataProviderTypeName),
                 composer.GetExportedValueByTypeName<IAlphaHandler>(alphaHandlerTypeName),
                 composer.GetExportedValueByTypeName<IObjectStore>(objectStoreTypeName),
-                composer.GetExportedValueByTypeName<IDataPermissionManager>(dataPermissionManager)
+                composer.GetExportedValueByTypeName<IDataPermissionManager>(dataPermissionManager),
+                Config.GetBool("live-mode")
                 );
 
             result.FactorFileProvider.Initialize(result.MapFileProvider, result.DataProvider);
@@ -230,6 +238,7 @@ namespace QuantConnect.Lean.Engine
         {
             Log.Trace("LeanEngineAlgorithmHandlers.Dispose(): start...");
 
+            DataCacheProvider.DisposeSafely();
             Setup.DisposeSafely();
             ObjectStore.DisposeSafely();
 

Engine/Setup/BacktestingSetupHandler.cs

@@ -15,7 +15,6 @@
 */
 
 using System;
-using System.Linq;
 using QuantConnect.Util;
 using QuantConnect.Logging;
 using QuantConnect.Packets;
@@ -171,10 +170,10 @@ namespace QuantConnect.Lean.Engine.Setup
                     algorithm.Schedule.SetEventSchedule(parameters.RealTimeHandler);
 
                     // set the option chain provider
-                    algorithm.SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider(parameters.DataProvider)));
+                    algorithm.SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider(parameters.DataCacheProvider, parameters.MapFileProvider)));
 
                     // set the future chain provider
-                    algorithm.SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider(parameters.DataProvider)));
+                    algorithm.SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider(parameters.DataCacheProvider)));
 
                     // set the object store
                     algorithm.SetObjectStore(parameters.ObjectStore);

Engine/Setup/BrokerageSetupHandler.cs

@@ -252,7 +252,7 @@ namespace QuantConnect.Lean.Engine.Setup
                         var optionChainProvider = Composer.Instance.GetPart<IOptionChainProvider>();
                         if (optionChainProvider == null)
                         {
-                            optionChainProvider = new CachingOptionChainProvider(new LiveOptionChainProvider());
+                            optionChainProvider = new CachingOptionChainProvider(new LiveOptionChainProvider(parameters.DataCacheProvider, parameters.MapFileProvider));
                         }
                         // set the option chain provider
                         algorithm.SetOptionChainProvider(optionChainProvider);
@@ -260,7 +260,7 @@ namespace QuantConnect.Lean.Engine.Setup
                         var futureChainProvider = Composer.Instance.GetPart<IFutureChainProvider>();
                         if (futureChainProvider == null)
                         {
-                            futureChainProvider = new CachingFutureChainProvider(new LiveFutureChainProvider());
+                            futureChainProvider = new CachingFutureChainProvider(new LiveFutureChainProvider(parameters.DataCacheProvider));
                         }
                         // set the future chain provider
                         algorithm.SetFutureChainProvider(futureChainProvider);

Engine/Setup/SetupHandlerParameters.cs

@@ -69,9 +69,14 @@ namespace QuantConnect.Lean.Engine.Setup
         public IObjectStore ObjectStore { get; }
 
         /// <summary>
-        /// Gets the DataProvider
+        /// Gets the DataCacheProvider
         /// </summary>
-        public IDataProvider DataProvider { get; }
+        public IDataCacheProvider DataCacheProvider { get; }
+
+        /// <summary>
+        /// The map file provider instance of the algorithm
+        /// </summary>
+        public IMapFileProvider MapFileProvider { get; }
 
         /// <summary>
         /// Creates a new instance
@@ -84,7 +89,8 @@ namespace QuantConnect.Lean.Engine.Setup
         /// <param name="transactionHandler">The configured transaction handler</param>
         /// <param name="realTimeHandler">The configured real time handler</param>
         /// <param name="objectStore">The configured object store</param>
-        /// <param name="dataProvider">The configured data provider</param>
+        /// <param name="dataCacheProvider">The configured data cache provider</param>
+        /// <param name="mapFileProvider">The map file provider</param>
         public SetupHandlerParameters(UniverseSelection universeSelection,
             IAlgorithm algorithm,
             IBrokerage brokerage,
@@ -93,7 +99,8 @@ namespace QuantConnect.Lean.Engine.Setup
             ITransactionHandler transactionHandler,
             IRealTimeHandler realTimeHandler,
             IObjectStore objectStore,
-            IDataProvider dataProvider
+            IDataCacheProvider dataCacheProvider,
+            IMapFileProvider mapFileProvider
             )
         {
             UniverseSelection = universeSelection;
@@ -104,7 +111,8 @@ namespace QuantConnect.Lean.Engine.Setup
             TransactionHandler = transactionHandler;
             RealTimeHandler = realTimeHandler;
             ObjectStore = objectStore;
-            DataProvider = dataProvider;
+            DataCacheProvider = dataCacheProvider;
+            MapFileProvider = mapFileProvider;
         }
     }
 }

Research/QuantBook.cs

@@ -188,8 +188,8 @@ namespace QuantConnect.Research
                     )
                 );
 
-                SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider(_dataProvider)));
-                SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider(_dataProvider)));
+                SetOptionChainProvider(new CachingOptionChainProvider(new BacktestingOptionChainProvider(_dataCacheProvider, mapFileProvider)));
+                SetFutureChainProvider(new CachingFutureChainProvider(new BacktestingFutureChainProvider(_dataCacheProvider)));
             }
             catch (Exception exception)
             {

Tests/Common/Data/SliceTests.cs

@@ -21,6 +21,7 @@ using NUnit.Framework;
 using Python.Runtime;
 using QuantConnect.Data;
 using QuantConnect.Data.Custom;
+using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Custom.IconicTypes;
 using QuantConnect.Data.Market;
 using QuantConnect.Indicators;
@@ -44,8 +45,9 @@ namespace QuantConnect.Tests.Common.Data
             var openInterest = new OpenInterest(now, Symbols.SPY, 1);
             var split = new Split(Symbols.SPY, now, 1, 1, SplitType.SplitOccurred);
             var delisting = new Delisting(Symbols.SPY, now, 1, DelistingType.Delisted);
+            var auxiliaryData = new ZipEntryName() { Symbol = Symbols.SPY, Time = now };
 
-            var slice = new Slice(now, new BaseData[] { quoteBar, tradeBar, unlinkedData, tick, split, delisting, openInterest }, now);
+            var slice = new Slice(now, new BaseData[] { quoteBar, tradeBar, unlinkedData, tick, split, delisting, openInterest, auxiliaryData }, now);
 
             Assert.AreEqual(slice.Get(typeof(TradeBar))[Symbols.SPY], tradeBar);
             Assert.AreEqual(slice.Get(typeof(UnlinkedData))[Symbols.SPY], unlinkedData);
@@ -54,6 +56,7 @@ namespace QuantConnect.Tests.Common.Data
             Assert.AreEqual(slice.Get(typeof(Split))[Symbols.SPY], split);
             Assert.AreEqual(slice.Get(typeof(Delisting))[Symbols.SPY], delisting);
             Assert.AreEqual(slice.Get(typeof(OpenInterest))[Symbols.SPY], openInterest);
+            Assert.AreEqual(slice.Get(typeof(ZipEntryName))[Symbols.SPY], auxiliaryData);
         }
 
         [Test]

Tests/Common/Securities/Options/OptionChainProviderTests.cs

@@ -21,6 +21,7 @@ using System.Threading;
 using NUnit.Framework;
 using QuantConnect.Interfaces;
 using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Securities;
 using QuantConnect.Securities.Future;
 using QuantConnect.Util;
 
@@ -29,10 +30,26 @@ namespace QuantConnect.Tests.Common.Securities.Options
     [TestFixture, Parallelizable(ParallelScope.Fixtures)]
     public class OptionChainProviderTests
     {
+        [Test]
+        public void BacktestingOptionChainProviderUsesPreviousTradableDateChain()
+        {
+            // the 7th is a saturday should fetch fridays data instead
+            var date = new DateTime(2014, 6, 7);
+            Assert.AreEqual(DayOfWeek.Saturday, date.DayOfWeek);
+
+            var provider = new BacktestingOptionChainProvider(TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider);
+            var twxOptionChain = provider.GetOptionContractList(Symbol.Create("TWX", SecurityType.Equity, Market.USA), date)
+                .ToList();
+
+            Assert.AreEqual(184, twxOptionChain.Count);
+            Assert.AreEqual(23m, twxOptionChain.OrderBy(s => s.ID.StrikePrice).First().ID.StrikePrice);
+            Assert.AreEqual(105m, twxOptionChain.OrderBy(s => s.ID.StrikePrice).Last().ID.StrikePrice);
+        }
+
         [Test]
         public void BacktestingOptionChainProviderLoadsEquityOptionChain()
         {
-            var provider = new BacktestingOptionChainProvider(TestGlobals.DataProvider);
+            var provider = new BacktestingOptionChainProvider(TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider);
             var twxOptionChain = provider.GetOptionContractList(Symbol.Create("TWX", SecurityType.Equity, Market.USA), new DateTime(2014, 6, 5))
                 .ToList();
 
@@ -44,7 +61,7 @@ namespace QuantConnect.Tests.Common.Securities.Options
         [Test]
         public void BacktestingOptionChainProviderLoadsFutureOptionChain()
         {
-            var provider = new BacktestingOptionChainProvider(TestGlobals.DataProvider);
+            var provider = new BacktestingOptionChainProvider(TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider);
             var esOptionChain = provider.GetOptionContractList(
                 Symbol.CreateFuture(
                     QuantConnect.Securities.Futures.Indices.SP500EMini,
@@ -62,7 +79,7 @@ namespace QuantConnect.Tests.Common.Securities.Options
         public void BacktestingOptionChainProviderResolvesSymbolMapping()
         {
             var ticker = "GOOCV"; // Old ticker, should resolve and fetch GOOG
-            var provider = new BacktestingOptionChainProvider(TestGlobals.DataProvider);
+            var provider = new BacktestingOptionChainProvider(TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider);
 
             var underlyingSymbol = QuantConnect.Symbol.Create(ticker, SecurityType.Equity, Market.USA);
             var alias = "?" + underlyingSymbol.Value;
@@ -113,7 +130,7 @@ namespace QuantConnect.Tests.Common.Securities.Options
         [Test]
         public void LiveOptionChainProviderReturnsData()
         {
-            var provider = new LiveOptionChainProvider();
+            var provider = new LiveOptionChainProvider(TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider);
 
             foreach (var symbol in new[] { Symbols.SPY, Symbols.AAPL, Symbols.MSFT })
             {
@@ -131,7 +148,7 @@ namespace QuantConnect.Tests.Common.Securities.Options
         {
             var symbol = Symbol.Create("ABCDEF123", SecurityType.Equity, Market.USA);
 
-            var provider = new LiveOptionChainProvider();
+            var provider = new LiveOptionChainProvider(TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider);
             var result = provider.GetOptionContractList(symbol, DateTime.Today);
 
             Assert.IsFalse(result.Any());
@@ -155,7 +172,7 @@ namespace QuantConnect.Tests.Common.Securities.Options
             }
 
             var underlyingFuture = Symbol.CreateFuture("ES", Market.CME, expiry);
-            var provider = new LiveOptionChainProvider();
+            var provider = new LiveOptionChainProvider(TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider);
             var result = provider.GetOptionContractList(underlyingFuture, now).ToList();
 
             Assert.AreNotEqual(0, result.Count);
@@ -177,7 +194,7 @@ namespace QuantConnect.Tests.Common.Securities.Options
             var december = now.AddMonths(-now.Month).AddYears(-1);
             var underlyingFuture = Symbol.CreateFuture("ES", Market.CME, december);
 
-            var provider = new LiveOptionChainProvider();
+            var provider = new LiveOptionChainProvider(TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider);
             var result = provider.GetOptionContractList(underlyingFuture, december);
 
             Assert.AreEqual(0, result.Count());
@@ -204,7 +221,7 @@ namespace QuantConnect.Tests.Common.Securities.Options
                 strike,
                 expiry);
 
-            var provider = new BacktestingOptionChainProvider(TestGlobals.DataProvider);
+            var provider = new BacktestingOptionChainProvider(TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider);
             var contracts = provider.GetOptionContractList(underlying, new DateTime(2020, 1, 5))
                 .ToHashSet();
 

Tests/Common/Securities/SecurityIdentifierTests.cs

@@ -26,6 +26,7 @@ using QuantConnect.Algorithm.CSharp;
 using QuantConnect.Data.Auxiliary;
 using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Logging;
+using QuantConnect.Securities;
 using QuantConnect.Util;
 
 namespace QuantConnect.Tests.Common.Securities
@@ -535,7 +536,7 @@ namespace QuantConnect.Tests.Common.Securities
         [Test, Ignore("Requires complete option data to validate chain")]
         public void ValidateAAPLOptionChainSecurityIdentifiers()
         {
-            var chainProvider = new BacktestingOptionChainProvider(TestGlobals.DataProvider);
+            var chainProvider = new BacktestingOptionChainProvider(TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider);
             var aapl = Symbol.Create("AAPL", SecurityType.Equity, Market.USA);
             var chains = new HashSet<Symbol>();
             var expectedChains = File.ReadAllLines("TestData/aapl_chain.csv")

Tests/Engine/DataFeeds/BacktestingFutureChainProviderTests.cs

@@ -33,7 +33,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds
         {
             // Store initial Log Handler
             _logHandler = Log.LogHandler;
-            _provider = new BacktestingFutureChainProvider(TestGlobals.DataProvider);
+            _provider = new BacktestingFutureChainProvider(TestGlobals.DataCacheProvider);
         }
 
         [OneTimeTearDown]
@@ -43,11 +43,14 @@ namespace QuantConnect.Tests.Engine.DataFeeds
             Log.LogHandler = _logHandler;
         }
 
-        [Test]
-        public void CorrectlyDeterminesContractList()
+        [TestCase("20131011")]
+        // saturday, will fetch previous tradable date instead
+        [TestCase("20131012")]
+        public void CorrectlyDeterminesContractList(string date)
         {
-            var symbol = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, DateTime.Today);
-            var result = _provider.GetFutureContractList(symbol, new DateTime(2013, 10, 11));
+            var dateTime = Time.ParseDate(date);
+            var symbol = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, dateTime.AddDays(10));
+            var result = _provider.GetFutureContractList(symbol, dateTime);
 
             Assert.IsNotEmpty(result);
         }
@@ -57,11 +60,11 @@ namespace QuantConnect.Tests.Engine.DataFeeds
         {
             var testHandler = new QueueLogHandler();
             Log.LogHandler = testHandler;
-            var symbol = Symbol.CreateFuture("NonExisting", Market.USA, DateTime.UtcNow);
+            var symbol = Symbol.CreateFuture("NonExisting", Market.USA, new DateTime(2013, 11, 11));
             var result = _provider.GetFutureContractList(symbol, new DateTime(2013, 10, 11)).ToList();
 
             Assert.IsTrue(testHandler.Logs.Any(entry => 
-            entry.Message.Contains("BacktestingFutureChainProvider.GetFutureContractList(): Failed, files not found:")));
+            entry.Message.Contains("found no source of contracts for NONEXISTING 2X for date 20131011 for any tick type")));
             Assert.IsEmpty(result);
         }
     }

Tests/Engine/DataFeeds/Enumerators/DataQueueFuturesChainUniverseDataCollectionEnumeratorTests.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -22,14 +22,14 @@ using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Interfaces;
 using QuantConnect.Lean.Engine.DataFeeds;
-using QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
 using QuantConnect.Securities;
 using QuantConnect.Securities.Future;
 
 namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
 {
     [TestFixture]
-    public class FuturesChainUniverseSubscriptionEnumeratorFactoryTests
+    public class DataQueueFuturesChainUniverseDataCollectionEnumeratorTests
     {
         [Test]
         public void RefreshesFutureChainUniverseOnDateChange()
@@ -38,7 +38,6 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
             var timeProvider = new ManualTimeProvider(startTime);
 
             var symbolUniverse = new TestDataQueueUniverseProvider(timeProvider);
-            var factory = new FuturesChainUniverseSubscriptionEnumeratorFactory(symbolUniverse, timeProvider);
 
             var canonicalSymbol = Symbol.Create(Futures.Indices.VIX, SecurityType.Future, Market.CFE, "/VX");
 
@@ -70,14 +69,14 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
             );
 
             var universeSettings = new UniverseSettings(Resolution.Minute, 0, true, false, TimeSpan.Zero);
-            var universe = new FuturesChainUniverse(future, universeSettings);
+            using var universe = new FuturesChainUniverse(future, universeSettings);
             var request = new SubscriptionRequest(true, universe, future, config, startTime, Time.EndOfTime);
-            var enumerator = factory.CreateEnumerator(request, new DefaultDataProvider());
+            var enumerator = new DataQueueFuturesChainUniverseDataCollectionEnumerator(request, symbolUniverse, timeProvider);
 
             Assert.IsTrue(enumerator.MoveNext());
             Assert.IsNotNull(enumerator.Current);
             Assert.AreEqual(1, symbolUniverse.TotalLookupCalls);
-            var data = enumerator.Current as FuturesChainUniverseDataCollection;
+            var data = enumerator.Current;
             Assert.IsNotNull(data);
             Assert.AreEqual(1, data.Data.Count);
 
@@ -98,7 +97,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
             Assert.IsTrue(enumerator.MoveNext());
             Assert.IsNotNull(enumerator.Current);
             Assert.AreEqual(2, symbolUniverse.TotalLookupCalls);
-            data = enumerator.Current as FuturesChainUniverseDataCollection;
+            data = enumerator.Current;
             Assert.IsNotNull(data);
             Assert.AreEqual(2, data.Data.Count);
 
@@ -111,7 +110,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
             enumerator.Dispose();
         }
 
-        public class TestDataQueueUniverseProvider : IDataQueueUniverseProvider
+        private class TestDataQueueUniverseProvider : IDataQueueUniverseProvider
         {
             private readonly Symbol[] _symbolList1 =
             {

Tests/Engine/DataFeeds/Enumerators/Factories/BaseDataCollectionSubscriptionEnumeratorFactoryTests.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -20,7 +20,6 @@ using NUnit.Framework;
 using QuantConnect.Brokerages;
 using QuantConnect.Data;
 using QuantConnect.Data.UniverseSelection;
-using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories;
 using QuantConnect.Logging;
 using QuantConnect.Securities;
@@ -50,8 +49,6 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
             var securityInitializer = new BrokerageModelSecurityInitializer(new DefaultBrokerageModel(), SecuritySeeder.Null);
             var universe = new CoarseFundamentalUniverse(universeSettings, x => new List<Symbol>{ Symbols.AAPL });
 
-            var fileProvider = new DefaultDataProvider();
-
             var factory = new BaseDataCollectionSubscriptionEnumeratorFactory();
 
             GC.Collect();
@@ -63,7 +60,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
             for (var i = 0; i < iterations; i++)
             {
                 var request = new SubscriptionRequest(true, universe, security, config, date, date);
-                using (var enumerator = factory.CreateEnumerator(request, fileProvider))
+                using (var enumerator = factory.CreateEnumerator(request, TestGlobals.DataProvider))
                 {
                     enumerator.MoveNext();
                 }
@@ -96,8 +93,6 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
             var securityInitializer = new BrokerageModelSecurityInitializer(new DefaultBrokerageModel(), SecuritySeeder.Null);
             var universe = new CoarseFundamentalUniverse(universeSettings, x => new List<Symbol> { Symbols.AAPL });
 
-            var fileProvider = new DefaultDataProvider();
-
             var factory = new BaseDataCollectionSubscriptionEnumeratorFactory();
 
             var dateStart = new DateTime(2014, 3, 26);
@@ -106,7 +101,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
 
             var request = new SubscriptionRequest(true, universe, security, config, dateStart, dateEnd);
 
-            using (var enumerator = factory.CreateEnumerator(request, fileProvider))
+            using (var enumerator = factory.CreateEnumerator(request, TestGlobals.DataProvider))
             {
                 dateStart = dateStart.AddDays(-1);
                 for (var i = 0; i <= days; i++)

Tests/Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactoryTests.cs

@@ -17,7 +17,6 @@
 using System;
 using NUnit.Framework;
 using QuantConnect.Data;
-using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Market;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Lean.Engine.DataFeeds;
@@ -34,7 +33,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
         [Test, Category("TravisExclude")]
         public void DoesNotLeakMemory()
         {
-            var symbol = Symbols.AAPL;
+            var symbol = Symbols.SPY_Option_Chain;
             var config = new SubscriptionDataConfig(typeof(TradeBar), symbol, Resolution.Daily, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, TickType.Trade, false);
             var security = new Security(
                 SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
@@ -47,8 +46,8 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
             );
 
             var fileProvider = TestGlobals.DataProvider;
-            using var cache = new ZipDataCacheProvider(fileProvider);
-            var factory = new BaseDataSubscriptionEnumeratorFactory(false, TestGlobals.MapFileProvider, cache);
+            var cache = TestGlobals.DataCacheProvider;
+            var factory = new BaseDataSubscriptionEnumeratorFactory(new BacktestingOptionChainProvider(cache, TestGlobals.MapFileProvider), new BacktestingFutureChainProvider(cache));
 
             GC.Collect();
             var ramUsageBeforeLoop = OS.TotalPhysicalMemoryUsed;

Tests/Engine/DataFeeds/Enumerators/Factories/OptionChainUniverseSubscriptionEnumeratorFactoryTests.cs

@@ -34,80 +34,6 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
     [TestFixture]
     public class OptionChainUniverseSubscriptionEnumeratorFactoryTests
     {
-        [Test]
-        public void DoesNotEmitInvalidData()
-        {
-            var startTime = new DateTime(2014, 06, 06, 0, 0, 0);
-            var endTime = new DateTime(2014, 06, 09, 20, 0, 0);
-
-            var canonicalSymbol = Symbol.Create("AAPL", SecurityType.Option, Market.USA, "?AAPL");
-
-            var quoteCurrency = new Cash(Currencies.USD, 0, 1);
-            var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(Market.USA, canonicalSymbol, SecurityType.Option);
-            var config = new SubscriptionDataConfig(
-                typeof(ZipEntryName),
-                canonicalSymbol,
-                Resolution.Minute,
-                TimeZones.Utc,
-                TimeZones.NewYork,
-                true,
-                false,
-                false,
-                false,
-                TickType.Quote,
-                false,
-                DataNormalizationMode.Raw
-            );
-
-            var option = new Option(
-                canonicalSymbol,
-                exchangeHours,
-                quoteCurrency,
-                new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
-                ErrorCurrencyConverter.Instance,
-                RegisteredSecurityDataTypesProvider.Null,
-                new SecurityCache(),
-                null
-            );
-
-            var dataProvider = TestGlobals.DataProvider;
-            using var cache = new ZipDataCacheProvider(dataProvider);
-            var enumeratorFactory = new BaseDataSubscriptionEnumeratorFactory(false, TestGlobals.MapFileProvider, cache);
-            var fillForwardResolution = Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan());
-            Func<SubscriptionRequest, IEnumerator<BaseData>> underlyingEnumeratorFunc = (req) =>
-                {
-                    var input = enumeratorFactory.CreateEnumerator(req, dataProvider);
-
-                    input = new BaseDataCollectionAggregatorEnumerator(input, req.Configuration.Symbol);
-                    return new FillForwardEnumerator(
-                        input,
-                        option.Exchange,
-                        fillForwardResolution,
-                        false,
-                        endTime,
-                        Resolution.Minute.ToTimeSpan(),
-                        TimeZones.Utc);
-                };
-            var factory = new OptionChainUniverseSubscriptionEnumeratorFactory(underlyingEnumeratorFunc);
-
-            var request = new SubscriptionRequest(true, null, option, config, startTime, endTime);
-            var enumerator = factory.CreateEnumerator(request, dataProvider);
-
-            var emittedCount = 0;
-            foreach (var data in enumerator.AsEnumerable())
-            {
-                emittedCount++;
-                var optionData = data as OptionChainUniverseDataCollection;
-
-                Assert.IsNotNull(optionData);
-                Assert.IsNotNull(optionData.Underlying);
-                Assert.AreNotEqual(0, optionData.Data.Count);
-            }
-
-            // 9:30 to 15:59 -> 6.5 hours * 60 => 390 minutes * 2 days = 780
-            Assert.AreEqual(780, emittedCount);
-        }
-
         [Test]
         public void RefreshesOptionChainUniverseOnDateChange()
         {

Tests/Engine/DataFeeds/FileSystemDataFeedTests.cs

@@ -19,7 +19,6 @@ using System.Diagnostics;
 using System.Linq;
 using System.Threading;
 using NUnit.Framework;
-using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories;
@@ -27,11 +26,12 @@ using QuantConnect.Lean.Engine.Results;
 using QuantConnect.Logging;
 using QuantConnect.Packets;
 using QuantConnect.Securities;
+using QuantConnect.Tests.Common.Securities;
 using QuantConnect.Util;
 
 namespace QuantConnect.Tests.Engine.DataFeeds
 {
-    [TestFixture, Category("TravisExclude")]
+    [TestFixture]
     public class FileSystemDataFeedTests
     {
         [Test]
@@ -50,7 +50,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds
                     algorithm,
                     new SecurityService(algorithm.Portfolio.CashBook, marketHoursDatabase, symbolPropertiesDataBase, algorithm, RegisteredSecurityDataTypesProvider.Null, new SecurityCacheProvider(algorithm.Portfolio)),
                     dataPermissionManager,
-                    new DefaultDataProvider()),
+                    TestGlobals.DataProvider),
                 algorithm,
                 algorithm.TimeKeeper,
                 marketHoursDatabase,
@@ -58,14 +58,14 @@ namespace QuantConnect.Tests.Engine.DataFeeds
                 RegisteredSecurityDataTypesProvider.Null,
                 dataPermissionManager);
             algorithm.SubscriptionManager.SetDataManager(dataManager);
-            var synchronizer = new Synchronizer();
+            using var synchronizer = new Synchronizer();
             synchronizer.Initialize(algorithm, dataManager);
 
             feed.Initialize(algorithm, job, resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, dataManager, synchronizer, dataPermissionManager.DataChannelProvider);
             algorithm.Initialize();
             algorithm.PostInitialize();
 
-            var cancellationTokenSource = new CancellationTokenSource();
+            using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
             var count = 0;
             var stopwatch = Stopwatch.StartNew();
             var lastMonth = algorithm.StartDate.Month;
@@ -95,8 +95,8 @@ namespace QuantConnect.Tests.Engine.DataFeeds
             algorithm.PostInitialize();
 
             var resultHandler = new BacktestingResultHandler();
-            using var cache = new ZipDataCacheProvider(TestGlobals.DataProvider);
-            var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, cache, enablePriceScaling: false);
+            using var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, TestGlobals.MapFileProvider,
+                TestGlobals.FactorFileProvider, TestGlobals.DataCacheProvider, enablePriceScaling: false);
 
             var universe = algorithm.UniverseManager.Single().Value;
             var security = algorithm.Securities.Single().Value;
@@ -141,8 +141,8 @@ namespace QuantConnect.Tests.Engine.DataFeeds
             algorithm.PostInitialize();
 
             var resultHandler = new TestResultHandler();
-            using var cache = new ZipDataCacheProvider(TestGlobals.DataProvider);
-            var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, cache, enablePriceScaling: false);
+            using var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, TestGlobals.MapFileProvider,
+                TestGlobals.FactorFileProvider, TestGlobals.DataCacheProvider, enablePriceScaling: false);
 
             var universe = algorithm.UniverseManager.Single().Value;
             var security = algorithm.AddEquity("AAA", Resolution.Daily);
@@ -158,9 +158,132 @@ namespace QuantConnect.Tests.Engine.DataFeeds
             factory.DisposeSafely();
             resultHandler.Exit();
 
-            var message = ((DebugPacket) resultHandler.Messages.Single()).Message;
+            var message = ((DebugPacket)resultHandler.Messages.Single()).Message;
             Assert.IsTrue(message.Equals(
                 "The starting dates for the following symbols have been adjusted to match their map files first date: [AAA, 2020-09-09]"));
         }
+
+        [TestCase(true)]
+        [TestCase(false)]
+        public void OptionChainEnumerator(bool fillForward)
+        {
+            var job = new BacktestNodePacket();
+            var resultHandler = new BacktestingResultHandler();
+            var feed = new FileSystemDataFeed();
+            var algorithm = new AlgorithmStub(feed);
+            algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor());
+            algorithm.SetStartDate(new DateTime(2014, 06, 06));
+            algorithm.SetEndDate(new DateTime(2014, 06, 09));
+            algorithm.SetOptionChainProvider(new BacktestingOptionChainProvider(TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider));
+
+            var dataPermissionManager = new DataPermissionManager();
+            using var synchronizer = new Synchronizer();
+            synchronizer.Initialize(algorithm, algorithm.DataManager);
+
+            feed.Initialize(algorithm, job, resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, algorithm.DataManager, synchronizer, dataPermissionManager.DataChannelProvider);
+            var option = algorithm.AddOption("AAPL", fillDataForward: fillForward);
+            option.SetFilter(filter => filter.FrontMonth());
+            algorithm.PostInitialize();
+
+            using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+            var count = 0;
+            var lastMonth = algorithm.StartDate.Month;
+            foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token))
+            {
+                if (!timeSlice.IsTimePulse && timeSlice.UniverseData?.Count > 0)
+                {
+                    var baseDataCollection = timeSlice.UniverseData.Single().Value;
+                    if (baseDataCollection.Symbol.SecurityType == SecurityType.Option)
+                    {
+                        var nyTime = timeSlice.Time.ConvertFromUtc(algorithm.TimeZone);
+                        Assert.AreEqual(new TimeSpan(9, 30, 0).Add(TimeSpan.FromMinutes((count % 390) + 1)), nyTime.TimeOfDay, $"Failed on: {nyTime}");
+                        Assert.IsNotNull(baseDataCollection.Underlying);
+                        Assert.IsTrue(!baseDataCollection.FilteredContracts.IsNullOrEmpty());
+                        count++;
+                    }
+                }
+            }
+            feed.Exit();
+            algorithm.DataManager.RemoveAllSubscriptions();
+
+            // 9:30 to 15:59 -> 6.5 hours * 60 => 390 minutes * 2 days = 780
+            Assert.AreEqual(780, count);
+        }
+
+        [TestCase(true)]
+        [TestCase(false)]
+        public void FutureChainEnumerator(bool fillForward)
+        {
+            var job = new BacktestNodePacket();
+            var resultHandler = new BacktestingResultHandler();
+            var feed = new FileSystemDataFeed();
+            var algorithm = new AlgorithmStub(feed);
+            algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor());
+            algorithm.SetStartDate(new DateTime(2013, 10, 07));
+            algorithm.SetEndDate(new DateTime(2013, 10, 08));
+            algorithm.SetFutureChainProvider(new BacktestingFutureChainProvider(TestGlobals.DataCacheProvider));
+
+            var dataPermissionManager = new DataPermissionManager();
+            using var synchronizer = new Synchronizer();
+            synchronizer.Initialize(algorithm, algorithm.DataManager);
+
+            feed.Initialize(algorithm, job, resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, algorithm.DataManager, synchronizer, dataPermissionManager.DataChannelProvider);
+            var future = algorithm.AddFuture("ES", fillDataForward: fillForward);
+            future.SetFilter(0, 300);
+            algorithm.PostInitialize();
+
+            using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+            var count = 0L;
+            var lastMonth = algorithm.StartDate.Month;
+            foreach (var timeSlice in synchronizer.StreamData(cancellationTokenSource.Token))
+            {
+                if (!timeSlice.IsTimePulse && timeSlice.UniverseData?.Count > 0)
+                {
+                    var nyTime = timeSlice.Time.ConvertFromUtc(algorithm.TimeZone);
+
+                    var currentExpectedTime = new TimeSpan(0, 0, 0).Add(TimeSpan.FromMinutes(count % (24 * 60)));
+                    while (!future.Exchange.DateTimeIsOpen(nyTime.Date.Add(currentExpectedTime).AddMinutes(-1)))
+                    {
+                        // skip closed market times
+                        currentExpectedTime = new TimeSpan(0, 0, 0).Add(TimeSpan.FromMinutes(++count % (24 * 60)));
+                    }
+                    var universeData = timeSlice.UniverseData.OrderBy(kvp => kvp.Key.Configuration.Symbol).ToList();
+
+                    var chainData = universeData[0].Value;
+
+                    Log.Trace($"{nyTime}. Count: {count}. Universe Data Count {universeData.Count}");
+                    Assert.AreEqual(currentExpectedTime, nyTime.TimeOfDay, $"Failed on: {nyTime}. Count: {count}");
+                    if (chainData.FilteredContracts.IsNullOrEmpty())
+                    {
+                        Assert.AreEqual(new DateTime(2013, 10, 09), nyTime, $"Unexpected chain FilteredContracts was empty on {nyTime}");
+                    }
+
+                    if (universeData.Count == 1)
+                    {
+                        // the chain
+                        Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol == future.Symbol));
+                    }
+                    else
+                    {
+                        // we have 2 universe data, the chain and the continuous future
+                        Assert.AreEqual(2, universeData.Count);
+                        Assert.IsTrue(universeData.All(kvp => kvp.Key.Configuration.Symbol.SecurityType == SecurityType.Future));
+                        Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol == future.Symbol));
+                        Assert.IsTrue(universeData.Any(kvp => kvp.Key.Configuration.Symbol.ID.Symbol.Contains("CONTINUOUS", StringComparison.InvariantCultureIgnoreCase)));
+
+                        var continuousData = universeData[1].Value;
+                        Assert.AreEqual(currentExpectedTime, nyTime.TimeOfDay, $"Failed on: {nyTime}");
+                        Assert.IsTrue(!chainData.FilteredContracts.IsNullOrEmpty());
+                    }
+
+                    count++;
+                }
+            }
+            feed.Exit();
+            algorithm.DataManager.RemoveAllSubscriptions();
+
+            // 2 days worth of minute data
+            Assert.AreEqual(24 * 2 * 60 + 1, count);
+        }
     }
 }

Tests/Engine/DataFeeds/InternalSubscriptionManagerTests.cs

@@ -60,13 +60,18 @@ namespace QuantConnect.Tests.Engine.DataFeeds
         }
 
         [TestCaseSource(nameof(DataTypeTestCases))]
-        public void CreatesSubscriptions(SubscriptionRequest subscriptionRequest, bool liveMode, bool expectNewSubscription)
+        public void CreatesSubscriptions(SubscriptionRequest subscriptionRequest, bool liveMode, bool expectNewSubscription, bool isWarmup)
         {
             _algorithm.SetLiveMode(liveMode);
+            if (isWarmup)
+            {
+                _algorithm.SetWarmUp(10, Resolution.Daily);
+            }
+            _algorithm.PostInitialize();
 
             var added = false;
             var start = DateTime.UtcNow;
-            var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(15));
+            using var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(15));
             foreach (var timeSlice in _synchronizer.StreamData(tokenSource.Token))
             {
                 if (!added)
@@ -84,6 +89,12 @@ namespace QuantConnect.Tests.Engine.DataFeeds
 
                     if (expectNewSubscription)
                     {
+                        var utcStartTime = _dataManager.DataFeedSubscriptions
+                            .Where(subscription => subscription.Configuration.IsInternalFeed && subscription.Configuration.Symbol == Symbols.BTCUSD)
+                            .Select(subscription => subscription.UtcStartTime)
+                            .First();
+                        Assert.Greater(utcStartTime.Ticks, start.Ticks);
+
                         // let's wait for a data point
                         if (timeSlice.DataPointCount > 0)
                         {
@@ -110,18 +121,23 @@ namespace QuantConnect.Tests.Engine.DataFeeds
         }
 
         [TestCaseSource(nameof(DataTypeTestCases))]
-        public void RemoveSubscriptions(SubscriptionRequest subscriptionRequest, bool liveMode, bool expectNewSubscription)
+        public void RemoveSubscriptions(SubscriptionRequest subscriptionRequest, bool liveMode, bool expectNewSubscription, bool isWarmup)
         {
-            _algorithm.SetLiveMode(liveMode);
             if (!expectNewSubscription)
             {
                 // we only test cases where we expect an internal subscription
                 return;
             }
+            _algorithm.SetLiveMode(liveMode);
+            if (isWarmup)
+            {
+                _algorithm.SetWarmUp(10, Resolution.Daily);
+            }
+            _algorithm.PostInitialize();
             var added = false;
             var shouldRemoved = false;
             var count = 0;
-            var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(15));
+            using var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(15));
             foreach (var timeSlice in _synchronizer.StreamData(tokenSource.Token))
             {
                 if (!added)
@@ -163,15 +179,16 @@ namespace QuantConnect.Tests.Engine.DataFeeds
             dataQueueTest.ManualTimeProvider.SetCurrentTimeUtc(new DateTime(2020, 09, 03, 10, 0, 0));
             TearDown();
             var liveSynchronizer = new TestableLiveSynchronizer(dataQueueTest.ManualTimeProvider);
-            var dataAggregator = new TestAggregationManager(dataQueueTest.ManualTimeProvider);
+            using var dataAggregator = new TestAggregationManager(dataQueueTest.ManualTimeProvider);
             SetupImpl(dataQueueTest, liveSynchronizer, dataAggregator);
 
             _algorithm.SetDateTime(dataQueueTest.ManualTimeProvider.GetUtcNow());
             _algorithm.SetLiveMode(true);
+            _algorithm.PostInitialize();
             var added = false;
             var first = true;
             var internalDataCount = 0;
-            var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+            using var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30));
             foreach (var timeSlice in _synchronizer.StreamData(tokenSource.Token))
             {
                 dataQueueTest.ManualTimeProvider.AdvanceSeconds(60);
@@ -230,11 +247,12 @@ namespace QuantConnect.Tests.Engine.DataFeeds
         public void UniverseSelectionAddAndRemove()
         {
             _algorithm.SetLiveMode(true);
+            _algorithm.PostInitialize();
             _algorithm.UniverseSettings.Resolution = Resolution.Hour;
             _algorithm.UniverseSettings.MinimumTimeInUniverse = TimeSpan.Zero;
             var added = false;
-            var manualEvent = new ManualResetEvent(false);
-            var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(15));
+            using var manualEvent = new ManualResetEvent(false);
+            using var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(15));
             foreach (var timeSlice in _synchronizer.StreamData(tokenSource.Token))
             {
                 if (!added)
@@ -285,18 +303,21 @@ namespace QuantConnect.Tests.Engine.DataFeeds
             {
                 var result = new List<TestCaseData>();
                 var config = GetConfig(Symbols.BTCUSD, Resolution.Second);
-                result.Add(new TestCaseData(new SubscriptionRequest(false, null, CreateSecurity(config), config, DateTime.UtcNow, DateTime.UtcNow), true, false));
+                result.Add(new TestCaseData(new SubscriptionRequest(false, null, CreateSecurity(config), config, DateTime.UtcNow, DateTime.UtcNow), true, false, false));
 
                 config = GetConfig(Symbols.BTCUSD, Resolution.Minute);
-                result.Add(new TestCaseData(new SubscriptionRequest(false, null, CreateSecurity(config), config, DateTime.UtcNow, DateTime.UtcNow), true, false));
+                result.Add(new TestCaseData(new SubscriptionRequest(false, null, CreateSecurity(config), config, DateTime.UtcNow, DateTime.UtcNow), true, false, false));
 
                 config = GetConfig(Symbols.BTCUSD, Resolution.Hour);
-                result.Add(new TestCaseData(new SubscriptionRequest(false, null, CreateSecurity(config), config, DateTime.UtcNow, DateTime.UtcNow), true, true));
+                result.Add(new TestCaseData(new SubscriptionRequest(false, null, CreateSecurity(config), config, DateTime.UtcNow, DateTime.UtcNow), true, true, false));
 
                 config = GetConfig(Symbols.BTCUSD, Resolution.Daily);
-                result.Add(new TestCaseData(new SubscriptionRequest(false, null, CreateSecurity(config), config, DateTime.UtcNow, DateTime.UtcNow), true, true));
+                result.Add(new TestCaseData(new SubscriptionRequest(false, null, CreateSecurity(config), config, DateTime.UtcNow, DateTime.UtcNow), true, true, false));
 
-                result.Add(new TestCaseData(new SubscriptionRequest(false, null, CreateSecurity(config), config, DateTime.UtcNow, DateTime.UtcNow), false, false));
+                config = GetConfig(Symbols.BTCUSD, Resolution.Daily);
+                result.Add(new TestCaseData(new SubscriptionRequest(false, null, CreateSecurity(config), config, DateTime.UtcNow, DateTime.UtcNow), true, true, true));
+
+                result.Add(new TestCaseData(new SubscriptionRequest(false, null, CreateSecurity(config), config, DateTime.UtcNow, DateTime.UtcNow), false, false, false));
 
                 return result.ToArray();
             }
@@ -331,7 +352,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds
             _dataFeed = new TestableLiveTradingDataFeed(dataQueueHandler ?? new FakeDataQueue(dataAggregator ?? new AggregationManager()));
             _algorithm = new AlgorithmStub(createDataManager: false);
             _synchronizer = synchronizer ?? new LiveSynchronizer();
-
+            _algorithm.SetStartDate(new DateTime(2022, 04, 13));
 
             var registeredTypesProvider = new RegisteredSecurityDataTypesProvider();
             var securityService = new SecurityService(_algorithm.Portfolio.CashBook,
@@ -364,13 +385,9 @@ namespace QuantConnect.Tests.Engine.DataFeeds
                 new DataChannelProvider());
             _algorithm.SubscriptionManager.SetDataManager(_dataManager);
             _algorithm.Securities.SetSecurityService(securityService);
-            _algorithm.SetFinishedWarmingUp();
             var backtestingTransactionHandler = new BacktestingTransactionHandler();
             backtestingTransactionHandler.Initialize(_algorithm, new PaperBrokerage(_algorithm, new LiveNodePacket()), _resultHandler);
             _algorithm.Transactions.SetOrderProcessor(backtestingTransactionHandler);
-
-            _algorithm.SetDateTime(new DateTime(2022, 04, 13));
-            _algorithm.PostInitialize();
         }
         private class TestAggregationManager : AggregationManager
         {

Tests/Engine/DataFeeds/LiveTradingDataFeedTests.cs

@@ -83,16 +83,102 @@ namespace QuantConnect.Tests.Engine.DataFeeds
         }
 
         [Test]
-        public void WarmupExpiredAsset()
+        public void WarmupOptionSelection()
         {
-            _startDate = new DateTime(2014, 6, 14);
-            CustomMockedFileBaseData.StartDate = _startDate;
+            _startDate = new DateTime(2014, 6, 9);
             _manualTimeProvider.SetCurrentTimeUtc(_startDate);
 
             var endDate = _startDate.AddDays(1);
+            _algorithm.SetBenchmark(x => 1);
+            _algorithm.SetWarmup(2, Resolution.Daily);
+            _algorithm.UniverseSettings.Resolution = Resolution.Hour;
+            var feed = RunDataFeed();
+            // after algorithm initialization let's set the time provider time to reflect warmup window
+            _manualTimeProvider.SetCurrentTimeUtc(_algorithm.UtcTime);
+
+            var es = _algorithm.AddOption("AAPL");
+            // allow time for the exchange to pick up the selection point
+            Thread.Sleep(50);
+            var countWarmup = 0;
+            var countLive = 0;
+            ConsumeBridge(feed, TimeSpan.FromSeconds(5), true, ts =>
+            {
+                if (ts.UniverseData?.Count > 0)
+                {
+                    if (_algorithm.IsWarmingUp)
+                    {
+                        countWarmup++;
+                    }
+                    else
+                    {
+                        countLive++;
+                        // we got what we wanted shortcut unit test
+                        _manualTimeProvider.SetCurrentTimeUtc(DateTime.UtcNow);
+                    }
+                }
+            },
+            endDate: endDate,
+            secondsTimeStep: 60 * 60);
+
+            Assert.AreNotEqual(0, countWarmup);
+            Assert.AreNotEqual(0, countLive);
+        }
+
+        [Test]
+        public void WarmupFutureSelection()
+        {
+            _startDate = new DateTime(2013, 10, 10);
+            _manualTimeProvider.SetCurrentTimeUtc(_startDate);
+
+            var endDate = _startDate.AddDays(1);
+            _algorithm.SetBenchmark(x => 1);
+            _algorithm.SetWarmup(2, Resolution.Daily);
+            _algorithm.UniverseSettings.Resolution = Resolution.Hour;
+            var feed = RunDataFeed();
+            // after algorithm initialization let's set the time provider time to reflect warmup window
+            _manualTimeProvider.SetCurrentTimeUtc(_algorithm.UtcTime);
+
+            var es = _algorithm.AddFuture("ES");
+            // allow time for the exchange to pick up the selection point
+            Thread.Sleep(50);
+            var countWarmup = 0;
+            var countLive = 0;
+            ConsumeBridge(feed, TimeSpan.FromSeconds(5), true, ts =>
+            {
+                if(ts.UniverseData?.Count > 0)
+                {
+                    if (_algorithm.IsWarmingUp)
+                    {
+                        countWarmup++;
+                    }
+                    else
+                    {
+                        countLive++;
+                        // we got what we wanted shortcut unit test
+                        _manualTimeProvider.SetCurrentTimeUtc(DateTime.UtcNow);
+                    }
+                }
+            },
+            endDate: endDate,
+            secondsTimeStep: 60 * 60);
+
+            Assert.AreNotEqual(0, countWarmup);
+            Assert.AreNotEqual(0, countLive);
+        }
+
+        [Test]
+        public void WarmupExpiredAsset()
+        {
+            _startDate = new DateTime(2014, 6, 14);
+            _manualTimeProvider.SetCurrentTimeUtc(_startDate);
+
+            var endDate = _startDate.AddDays(1);
+            _algorithm.SetBenchmark(x => 1);
             _algorithm.UniverseSettings.Resolution = Resolution.Daily;
             _algorithm.SetWarmup(10, Resolution.Daily);
             var feed = RunDataFeed();
+            // after algorithm initialization let's set the time provider time to reflect warmup window
+            _manualTimeProvider.SetCurrentTimeUtc(_algorithm.UtcTime);
 
             var aapl = _algorithm.AddEquity("AAPL");
             // the expiration of this option contract is before the start date of the algorithm but we should still get some data during warmup
@@ -1606,7 +1692,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds
         {
             var endTime = DateTime.UtcNow.Add(timeout);
             bool startedReceivingata = false;
-            var cancellationTokenSource = new CancellationTokenSource();
+            using var cancellationTokenSource = new CancellationTokenSource(timeout * 2);
             _algorithm.SetLocked();
             foreach (var timeSlice in _synchronizer.StreamData(cancellationTokenSource.Token))
             {

Tests/Engine/Setup/BacktestingSetupHandlerTests.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -25,6 +25,7 @@ using QuantConnect.Lean.Engine.RealTime;
 using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Tests.Engine.DataFeeds;
 using QuantConnect.Algorithm.Framework.Selection;
+using QuantConnect.Securities;
 
 namespace QuantConnect.Tests.Engine.Setup
 {
@@ -55,7 +56,7 @@ namespace QuantConnect.Tests.Engine.Setup
 
             Assert.IsFalse(setupHandler.Setup(new SetupHandlerParameters(_dataManager.UniverseSelection, _algorithm,
                 null, new BacktestNodePacket(), new TestResultHandler(),
-                null, new BacktestingRealTimeHandler(), null, TestGlobals.DataProvider)));
+                null, new BacktestingRealTimeHandler(), null, TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider)));
 
             setupHandler.DisposeSafely();
             Assert.AreEqual(1, setupHandler.Errors.Count);

Tests/Engine/Setup/BrokerageSetupHandlerTests.cs

@@ -153,7 +153,7 @@ namespace QuantConnect.Tests.Engine.Setup
             setupHandler.CreateBrokerage(job, algorithm, out factory);
 
             var result = setupHandler.Setup(new SetupHandlerParameters(_dataManager.UniverseSelection, algorithm, brokerage.Object, job, resultHandler.Object,
-                transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataProvider));
+                transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider));
 
             Assert.AreEqual(expected, result);
 
@@ -197,7 +197,7 @@ namespace QuantConnect.Tests.Engine.Setup
             setupHandler.CreateBrokerage(job, algorithm, out factory);
 
             var result = setupHandler.Setup(new SetupHandlerParameters(_dataManager.UniverseSelection, algorithm, brokerage.Object, job, resultHandler.Object,
-                transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataProvider));
+                transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider));
 
             if (result != expected)
             {
@@ -252,7 +252,7 @@ namespace QuantConnect.Tests.Engine.Setup
             setupHandler.CreateBrokerage(job, algorithm, out factory);
 
             var result = setupHandler.Setup(new SetupHandlerParameters(_dataManager.UniverseSelection, algorithm, brokerage.Object, job, resultHandler.Object,
-                transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataProvider));
+                transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider));
 
             Assert.AreEqual(expected, result);
 
@@ -297,7 +297,7 @@ namespace QuantConnect.Tests.Engine.Setup
             setupHandler.CreateBrokerage(job, algorithm, out factory);
 
             Assert.AreEqual(!fails, setupHandler.Setup(new SetupHandlerParameters(algorithm.DataManager.UniverseSelection, algorithm, brokerage.Object, job, resultHandler.Object,
-                transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataProvider)));
+                transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider)));
 
             Assert.AreEqual(10000, algorithm.Portfolio.CashBook[Currencies.USD].Amount);
             Assert.AreEqual(11, algorithm.Portfolio.CashBook[Currencies.GBP].Amount);
@@ -332,7 +332,7 @@ namespace QuantConnect.Tests.Engine.Setup
             setupHandler.CreateBrokerage(job, algorithm, out factory);
 
             Assert.IsTrue(setupHandler.Setup(new SetupHandlerParameters(_dataManager.UniverseSelection, algorithm, brokerage.Object, job, resultHandler.Object,
-                transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataProvider)));
+                transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider)));
 
             Assert.AreEqual(enforceAccountCurrency ? Currencies.USD : Currencies.EUR, algorithm.AccountCurrency);
         }
@@ -363,7 +363,7 @@ namespace QuantConnect.Tests.Engine.Setup
             setupHandler.CreateBrokerage(job, algorithm, out factory);
 
             Assert.IsFalse(setupHandler.Setup(new SetupHandlerParameters(_dataManager.UniverseSelection, algorithm, brokerage.Object, job, resultHandler.Object,
-                transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataProvider)));
+                transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider)));
 
             setupHandler.DisposeSafely();
             Assert.AreEqual(1, setupHandler.Errors.Count);
@@ -402,7 +402,7 @@ namespace QuantConnect.Tests.Engine.Setup
             setupHandler.CreateBrokerage(job, algorithm, out factory);
 
             Assert.IsTrue(setupHandler.Setup(new SetupHandlerParameters(_dataManager.UniverseSelection, algorithm, brokerage.Object, job, resultHandler.Object,
-                transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataProvider)));
+                transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider)));
 
             Security security;
             Assert.IsTrue(algorithm.Portfolio.Securities.TryGetValue(symbol, out security));
@@ -441,7 +441,7 @@ namespace QuantConnect.Tests.Engine.Setup
             setupHandler.CreateBrokerage(job, algorithm, out factory);
 
             Assert.IsTrue(setupHandler.Setup(new SetupHandlerParameters(_dataManager.UniverseSelection, algorithm, brokerage.Object, job, resultHandler.Object,
-                transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataProvider)));
+                transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider)));
 
             Security security;
             Assert.IsTrue(algorithm.Portfolio.Securities.TryGetValue(symbol, out security));
@@ -486,7 +486,7 @@ namespace QuantConnect.Tests.Engine.Setup
             setupHandler.CreateBrokerage(job, algorithm, out factory);
 
             Assert.IsTrue(setupHandler.Setup(new SetupHandlerParameters(_dataManager.UniverseSelection, algorithm, brokerage.Object, job, resultHandler.Object,
-                transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataProvider)));
+                transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider)));
 
             Assert.Greater(algorithm.UtcTime, time);
         }
@@ -536,7 +536,7 @@ namespace QuantConnect.Tests.Engine.Setup
             var dataManager = new DataManagerStub(algorithm, new MockDataFeed(), true);
 
             Assert.IsTrue(setupHandler.Setup(new SetupHandlerParameters(dataManager.UniverseSelection, algorithm, brokerage.Object, job, resultHandler.Object,
-                transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataProvider)));
+                transactionHandler.Object, realTimeHandler.Object, objectStore.Object, TestGlobals.DataCacheProvider, TestGlobals.MapFileProvider)));
 
             if (!hasCashBalance && !hasHoldings)
             {

Tests/TestGlobals.cs

@@ -16,6 +16,7 @@
 using QuantConnect.Util;
 using QuantConnect.Interfaces;
 using QuantConnect.Configuration;
+using QuantConnect.Lean.Engine.DataFeeds;
 
 namespace QuantConnect.Tests
 {
@@ -33,6 +34,8 @@ namespace QuantConnect.Tests
         public static IFactorFileProvider FactorFileProvider
             = Composer.Instance.GetExportedValueByTypeName<IFactorFileProvider>(Config.Get("factor-file-provider", "LocalDiskFactorFileProvider"));
 
+        public static IDataCacheProvider DataCacheProvider = new ZipDataCacheProvider(DataProvider);
+
         /// <summary>
         /// Initialize our providers, called by AssemblyInitialize.cs so all tests
         /// can access initialized providers