Update moq test package

- Add support for live mapping, refreshing mapfiles
- Fix future expiration functions
- Adding unit tests

Algorithm.CSharp/BasicTemplateFuturesAlgorithm.cs

@@ -131,30 +131,30 @@ namespace QuantConnect.Algorithm.CSharp
             {"Average Win", "0.00%"},
             {"Average Loss", "0.00%"},
             {"Compounding Annual Return", "-100.000%"},
-            {"Drawdown", "13.500%"},
+            {"Drawdown", "13.400%"},
             {"Expectancy", "-0.818"},
-            {"Net Profit", "-13.517%"},
-            {"Sharpe Ratio", "-98.781"},
+            {"Net Profit", "-13.418%"},
+            {"Sharpe Ratio", "-321.172"},
             {"Probabilistic Sharpe Ratio", "0%"},
             {"Loss Rate", "89%"},
             {"Win Rate", "11%"},
             {"Profit-Loss Ratio", "0.69"},
-            {"Alpha", "-1.676"},
-            {"Beta", "0.042"},
-            {"Annual Standard Deviation", "0.01"},
+            {"Alpha", "-1.208"},
+            {"Beta", "0.013"},
+            {"Annual Standard Deviation", "0.003"},
             {"Annual Variance", "0"},
-            {"Information Ratio", "-73.981"},
-            {"Tracking Error", "0.233"},
-            {"Treynor Ratio", "-23.975"},
+            {"Information Ratio", "-71.816"},
+            {"Tracking Error", "0.24"},
+            {"Treynor Ratio", "-77.951"},
             {"Total Fees", "$15207.00"},
             {"Estimated Strategy Capacity", "$8000.00"},
             {"Lowest Capacity Asset", "GC VOFJUCDY9XNH"},
-            {"Fitness Score", "0.033"},
+            {"Fitness Score", "0.031"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "-8.62"},
-            {"Return Over Maximum Drawdown", "-7.81"},
-            {"Portfolio Turnover", "302.321"},
+            {"Sortino Ratio", "-9.206"},
+            {"Return Over Maximum Drawdown", "-7.871"},
+            {"Portfolio Turnover", "302.123"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
             {"Total Insights Analysis Completed", "0"},
@@ -168,7 +168,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "35b3f4b7a225468d42ca085386a2383e"}
+            {"OrderListHash", "9e50b7d8e41033110f927658e731f4c6"}
         };
     }
 }

Algorithm.CSharp/ContinuousFutureRegressionAlgorithm.cs

@@ -0,0 +1,174 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Orders;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+using QuantConnect.Securities.Future;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Continuous Futures Regression algorithm. Asserting and showcasing the behavior of adding a continuous future
+    /// </summary>
+    public class ContinuousFutureRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        private List<SymbolChangedEvent> _mappings = new();
+        private Future _continuousContract;
+        private DateTime _lastDateLog;
+
+        /// <summary>
+        /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
+        /// </summary>
+        public override void Initialize()
+        {
+            SetStartDate(2013, 7, 1);
+            SetEndDate(2014, 1, 1);
+
+            _continuousContract = AddFuture(Futures.Indices.SP500EMini,
+                dataNormalizationMode: DataNormalizationMode.BackwardsRatio,
+                dataMappingMode: DataMappingMode.FirstDayMonth,
+                contractDepthOffset: 0
+            );
+        }
+
+        /// <summary>
+        /// OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
+        /// </summary>
+        /// <param name="data">Slice object keyed by symbol containing the stock data</param>
+        public override void OnData(Slice data)
+        {
+            if (data.Keys.Count != 1)
+            {
+                throw new Exception($"We are getting data for more than one symbols! {string.Join(",", data.Keys.Select(symbol => symbol))}");
+            }
+
+            foreach (var changedEvent in data.SymbolChangedEvents.Values)
+            {
+                if (changedEvent.Symbol == _continuousContract.Symbol)
+                {
+                    _mappings.Add(changedEvent);
+                    Log($"SymbolChanged event: {changedEvent}");
+                }
+            }
+
+            if (_lastDateLog.Month != Time.Month)
+            {
+                _lastDateLog = Time;
+
+                Log($"{Time}- {Securities[_continuousContract.Symbol].GetLastData()}");
+                if (Portfolio.Invested)
+                {
+                    Liquidate();
+                }
+                else if(_continuousContract.HasData)
+                {
+                    // This works because we set this contract as tradable, even if it's a canonical security
+                    Buy(_continuousContract.Symbol, 1);
+                }
+
+                if(Time.Month == 1 && Time.Year == 2013)
+                {
+                    var response = History(new[] { _continuousContract.Symbol }, 60 * 24 * 90);
+                    if (!response.Any())
+                    {
+                        throw new Exception("Unexpected empty history response");
+                    }
+                }
+            }
+        }
+
+        public override void OnOrderEvent(OrderEvent orderEvent)
+        {
+            if (orderEvent.Status == OrderStatus.Filled)
+            {
+                Log($"{orderEvent}");
+            }
+        }
+
+        public override void OnEndOfAlgorithm()
+        {
+            var expectedMappingCounts = 2;
+            if (_mappings.Count != expectedMappingCounts)
+            {
+                throw new Exception($"Unexpected symbol changed events: {_mappings.Count}, was expecting {expectedMappingCounts}");
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public Language[] Languages { get; } = { Language.CSharp, Language.Python };
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Trades", "2"},
+            {"Average Win", "1.13%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "2.249%"},
+            {"Drawdown", "1.600%"},
+            {"Expectancy", "0"},
+            {"Net Profit", "1.134%"},
+            {"Sharpe Ratio", "0.727"},
+            {"Probabilistic Sharpe Ratio", "38.512%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "100%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "-0.006"},
+            {"Beta", "0.099"},
+            {"Annual Standard Deviation", "0.022"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "-2.728"},
+            {"Tracking Error", "0.076"},
+            {"Treynor Ratio", "0.159"},
+            {"Total Fees", "$3.70"},
+            {"Estimated Strategy Capacity", "$810000000.00"},
+            {"Lowest Capacity Asset", "ES 1S1"},
+            {"Fitness Score", "0.007"},
+            {"Kelly Criterion Estimate", "0"},
+            {"Kelly Criterion Probability Value", "0"},
+            {"Sortino Ratio", "0.495"},
+            {"Return Over Maximum Drawdown", "1.776"},
+            {"Portfolio Turnover", "0.011"},
+            {"Total Insights Generated", "0"},
+            {"Total Insights Closed", "0"},
+            {"Total Insights Analysis Completed", "0"},
+            {"Long Insight Count", "0"},
+            {"Short Insight Count", "0"},
+            {"Long/Short Ratio", "100%"},
+            {"Estimated Monthly Alpha Value", "$0"},
+            {"Total Accumulated Estimated Alpha Value", "$0"},
+            {"Mean Population Estimated Insight Value", "$0"},
+            {"Mean Population Direction", "0%"},
+            {"Mean Population Magnitude", "0%"},
+            {"Rolling Averaged Population Direction", "0%"},
+            {"Rolling Averaged Population Magnitude", "0%"},
+            {"OrderListHash", "b622c3dcec2c38a81e336d59a3e4c92d"}
+        };
+    }
+}

Algorithm.CSharp/DelistedFutureLiquidateRegressionAlgorithm.cs

@@ -97,7 +97,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Average Win", "1.63%"},
             {"Average Loss", "0%"},
             {"Compounding Annual Return", "7.292%"},
-            {"Drawdown", "1.300%"},
+            {"Drawdown", "1.500%"},
             {"Expectancy", "0"},
             {"Net Profit", "1.634%"},
             {"Sharpe Ratio", "2.351"},

Algorithm.Python/ContinuousFutureRegressionAlgorithm.py

@@ -0,0 +1,71 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+### <summary>
+### Continuous Futures Regression algorithm. Asserting and showcasing the behavior of adding a continuous future
+### </summary>
+class ContinuousFutureRegressionAlgorithm(QCAlgorithm):
+    '''Basic template algorithm simply initializes the date range and cash'''
+
+    def Initialize(self):
+        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
+
+        self.SetStartDate(2013, 7, 1)
+        self.SetEndDate(2014, 1, 1)
+
+        self._mappings = []
+        self._lastDateLog = -1
+        self._continuousContract = self.AddFuture(Futures.Indices.SP500EMini,
+                                                  dataNormalizationMode = DataNormalizationMode.BackwardsRatio,
+                                                  dataMappingMode = DataMappingMode.FirstDayMonth,
+                                                  contractDepthOffset= 0)
+
+    def OnData(self, data):
+        '''OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
+
+        Arguments:
+            data: Slice object keyed by symbol containing the stock data
+        '''
+        if len(data.Keys) != 1:
+            raise ValueError(f"We are getting data for more than one symbols! {','.join(data.Keys)}")
+
+        for changedEvent in data.SymbolChangedEvents.Values:
+            if changedEvent.Symbol == self._continuousContract.Symbol:
+                self._mappings.append(changedEvent)
+                self.Log(f"SymbolChanged event: {changedEvent}")
+
+        if self._lastDateLog != self.Time.month:
+            self._lastDateLog = self.Time.month
+
+            self.Log(f"{self.Time}- {self._continuousContract.GetLastData()}")
+            if self.Portfolio.Invested:
+                self.Liquidate()
+            elif self._continuousContract.HasData:
+                # This works because we set this contract as tradable, even if it's a canonical security
+                self.Buy(self._continuousContract.Symbol, 1)
+
+            if self.Time.month == 1 and self.Time.year == 2013:
+                response = self.History( [ self._continuousContract.Symbol ], 60 * 24 * 90)
+                if response.empty:
+                    raise ValueError("Unexpected empty history response")
+
+    def OnOrderEvent(self, orderEvent):
+        if orderEvent.Status == OrderStatus.Filled:
+            self.Debug("Purchased Stock: {0}".format(orderEvent.Symbol))
+
+    def OnEndOfAlgorithm(self):
+        expectedMappingCounts = 2
+        if len(self._mappings) != expectedMappingCounts:
+            raise ValueError(f"Unexpected symbol changed events: {self._mappings.count()}, was expecting {expectedMappingCounts}")

Algorithm/QCAlgorithm.History.cs

@@ -475,7 +475,7 @@ namespace QuantConnect.Algorithm
         /// <returns>Securities historical data</returns>
         public IEnumerable<BaseData> GetLastKnownPrices(Symbol symbol)
         {
-            if (symbol.IsCanonical() || HistoryProvider == null)
+            if (!HistoryRequestValid(symbol) || HistoryProvider == null)
             {
                 return Enumerable.Empty<BaseData>();
             }
@@ -568,7 +568,7 @@ namespace QuantConnect.Algorithm
         {
             var sentMessage = false;
             // filter out any universe securities that may have made it this far
-            var filteredRequests = requests.Where(hr => !UniverseManager.ContainsKey(hr.Symbol)).ToList();
+            var filteredRequests = requests.Where(hr => HistoryRequestValid(hr.Symbol)).ToList();
             for (var i = 0; i < filteredRequests.Count; i++)
             {
                 var request  = filteredRequests[i];
@@ -604,7 +604,7 @@ namespace QuantConnect.Algorithm
         /// </summary>
         private IEnumerable<HistoryRequest> CreateDateRangeHistoryRequests(IEnumerable<Symbol> symbols, DateTime startAlgoTz, DateTime endAlgoTz, Resolution? resolution = null, bool? fillForward = null, bool? extendedMarket = null)
         {
-            return symbols.Where(x => !x.IsCanonical()).SelectMany(x =>
+            return symbols.Where(HistoryRequestValid).SelectMany(x =>
             {
                 var requests = new List<HistoryRequest>();
 
@@ -629,7 +629,7 @@ namespace QuantConnect.Algorithm
         /// </summary>
         private IEnumerable<HistoryRequest> CreateBarCountHistoryRequests(IEnumerable<Symbol> symbols, int periods, Resolution? resolution = null)
         {
-            return symbols.Where(x => !x.IsCanonical()).SelectMany(x =>
+            return symbols.Where(HistoryRequestValid).SelectMany(x =>
             {
                 var res = GetResolution(x, resolution);
                 var exchange = GetExchangeHours(x);
@@ -778,5 +778,14 @@ namespace QuantConnect.Algorithm
                 return resolution ?? UniverseSettings.Resolution;
             }
         }
+
+        /// <summary>
+        /// Validate a symbol for a history request.
+        /// Universe and canonical symbols are only valid for future security types
+        /// </summary>
+        private bool HistoryRequestValid(Symbol symbol)
+        {
+            return symbol.SecurityType == SecurityType.Future || !UniverseManager.ContainsKey(symbol) && !symbol.IsCanonical();
+        }
     }
 }

Algorithm/QCAlgorithm.cs

@@ -45,6 +45,7 @@ using QuantConnect.Algorithm.Framework.Portfolio;
 using QuantConnect.Algorithm.Framework.Risk;
 using QuantConnect.Algorithm.Framework.Selection;
 using QuantConnect.Algorithm.Selection;
+using QuantConnect.Data.Market;
 using QuantConnect.Storage;
 using Index = QuantConnect.Securities.Index.Index;
 
@@ -1403,9 +1404,12 @@ namespace QuantConnect.Algorithm
         /// <param name="resolution">Resolution of the Data Required</param>
         /// <param name="fillDataForward">When no data available on a tradebar, return the last data that was generated</param>
         /// <param name="extendedMarketHours">Show the after market data as well</param>
-        public Security AddSecurity(SecurityType securityType, string ticker, Resolution? resolution = null, bool fillDataForward = true, bool extendedMarketHours = false)
+        /// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
+        /// <param name="dataNormalizationMode">The price scaling mode to use for the security</param>
+        public Security AddSecurity(SecurityType securityType, string ticker, Resolution? resolution = null, bool fillDataForward = true, bool extendedMarketHours = false,
+            DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null)
         {
-            return AddSecurity(securityType, ticker, resolution, fillDataForward, Security.NullLeverage, extendedMarketHours);
+            return AddSecurity(securityType, ticker, resolution, fillDataForward, Security.NullLeverage, extendedMarketHours, dataMappingMode, dataNormalizationMode);
         }
 
         /// <summary>
@@ -1417,10 +1421,13 @@ namespace QuantConnect.Algorithm
         /// <param name="fillDataForward">When no data available on a tradebar, return the last data that was generated</param>
         /// <param name="leverage">Custom leverage per security</param>
         /// <param name="extendedMarketHours">Extended market hours</param>
+        /// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
+        /// <param name="dataNormalizationMode">The price scaling mode to use for the security</param>
         /// <remarks> AddSecurity(SecurityType securityType, Symbol symbol, Resolution resolution, bool fillDataForward, decimal leverage, bool extendedMarketHours)</remarks>
-        public Security AddSecurity(SecurityType securityType, string ticker, Resolution? resolution, bool fillDataForward, decimal leverage, bool extendedMarketHours)
+        public Security AddSecurity(SecurityType securityType, string ticker, Resolution? resolution, bool fillDataForward, decimal leverage, bool extendedMarketHours,
+            DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null)
         {
-            return AddSecurity(securityType, ticker, resolution, null, fillDataForward, leverage, extendedMarketHours);
+            return AddSecurity(securityType, ticker, resolution, null, fillDataForward, leverage, extendedMarketHours, dataMappingMode, dataNormalizationMode);
         }
 
         /// <summary>
@@ -1433,7 +1440,10 @@ namespace QuantConnect.Algorithm
         /// <param name="fillDataForward">If true, returns the last available data even if none in that timeslice.</param>
         /// <param name="leverage">leverage for this security</param>
         /// <param name="extendedMarketHours">ExtendedMarketHours send in data from 4am - 8pm, not used for FOREX</param>
-        public Security AddSecurity(SecurityType securityType, string ticker, Resolution? resolution, string market, bool fillDataForward, decimal leverage, bool extendedMarketHours)
+        /// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
+        /// <param name="dataNormalizationMode">The price scaling mode to use for the security</param>
+        public Security AddSecurity(SecurityType securityType, string ticker, Resolution? resolution, string market, bool fillDataForward, decimal leverage, bool extendedMarketHours,
+            DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null)
         {
             // if AddSecurity method is called to add an option or a future, we delegate a call to respective methods
             if (securityType == SecurityType.Option)
@@ -1464,7 +1474,7 @@ namespace QuantConnect.Algorithm
                     symbol = QuantConnect.Symbol.Create(ticker, securityType, market);
                 }
 
-                return AddSecurity(symbol, resolution, fillDataForward, leverage, extendedMarketHours);
+                return AddSecurity(symbol, resolution, fillDataForward, leverage, extendedMarketHours, dataMappingMode, dataNormalizationMode);
             }
             catch (Exception err)
             {
@@ -1481,8 +1491,13 @@ namespace QuantConnect.Algorithm
         /// <param name="fillDataForward">If true, returns the last available data even if none in that timeslice.</param>
         /// <param name="leverage">leverage for this security</param>
         /// <param name="extendedMarketHours">ExtendedMarketHours send in data from 4am - 8pm, not used for FOREX</param>
+        /// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
+        /// <param name="dataNormalizationMode">The price scaling mode to use for the security</param>
+        /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
+        /// For example, 0 (default) will use the front month, 1 will use the back month contract</param>
         /// <returns>The new Security that was added to the algorithm</returns>
-        public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool fillDataForward = true, decimal leverage = Security.NullLeverage, bool extendedMarketHours = false)
+        public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool fillDataForward = true, decimal leverage = Security.NullLeverage, bool extendedMarketHours = false,
+            DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null, int contractDepthOffset = 0)
         {
             var isCanonical = symbol.IsCanonical();
 
@@ -1516,6 +1531,23 @@ namespace QuantConnect.Algorithm
                     }
                     else
                     {
+                        // allow users to specify negative numbers, we get the abs of it
+                        var contractOffset = (uint)Math.Abs(contractDepthOffset);
+                        security.IsTradable = true;
+
+                        var continuousConfigs = SubscriptionManager.SubscriptionDataConfigService.Add(symbol,
+                            resolution,
+                            fillDataForward,
+                            extendedMarketHours,
+                            isFilteredSubscription: false,
+                            subscriptionDataTypes: SubscriptionManager.LookupSubscriptionConfigDataTypes(SecurityType.Future,
+                                GetResolution(symbol, resolution), isCanonical:false),
+                            dataNormalizationMode: dataNormalizationMode ?? UniverseSettings.DataNormalizationMode,
+                            dataMappingMode: dataMappingMode ?? UniverseSettings.DataMappingMode,
+                            contractDepthOffset: contractOffset
+                            );
+                        AddToUserDefinedUniverse(security, continuousConfigs);
+
                         universe = new FuturesChainUniverse((Future)security, settings);
                     }
 
@@ -1616,8 +1648,14 @@ namespace QuantConnect.Algorithm
         /// <param name="market">The futures market, <seealso cref="Market"/>. Default is value null and looked up using BrokerageModel.DefaultMarkets in <see cref="AddSecurity{T}"/></param>
         /// <param name="fillDataForward">If true, returns the last available data even if none in that timeslice. Default is <value>true</value></param>
         /// <param name="leverage">The requested leverage for this equity. Default is set by <see cref="SecurityInitializer"/></param>
+        /// <param name="dataMappingMode">The contract mapping mode to use for the continuous future contract</param>
+        /// <param name="dataNormalizationMode">The price scaling mode to use for the continuous future contract</param>
+        /// <param name="contractDepthOffset">The continuous future contract desired offset from the current front month.
+        /// For example, 0 (default) will use the front month, 1 will use the back month contract</param>
         /// <returns>The new <see cref="Future"/> security</returns>
-        public Future AddFuture(string ticker, Resolution? resolution = null, string market = null, bool fillDataForward = true, decimal leverage = Security.NullLeverage)
+        public Future AddFuture(string ticker, Resolution? resolution = null, string market = null,
+            bool fillDataForward = true, decimal leverage = Security.NullLeverage,
+            DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null, int contractDepthOffset = 0)
         {
             if (market == null)
             {
@@ -1637,7 +1675,8 @@ namespace QuantConnect.Algorithm
                 canonicalSymbol = QuantConnect.Symbol.Create(ticker, SecurityType.Future, market, alias);
             }
 
-            return (Future)AddSecurity(canonicalSymbol, resolution, fillDataForward, leverage);
+            return (Future)AddSecurity(canonicalSymbol, resolution, fillDataForward, leverage, dataMappingMode: dataMappingMode,
+                dataNormalizationMode: dataNormalizationMode, contractDepthOffset: contractDepthOffset);
         }
 
         /// <summary>
@@ -2261,7 +2300,8 @@ namespace QuantConnect.Algorithm
         /// <summary>
         /// Creates and adds a new <see cref="Security"/> to the algorithm
         /// </summary>
-        private T AddSecurity<T>(SecurityType securityType, string ticker, Resolution? resolution, string market, bool fillDataForward, decimal leverage, bool extendedMarketHours)
+        private T AddSecurity<T>(SecurityType securityType, string ticker, Resolution? resolution, string market, bool fillDataForward, decimal leverage, bool extendedMarketHours,
+            DataMappingMode? mappingMode = null, DataNormalizationMode? normalizationMode = null)
             where T : Security
         {
             if (market == null)
@@ -2280,7 +2320,9 @@ namespace QuantConnect.Algorithm
                 symbol = QuantConnect.Symbol.Create(ticker, securityType, market);
             }
 
-            var configs = SubscriptionManager.SubscriptionDataConfigService.Add(symbol, resolution, fillDataForward, extendedMarketHours, dataNormalizationMode: UniverseSettings.DataNormalizationMode);
+            var configs = SubscriptionManager.SubscriptionDataConfigService.Add(symbol, resolution, fillDataForward, extendedMarketHours,
+                dataNormalizationMode: normalizationMode ?? UniverseSettings.DataNormalizationMode,
+                dataMappingMode: mappingMode ?? UniverseSettings.DataMappingMode);
             var security = Securities.CreateSecurity(symbol, configs, leverage);
 
             return (T) AddToUserDefinedUniverse(security, configs);

AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs

@@ -440,8 +440,11 @@ namespace QuantConnect.AlgorithmFactory.Python.Wrappers
         /// <param name="fillDataForward">If true, returns the last available data even if none in that timeslice.</param>
         /// <param name="leverage">leverage for this security</param>
         /// <param name="extendedMarketHours">ExtendedMarketHours send in data from 4am - 8pm, not used for FOREX</param>
-        public Security AddSecurity(SecurityType securityType, string symbol, Resolution? resolution, string market, bool fillDataForward, decimal leverage, bool extendedMarketHours)
-            => _baseAlgorithm.AddSecurity(securityType, symbol, resolution, market, fillDataForward, leverage, extendedMarketHours);
+        /// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
+        /// <param name="dataNormalizationMode">The price scaling mode to use for the security</param>
+        public Security AddSecurity(SecurityType securityType, string symbol, Resolution? resolution, string market, bool fillDataForward, decimal leverage, bool extendedMarketHours,
+            DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null)
+            => _baseAlgorithm.AddSecurity(securityType, symbol, resolution, market, fillDataForward, leverage, extendedMarketHours, dataMappingMode, dataNormalizationMode);
 
         /// <summary>
         /// Creates and adds a new single <see cref="Future"/> contract to the algorithm

Brokerages/InteractiveBrokers/InteractiveBrokersSymbolMapper.cs

@@ -25,6 +25,7 @@ using System.IO;
 using System.Linq;
 using System.Text.RegularExpressions;
 using IBApi;
+using QuantConnect.Data.Auxiliary;
 using QuantConnect.Logging;
 using QuantConnect.Securities;
 
@@ -283,7 +284,7 @@ namespace QuantConnect.Brokerages.InteractiveBrokers
             var ticker = symbol.ID.Symbol;
             if (symbol.ID.SecurityType == SecurityType.Equity)
             {
-                var mapFile = _mapFileProvider.Get(symbol.ID.Market).ResolveMapFile(symbol.ID.Symbol, symbol.ID.Date);
+                var mapFile = _mapFileProvider.Get(CorporateActionsKey.Create(symbol)).ResolveMapFile(symbol);
                 ticker = mapFile.GetMappedSymbol(DateTime.UtcNow, symbol.Value);
             }
 

Common/Data/Auxiliary/CorporateActionsKey.cs

@@ -0,0 +1,97 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+namespace QuantConnect.Data.Auxiliary
+{
+    /// <summary>
+    /// Unique definition key for a collection of corporate actions for a Market and SecurityType
+    /// </summary>
+    public class CorporateActionsKey
+    {
+        /// <summary>
+        /// USA equities market corporate actions key definition
+        /// </summary>
+        public static CorporateActionsKey EquityUsa { get; } = new (QuantConnect.Market.USA, SecurityType.Equity);
+
+        /// <summary>
+        /// The market associated with these corporate actions
+        /// </summary>
+        public string Market { get; }
+
+        /// <summary>
+        /// The associated security type
+        /// </summary>
+        public SecurityType SecurityType { get; }
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        public CorporateActionsKey(string market, SecurityType securityType)
+        {
+            Market = market;
+            SecurityType = securityType;
+        }
+
+        /// <summary>
+        /// Serves as a hash function for a particular type.
+        /// </summary>
+        public override int GetHashCode()
+        {
+            unchecked
+            {
+                var hashCode = Market.GetHashCode();
+                return (hashCode*397) ^ SecurityType.GetHashCode();
+            }
+        }
+
+        /// <summary>
+        /// Determines whether the specified <see cref="T:System.Object"/> is equal to the current <see cref="T:System.Object"/>.
+        /// </summary>
+        /// <returns>
+        /// true if the specified object  is equal to the current object; otherwise, false.
+        /// </returns>
+        /// <param name="obj">The object to compare with the current object. </param><filterpriority>2</filterpriority>
+        public override bool Equals(object? obj)
+        {
+            if (ReferenceEquals(null, obj)) return false;
+            if (obj.GetType() != GetType()) return false;
+
+            var other = (CorporateActionsKey)obj;
+
+            return other.Market == Market
+                && other.SecurityType == SecurityType;
+        }
+
+        public override string ToString()
+        {
+            return $"{Market}:{SecurityType}";
+        }
+
+        /// <summary>
+        /// Helper method to create a new instance from a Symbol
+        /// </summary>
+        public static CorporateActionsKey Create(Symbol symbol) => Create(symbol.HasUnderlying ? symbol.Underlying.ID : symbol.ID);
+
+        /// <summary>
+        /// Helper method to create a new instance from a SecurityIdentifier
+        /// </summary>
+        public static CorporateActionsKey Create(SecurityIdentifier securityIdentifier)
+        {
+            securityIdentifier = securityIdentifier.HasUnderlying ? securityIdentifier.Underlying : securityIdentifier;
+            return new CorporateActionsKey(securityIdentifier.Market, securityIdentifier.SecurityType);
+        }
+    }
+}

Common/Data/Auxiliary/FactorFileZipHelper.cs

@@ -25,10 +25,18 @@ namespace QuantConnect.Data.Auxiliary
     /// </summary>
     public static class FactorFileZipHelper
     {
+        /// <summary>
+        /// Gets the factor file zip filename for the specified date
+        /// </summary>
+        public static string GetFactorFileZipFileName(string market, DateTime date, SecurityType securityType)
+        {
+            return Path.Combine(Globals.DataFolder, $"{securityType.SecurityTypeToLower()}/{market}/factor_files/factor_files_{date:yyyyMMdd}.zip");
+        }
+
         /// <summary>
         /// Reads the zip bytes as text and parses as FactorFileRows to create FactorFiles
         /// </summary>
-        public static IEnumerable<KeyValuePair<Symbol, FactorFile>> ReadFactorFileZip(Stream file, MapFileResolver mapFileResolver, string market)
+        public static IEnumerable<KeyValuePair<Symbol, FactorFile>> ReadFactorFileZip(Stream file, MapFileResolver mapFileResolver, string market, SecurityType securityType)
         {
             if (file == null || file.Length == 0)
             {
@@ -42,9 +50,7 @@ namespace QuantConnect.Data.Auxiliary
                     let factorFile = SafeRead(filename, lines)
                     let mapFile = mapFileResolver.GetByPermtick(factorFile.Permtick)
                     where mapFile != null
-                    let sid = SecurityIdentifier.GenerateEquity(mapFile.FirstDate, mapFile.FirstTicker, market)
-                    let symbol = new Symbol(sid, mapFile.Permtick)
-                    select new KeyValuePair<Symbol, FactorFile>(symbol, factorFile)
+                    select new KeyValuePair<Symbol, FactorFile>(GetSymbol(mapFile, market, securityType), factorFile)
                 );
 
             return keyValuePairs;
@@ -68,5 +74,22 @@ namespace QuantConnect.Data.Auxiliary
                 return new FactorFile(permtick, Enumerable.Empty<FactorFileRow>());
             }
         }
+
+        private static Symbol GetSymbol(MapFile mapFile, string market, SecurityType securityType)
+        {
+            SecurityIdentifier sid;
+            switch (securityType)
+            {
+                case SecurityType.Equity:
+                    sid = SecurityIdentifier.GenerateEquity(mapFile.FirstDate, mapFile.FirstTicker, market);
+                    break;
+                case SecurityType.Future:
+                    sid = SecurityIdentifier.GenerateFuture(SecurityIdentifier.DefaultDate, mapFile.Permtick, market);
+                    break;
+                default:
+                    throw new ArgumentOutOfRangeException(nameof(securityType), securityType, null);
+            }
+            return new Symbol(sid, mapFile.Permtick);
+        }
     }
 }

Common/Data/Auxiliary/LocalDiskFactorFileProvider.cs

@@ -14,10 +14,10 @@
  *
 */
 
-using System.Collections.Concurrent;
 using System.IO;
-using QuantConnect.Interfaces;
 using QuantConnect.Util;
+using QuantConnect.Interfaces;
+using System.Collections.Concurrent;
 
 namespace QuantConnect.Data.Auxiliary
 {
@@ -63,32 +63,30 @@ namespace QuantConnect.Data.Auxiliary
                 return factorFile;
             }
 
-            var market = symbol.ID.Market;
-
             // we first need to resolve the map file to get a permtick, that's how the factor files are stored
-            var mapFileResolver = _mapFileProvider.Get(market);
+            var mapFileResolver = _mapFileProvider.Get(CorporateActionsKey.Create(symbol));
             if (mapFileResolver == null)
             {
-                return GetFactorFile(symbol, symbol.Value, market);
+                return GetFactorFile(symbol, symbol.Value);
             }
 
-            var mapFile = mapFileResolver.ResolveMapFile(symbol.ID.Symbol, symbol.ID.Date);
+            var mapFile = mapFileResolver.ResolveMapFile(symbol);
             if (mapFile.IsNullOrEmpty())
             {
-                return GetFactorFile(symbol, symbol.Value, market);
+                return GetFactorFile(symbol, symbol.Value);
             }
 
-            return GetFactorFile(symbol, mapFile.Permtick, market);
+            return GetFactorFile(symbol, mapFile.Permtick);
         }
 
         /// <summary>
         /// Checks that the factor file exists on disk, and if it does, loads it into memory
         /// </summary>
-        private FactorFile GetFactorFile(Symbol symbol, string permtick, string market)
+        private FactorFile GetFactorFile(Symbol symbol, string permtick)
         {
             FactorFile factorFile = null;
 
-            var path = Path.Combine(Globals.CacheDataFolder, "equity", market, "factor_files", permtick.ToLowerInvariant() + ".csv");
+            var path = Path.Combine(Globals.CacheDataFolder, symbol.SecurityType.SecurityTypeToLower(), symbol.ID.Market, "factor_files", permtick.ToLowerInvariant() + ".csv");
 
             var factorFileStream = _dataProvider.Fetch(path);
             if (factorFileStream != null)

Common/Data/Auxiliary/LocalDiskMapFileProvider.cs

@@ -14,11 +14,11 @@
  *
 */
 
-using System.Collections.Concurrent;
 using System.IO;
 using System.Threading;
-using QuantConnect.Interfaces;
 using QuantConnect.Logging;
+using QuantConnect.Interfaces;
+using System.Collections.Concurrent;
 
 namespace QuantConnect.Data.Auxiliary
 {
@@ -29,7 +29,7 @@ namespace QuantConnect.Data.Auxiliary
     public class LocalDiskMapFileProvider : IMapFileProvider
     {
         private static int _wroteTraceStatement;
-        private readonly ConcurrentDictionary<string, MapFileResolver> _cache;
+        private readonly ConcurrentDictionary<CorporateActionsKey, MapFileResolver> _cache;
         private IDataProvider _dataProvider;
 
         /// <summary>
@@ -37,7 +37,7 @@ namespace QuantConnect.Data.Auxiliary
         /// </summary>
         public LocalDiskMapFileProvider()
         {
-            _cache = new ConcurrentDictionary<string, MapFileResolver>();
+            _cache = new ConcurrentDictionary<CorporateActionsKey, MapFileResolver>();
         }
 
         /// <summary>
@@ -53,20 +53,19 @@ namespace QuantConnect.Data.Auxiliary
         /// Gets a <see cref="MapFileResolver"/> representing all the map
         /// files for the specified market
         /// </summary>
-        /// <param name="market">The equity market, for example, 'usa'</param>
+        /// <param name="corporateActionsKey">Key used to fetch a map file resolver. Specifying market and security type</param>
         /// <returns>A <see cref="MapFileRow"/> containing all map files for the specified market</returns>
-        public MapFileResolver Get(string market)
+        public MapFileResolver Get(CorporateActionsKey corporateActionsKey)
         {
-            // TODO: Consider using DataProvider to load in the files from disk to unify data fetching behavior
-            // Reference LocalDiskFactorFile, LocalZipFactorFile, and LocalZipMapFile providers for examples.
-
-            market = market.ToLowerInvariant();
-            return _cache.GetOrAdd(market, GetMapFileResolver);
+            return _cache.GetOrAdd(corporateActionsKey, GetMapFileResolver);
         }
 
-        private static MapFileResolver GetMapFileResolver(string market)
+        private MapFileResolver GetMapFileResolver(CorporateActionsKey key)
         {
-            var mapFileDirectory = Path.Combine(Globals.CacheDataFolder, "equity", market, "map_files");
+            var securityType = key.SecurityType;
+            var market = key.Market;
+
+            var mapFileDirectory =  MapFile.GetMapFilePath(market, securityType);
             if (!Directory.Exists(mapFileDirectory))
             {
                 // only write this message once per application instance
@@ -78,7 +77,7 @@ namespace QuantConnect.Data.Auxiliary
                 }
                 return MapFileResolver.Empty;
             }
-            return new MapFileResolver(MapFile.GetMapFiles(mapFileDirectory, market));
+            return new MapFileResolver(MapFile.GetMapFiles(mapFileDirectory, market, securityType, _dataProvider));
         }
     }
 }

Common/Data/Auxiliary/LocalZipFactorFileProvider.cs

@@ -14,7 +14,6 @@
 */
 
 using System;
-using System.IO;
 using QuantConnect.Util;
 using QuantConnect.Logging;
 using System.Threading.Tasks;
@@ -31,7 +30,7 @@ namespace QuantConnect.Data.Auxiliary
         private readonly object _lock;
         private IDataProvider _dataProvider;
         private IMapFileProvider _mapFileProvider;
-        private Dictionary<string, bool> _seededMarket;
+        private Dictionary<CorporateActionsKey, bool> _seededMarket;
         private readonly Dictionary<Symbol, FactorFile> _factorFiles;
 
         /// <summary>
@@ -46,7 +45,7 @@ namespace QuantConnect.Data.Auxiliary
         public LocalZipFactorFileProvider()
         {
             _factorFiles = new Dictionary<Symbol, FactorFile>();
-            _seededMarket = new Dictionary<string, bool>();
+            _seededMarket = new Dictionary<CorporateActionsKey, bool>();
             _lock = new object();
         }
 
@@ -75,13 +74,13 @@ namespace QuantConnect.Data.Auxiliary
         /// <returns>The resolved factor file, or null if not found</returns>
         public FactorFile Get(Symbol symbol)
         {
-            var market = symbol.ID.Market.ToLowerInvariant();
+            var key = CorporateActionsKey.Create(symbol);
             lock (_lock)
             {
-                if (!_seededMarket.ContainsKey(market))
+                if (!_seededMarket.ContainsKey(key))
                 {
-                    HydrateFactorFileFromLatestZip(market);
-                    _seededMarket[market] = true;
+                    HydrateFactorFileFromLatestZip(key);
+                    _seededMarket[key] = true;
                 }
 
                 FactorFile factorFile;
@@ -104,19 +103,15 @@ namespace QuantConnect.Data.Auxiliary
             lock (_lock)
             {
                 // we clear the seeded markets so they are reloaded
-                _seededMarket = new Dictionary<string, bool>();
+                _seededMarket = new Dictionary<CorporateActionsKey, bool>();
             }
             _ = Task.Delay(CacheRefreshPeriod).ContinueWith(_ => StartExpirationTask());
         }
 
         /// Hydrate the <see cref="_factorFiles"/> from the latest zipped factor file on disk
-        private void HydrateFactorFileFromLatestZip(string market)
+        private void HydrateFactorFileFromLatestZip(CorporateActionsKey key)
         {
-            if (market != QuantConnect.Market.USA.ToLowerInvariant())
-            {
-                // don't explode for other markets which request factor files and we don't have
-                return;
-            }
+            var market = key.Market;
             // start the search with yesterday, today's file will be available tomorrow
             var todayNewYork = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork).Date;
             var date = todayNewYork.AddDays(-1);
@@ -125,8 +120,7 @@ namespace QuantConnect.Data.Auxiliary
 
             do
             {
-                var zipFileName = $"equity/{market}/factor_files/factor_files_{date:yyyyMMdd}.zip";
-                var factorFilePath = Path.Combine(Globals.DataFolder, zipFileName);
+                var factorFilePath = FactorFileZipHelper.GetFactorFileZipFileName(market, date, key.SecurityType);
 
                 // Fetch a stream for our zip from our data provider
                 var stream = _dataProvider.Fetch(factorFilePath);
@@ -134,8 +128,8 @@ namespace QuantConnect.Data.Auxiliary
                 // If the file was found we can read the file
                 if (stream != null)
                 {
-                    var mapFileResolver = _mapFileProvider.Get(market);
-                    foreach (var keyValuePair in FactorFileZipHelper.ReadFactorFileZip(stream, mapFileResolver, market))
+                    var mapFileResolver = _mapFileProvider.Get(key);
+                    foreach (var keyValuePair in FactorFileZipHelper.ReadFactorFileZip(stream, mapFileResolver, market, key.SecurityType))
                     {
                         // we merge with existing, this will allow to hold multiple markets
                         _factorFiles[keyValuePair.Key] = keyValuePair.Value;

Common/Data/Auxiliary/LocalZipMapFileProvider.cs

@@ -14,7 +14,6 @@
 */
 
 using System;
-using System.IO;
 using QuantConnect.Util;
 using QuantConnect.Logging;
 using System.Threading.Tasks;
@@ -28,7 +27,7 @@ namespace QuantConnect.Data.Auxiliary
     /// </summary>
     public class LocalZipMapFileProvider : IMapFileProvider
     {
-        private Dictionary<string, MapFileResolver> _cache;
+        private Dictionary<CorporateActionsKey, MapFileResolver> _cache;
         private IDataProvider _dataProvider;
         private object _lock;
 
@@ -44,7 +43,7 @@ namespace QuantConnect.Data.Auxiliary
         public LocalZipMapFileProvider()
         {
             _lock = new object();
-            _cache = new Dictionary<string, MapFileResolver>();
+            _cache = new Dictionary<CorporateActionsKey, MapFileResolver>();
         }
 
         /// <summary>
@@ -65,19 +64,18 @@ namespace QuantConnect.Data.Auxiliary
         /// <summary>
         /// Gets a <see cref="MapFileResolver"/> representing all the map files for the specified market
         /// </summary>
-        /// <param name="market">The equity market, for example, 'usa'</param>
+        /// <param name="corporateActionsKey">Key used to fetch a map file resolver. Specifying market and security type</param>
         /// <returns>A <see cref="MapFileResolver"/> containing all map files for the specified market</returns>
-        public MapFileResolver Get(string market)
+        public MapFileResolver Get(CorporateActionsKey corporateActionsKey)
         {
-            market = market.ToLowerInvariant();
             MapFileResolver result;
             // we use a lock so that only 1 thread loads the map file resolver while the rest wait
             // else we could have multiple threads loading the map file resolver at the same time!
             lock (_lock)
             {
-                if (!_cache.TryGetValue(market, out result))
+                if (!_cache.TryGetValue(corporateActionsKey, out result))
                 {
-                    _cache[market] = result = GetMapFileResolver(market);
+                    _cache[corporateActionsKey] = result = GetMapFileResolver(corporateActionsKey);
                 }
             }
             return result;
@@ -91,18 +89,14 @@ namespace QuantConnect.Data.Auxiliary
             lock (_lock)
             {
                 // we clear the seeded markets so they are reloaded
-                _cache = new Dictionary<string, MapFileResolver>();
+                _cache = new Dictionary<CorporateActionsKey, MapFileResolver>();
             }
             _ = Task.Delay(CacheRefreshPeriod).ContinueWith(_ => StartExpirationTask());
         }
 
-        private MapFileResolver GetMapFileResolver(string market)
+        private MapFileResolver GetMapFileResolver(CorporateActionsKey corporateActionsKey)
         {
-            if (market != QuantConnect.Market.USA.ToLowerInvariant())
-            {
-                // don't explode for other markets which request map files and we don't have
-                return MapFileResolver.Empty;
-            }
+            var market = corporateActionsKey.Market;
             var timestamp = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
             var todayNewYork = timestamp.Date;
             var yesterdayNewYork = todayNewYork.AddDays(-1);
@@ -112,7 +106,7 @@ namespace QuantConnect.Data.Auxiliary
             var date = yesterdayNewYork;
             do
             {
-                var zipFileName = Path.Combine(Globals.DataFolder, MapFileZipHelper.GetMapFileZipFileName(market, date));
+                var zipFileName = MapFileZipHelper.GetMapFileZipFileName(market, date, corporateActionsKey.SecurityType);
 
                 // Fetch a stream for our zip from our data provider
                 var stream = _dataProvider.Fetch(zipFileName);
@@ -121,7 +115,7 @@ namespace QuantConnect.Data.Auxiliary
                 if (stream != null)
                 {
                     Log.Trace("LocalZipMapFileProvider.Get({0}): Fetched map files for: {1} NY", market, date.ToShortDateString());
-                    var result =  new MapFileResolver(MapFileZipHelper.ReadMapFileZip(stream, market));
+                    var result =  new MapFileResolver(MapFileZipHelper.ReadMapFileZip(stream, market, corporateActionsKey.SecurityType));
                     stream.DisposeSafely();
                     return result;
                 }

Common/Data/Auxiliary/MapFile.cs

@@ -21,6 +21,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
+using QuantConnect.Interfaces;
 using QuantConnect.Logging;
 
 namespace QuantConnect.Data.Auxiliary
@@ -30,7 +31,7 @@ namespace QuantConnect.Data.Auxiliary
     /// </summary>
     public class MapFile : IEnumerable<MapFileRow>
     {
-        private readonly SortedDictionary<DateTime, MapFileRow> _data;
+        private readonly List<MapFileRow> _data;
 
         /// <summary>
         /// Gets the entity's unique symbol, i.e OIH.1
@@ -52,24 +53,29 @@ namespace QuantConnect.Data.Auxiliary
         /// </summary>
         public string FirstTicker { get; }
 
+        /// <summary>
+        /// Allows the consumer to specify a desired mapping mode
+        /// </summary>
+        public DataMappingMode? DataMappingMode { get; set; }
+
         /// <summary>
         /// Initializes a new instance of the <see cref="MapFile"/> class.
         /// </summary>
         public MapFile(string permtick, IEnumerable<MapFileRow> data)
         {
             Permtick = permtick.LazyToUpper();
-            _data = new SortedDictionary<DateTime, MapFileRow>(data.Distinct().ToDictionary(x => x.Date));
+            _data = data.Distinct().OrderBy(row => row.Date).ToList();
 
             // for performance we set first and last date on ctr
-            if (_data.Keys.Count == 0)
+            if (_data.Count == 0)
             {
                 FirstDate = Time.BeginningOfTime;
                 DelistingDate = Time.EndOfTime;
             }
             else
             {
-                FirstDate = _data.Keys.First();
-                DelistingDate = _data.Keys.Last();
+                FirstDate = _data[0].Date;
+                DelistingDate = _data[_data.Count - 1].Date;
             }
 
             var firstTicker = GetMappedSymbol(FirstDate, Permtick);
@@ -79,7 +85,7 @@ namespace QuantConnect.Data.Auxiliary
                 if (dotIndex > 0)
                 {
                     int value;
-                    var number = firstTicker.Substring(dotIndex + 1);
+                    var number = firstTicker.AsSpan(dotIndex + 1);
                     if (int.TryParse(number, out value))
                     {
                         firstTicker = firstTicker.Substring(0, dotIndex);
@@ -100,10 +106,14 @@ namespace QuantConnect.Data.Auxiliary
         {
             var mappedSymbol = defaultReturnValue;
             //Iterate backwards to find the most recent factor:
-            foreach (var splitDate in _data.Keys)
+            for (var i = 0; i < _data.Count; i++)
             {
-                if (splitDate < searchDate) continue;
-                mappedSymbol = _data[splitDate].MappedSymbol;
+                var row = _data[i];
+                if (row.Date < searchDate || row.DataMappingMode.HasValue && row.DataMappingMode != DataMappingMode)
+                {
+                    continue;
+                }
+                mappedSymbol = row.MappedSymbol;
                 break;
             }
             return mappedSymbol;
@@ -134,27 +144,17 @@ namespace QuantConnect.Data.Auxiliary
         /// <returns>Enumerable of csv lines</returns>
         public IEnumerable<string> ToCsvLines()
         {
-            foreach (var mapRow in _data.Values)
-            {
-                yield return mapRow.ToCsv();
-            }
-        }
-
-        /// <summary>
-        /// Reads in an entire map file for the requested symbol from the DataFolder
-        /// </summary>
-        public static MapFile Read(string permtick, string market)
-        {
-            return new MapFile(permtick, MapFileRow.Read(GetMapFilePath(permtick, market), market));
+            return _data.Select(mapRow => mapRow.ToCsv());
         }
 
         /// <summary>
         /// Writes the map file to a CSV file
         /// </summary>
         /// <param name="market">The market to save the MapFile to</param>
-        public void WriteToCsv(string market)
+        /// <param name="securityType">The map file security type</param>
+        public void WriteToCsv(string market, SecurityType securityType)
         {
-            var filePath = GetMapFilePath(Permtick, market);
+            var filePath = Path.Combine(GetMapFilePath(market, securityType), Permtick.ToLowerInvariant() + ".csv");
             var fileDir = Path.GetDirectoryName(filePath);
 
             if (!Directory.Exists(fileDir))
@@ -169,12 +169,12 @@ namespace QuantConnect.Data.Auxiliary
         /// <summary>
         /// Constructs the map file path for the specified market and symbol
         /// </summary>
-        /// <param name="permtick">The symbol as on disk, OIH or OIH.1</param>
         /// <param name="market">The market this symbol belongs to</param>
+        /// <param name="securityType">The map file security type</param>
         /// <returns>The file path to the requested map file</returns>
-        public static string GetMapFilePath(string permtick, string market)
+        public static string GetMapFilePath(string market, SecurityType securityType)
         {
-            return Path.Combine(Globals.CacheDataFolder, "equity", market, "map_files", permtick.ToLowerInvariant() + ".csv");
+            return Path.Combine(Globals.CacheDataFolder, securityType.SecurityTypeToLower(), market, "map_files");
         }
 
         #region Implementation of IEnumerable
@@ -188,7 +188,7 @@ namespace QuantConnect.Data.Auxiliary
         /// <filterpriority>1</filterpriority>
         public IEnumerator<MapFileRow> GetEnumerator()
         {
-            return _data.Values.GetEnumerator();
+            return _data.GetEnumerator();
         }
 
         /// <summary>
@@ -210,8 +210,10 @@ namespace QuantConnect.Data.Auxiliary
         /// </summary>
         /// <param name="mapFileDirectory">The map file directory path</param>
         /// <param name="market">The map file market</param>
+        /// <param name="securityType">The map file security type</param>
+        /// <param name="dataProvider">The data provider instance to use</param>
         /// <returns>An enumerable of all map files</returns>
-        public static IEnumerable<MapFile> GetMapFiles(string mapFileDirectory, string market)
+        public static IEnumerable<MapFile> GetMapFiles(string mapFileDirectory, string market, SecurityType securityType, IDataProvider dataProvider)
         {
             var mapFiles = new ConcurrentBag<MapFile>();
             Parallel.ForEach(Directory.EnumerateFiles(mapFileDirectory), file =>
@@ -219,7 +221,7 @@ namespace QuantConnect.Data.Auxiliary
                 if (file.EndsWith(".csv"))
                 {
                     var permtick = Path.GetFileNameWithoutExtension(file);
-                    var fileRead = SafeMapFileRowRead(file, market);
+                    var fileRead = SafeMapFileRowRead(file, market, securityType, dataProvider);
                     mapFiles.Add(new MapFile(permtick, fileRead));
                 }
             });
@@ -229,11 +231,11 @@ namespace QuantConnect.Data.Auxiliary
         /// <summary>
         /// Reads in the map file at the specified path, returning null if any exceptions are encountered
         /// </summary>
-        private static List<MapFileRow> SafeMapFileRowRead(string file, string market)
+        private static List<MapFileRow> SafeMapFileRowRead(string file, string market, SecurityType securityType, IDataProvider dataProvider)
         {
             try
             {
-                return MapFileRow.Read(file, market).ToList();
+                return MapFileRow.Read(file, market, securityType, dataProvider).ToList();
             }
             catch (Exception err)
             {

Common/Data/Auxiliary/MapFilePrimaryExchangeProvider.cs

@@ -48,7 +48,8 @@ namespace QuantConnect.Data.Auxiliary
             Exchange primaryExchange;
             if (!_primaryExchangeBySid.TryGetValue(securityIdentifier, out primaryExchange))
             {
-                var mapFile = _mapFileProvider.Get(securityIdentifier.Market).ResolveMapFile(securityIdentifier.Symbol, securityIdentifier.Date);
+                var mapFile = _mapFileProvider.Get(CorporateActionsKey.Create(securityIdentifier))
+                    .ResolveMapFile(securityIdentifier.Symbol, securityIdentifier.Date);
                 if (mapFile != null && mapFile.Any())
                 {
                     primaryExchange = mapFile.Last().PrimaryExchange;

Common/Data/Auxiliary/MapFileResolver.cs

@@ -20,6 +20,7 @@ using System.Collections.Generic;
 using System.Data;
 using System.IO;
 using System.Linq;
+using QuantConnect.Interfaces;
 using QuantConnect.Util;
 
 namespace QuantConnect.Data.Auxiliary
@@ -81,20 +82,6 @@ namespace QuantConnect.Data.Auxiliary
             }
         }
 
-        /// <summary>
-        /// Creates a new instance of the <see cref="MapFileResolver"/> class by reading all map files
-        /// for the specified market into memory
-        /// </summary>
-        /// <param name="dataDirectory">The root data directory</param>
-        /// <param name="market">The equity market to produce a map file collection for</param>
-        /// <returns>The collection of map files capable of mapping equity symbols within the specified market</returns>
-        public static MapFileResolver Create(string dataDirectory, string market)
-        {
-            market = market.ToLowerInvariant();
-            var path = Path.Combine(dataDirectory, "equity", market, "map_files");
-            return new MapFileResolver(MapFile.GetMapFiles(path, market));
-        }
-
         /// <summary>
         /// Gets the map file matching the specified permtick
         /// </summary>
@@ -121,7 +108,7 @@ namespace QuantConnect.Data.Auxiliary
             {
                 if (entries.Count == 0)
                 {
-                    return new MapFile(symbol, new List<MapFileRow>());
+                    return new MapFile(symbol, Enumerable.Empty<MapFileRow>());
                 }
 
                 // Return value of BinarySearch (from MSDN):
@@ -152,9 +139,9 @@ namespace QuantConnect.Data.Auxiliary
             // secondary search for exact mapping, find path than ends with symbol.csv
             MapFile mapFile;
             if (!_mapFilesByPermtick.TryGetValue(symbol, out mapFile)
-                || mapFile.FirstDate > date)
+                || mapFile.FirstDate > date && date != Time.BeginningOfTime)
             {
-                return new MapFile(symbol, new List<MapFileRow>());
+                return new MapFile(symbol, Enumerable.Empty<MapFileRow>());
             }
             return mapFile;
         }

Common/Data/Auxiliary/MapFileRow.cs

@@ -15,8 +15,8 @@
 */
 
 using System;
-using System.IO;
 using System.Linq;
+using QuantConnect.Interfaces;
 using System.Collections.Generic;
 
 namespace QuantConnect.Data.Auxiliary
@@ -41,46 +41,59 @@ namespace QuantConnect.Data.Auxiliary
         /// </summary>
         public Exchange PrimaryExchange { get; }
 
+        /// <summary>
+        /// Gets the securities mapping mode associated to this mapping row
+        /// </summary>
+        public DataMappingMode? DataMappingMode { get; }
+
         /// <summary>
         /// Initializes a new instance of the <see cref="MapFileRow"/> class.
         /// </summary>
-        public MapFileRow(DateTime date, string mappedSymbol, string primaryExchange, string market = QuantConnect.Market.USA)
-            : this(date, mappedSymbol, primaryExchange.GetPrimaryExchange(SecurityType.Equity, market))
+        public MapFileRow(DateTime date, string mappedSymbol, string primaryExchange,
+            string market = QuantConnect.Market.USA, SecurityType securityType = SecurityType.Equity, DataMappingMode? dataMappingMode = null)
+            : this(date, mappedSymbol, primaryExchange.GetPrimaryExchange(securityType, market), dataMappingMode)
         { }
         
         /// <summary>
         /// Initializes a new instance of the <see cref="MapFileRow"/> class.
         /// </summary>
-        public MapFileRow(DateTime date, string mappedSymbol, Exchange primaryExchange = null)
+        public MapFileRow(DateTime date, string mappedSymbol, Exchange primaryExchange = null, DataMappingMode? dataMappingMode = null)
         {
             Date = date;
             MappedSymbol = mappedSymbol.LazyToUpper();
             PrimaryExchange = primaryExchange ?? Exchange.UNKNOWN;
+            DataMappingMode = dataMappingMode;
         }
 
         /// <summary>
         /// Reads in the map_file for the specified equity symbol
         /// </summary>
-        public static IEnumerable<MapFileRow> Read(string file, string market)
+        public static IEnumerable<MapFileRow> Read(string file, string market, SecurityType securityType, IDataProvider dataProvider)
         {
-            return File.Exists(file)
-                ? File.ReadAllLines(file).Where(l => !string.IsNullOrWhiteSpace(l)).Select(s => Parse(s, market))
-                : Enumerable.Empty<MapFileRow>();
+            return dataProvider.ReadLines(file)
+                .Where(l => !string.IsNullOrWhiteSpace(l))
+                .Select(s => Parse(s, market, securityType));
         }
 
         /// <summary>
         /// Parses the specified line into a MapFileRow
         /// </summary>
-        public static MapFileRow Parse(string line, string market)
+        public static MapFileRow Parse(string line, string market, SecurityType securityType)
         {
             var csv = line.Split(',');
             var primaryExchange = Exchange.UNKNOWN;
-            if (csv.Length == 3)
+            DataMappingMode? mappingMode = null;
+
+            if (csv.Length >= 3)
             {
-                primaryExchange = csv[2].GetPrimaryExchange(SecurityType.Equity, market);
+                primaryExchange = csv[2].GetPrimaryExchange(securityType, market);
+            }
+            if (csv.Length >= 4)
+            {
+                mappingMode = csv[3].ParseDataMappingMode();
             }
 
-            return new MapFileRow(DateTime.ParseExact(csv[0], DateFormat.EightCharacter, null), csv[1], primaryExchange);
+            return new MapFileRow(DateTime.ParseExact(csv[0], DateFormat.EightCharacter, null), csv[1], primaryExchange, mappingMode);
         }
 
         #region Equality members
@@ -98,7 +111,8 @@ namespace QuantConnect.Data.Auxiliary
             if (ReferenceEquals(this, other)) return true;
             return Date.Equals(other.Date) &&
                    string.Equals(MappedSymbol, other.MappedSymbol) &&
-                   string.Equals(PrimaryExchange, other.PrimaryExchange);
+                   string.Equals(PrimaryExchange, other.PrimaryExchange) &&
+                   DataMappingMode == other.DataMappingMode;
         }
 
         /// <summary>
@@ -129,6 +143,7 @@ namespace QuantConnect.Data.Auxiliary
             {
                 return (Date.GetHashCode() * 397) ^ 
                        (MappedSymbol != null ? MappedSymbol.GetHashCode() : 0) ^
+                       (DataMappingMode != null ? DataMappingMode.GetHashCode() : 0) ^
                        (PrimaryExchange.GetHashCode());
             }
         }
@@ -156,8 +171,21 @@ namespace QuantConnect.Data.Auxiliary
         /// </summary>
         public string ToCsv()
         {
-            var encodedExchange = PrimaryExchange == Exchange.UNKNOWN? string.Empty : $",{PrimaryExchange.Code}";
-            return $"{Date.ToStringInvariant(DateFormat.EightCharacter)},{MappedSymbol.ToLowerInvariant()}{encodedExchange}";
+            var encodedExchange = string.Empty;
+            if (PrimaryExchange == Exchange.UNKNOWN)
+            {
+                if (DataMappingMode != null)
+                {
+                    // be lazy, only add a comma if we have a mapping mode after
+                    encodedExchange = ",";
+                }
+            }
+            else
+            {
+                encodedExchange = $",{PrimaryExchange.Code}";
+            }
+            var mappingMode = DataMappingMode != null ? $",{DataMappingMode}" : string.Empty;
+            return $"{Date.ToStringInvariant(DateFormat.EightCharacter)},{MappedSymbol.ToLowerInvariant()}{encodedExchange}{mappingMode}";
         }
 
         /// <summary>
@@ -167,7 +195,8 @@ namespace QuantConnect.Data.Auxiliary
         public override string ToString()
         {
             var mainExchange = PrimaryExchange == Exchange.UNKNOWN ? string.Empty : $" - {PrimaryExchange}";
-            return Date.ToShortDateString() + ": " + MappedSymbol + mainExchange;
+            var mappingMode = DataMappingMode != null ? $" - {DataMappingMode}" : string.Empty;
+            return Date.ToShortDateString() + ": " + MappedSymbol + mainExchange + mappingMode;
         }
     }
 }

Common/Data/Auxiliary/MapFileZipHelper.cs

@@ -28,15 +28,15 @@ namespace QuantConnect.Data.Auxiliary
         /// <summary>
         /// Gets the mapfile zip filename for the specified date
         /// </summary>
-        public static string GetMapFileZipFileName(string market, DateTime date)
+        public static string GetMapFileZipFileName(string market, DateTime date, SecurityType securityType)
         {
-            return $"equity/{market}/map_files/map_files_{date:yyyyMMdd}.zip";
+            return Path.Combine(Globals.DataFolder, $"{securityType.SecurityTypeToLower()}/{market}/map_files/map_files_{date:yyyyMMdd}.zip");
         }
 
         /// <summary>
         /// Reads the zip bytes as text and parses as MapFileRows to create MapFiles
         /// </summary>
-        public static IEnumerable<MapFile> ReadMapFileZip(Stream file, string market)
+        public static IEnumerable<MapFile> ReadMapFileZip(Stream file, string market, SecurityType securityType)
         {
             if (file == null || file.Length == 0)
             {
@@ -47,7 +47,7 @@ namespace QuantConnect.Data.Auxiliary
                    let filename = kvp.Key
                    where filename.EndsWith(".csv", StringComparison.InvariantCultureIgnoreCase)
                    let lines = kvp.Value.Where(line => !string.IsNullOrEmpty(line))
-                   let mapFile = SafeRead(filename, lines, market)
+                   let mapFile = SafeRead(filename, lines, market, securityType)
                    select mapFile;
             return result;
         }
@@ -55,12 +55,12 @@ namespace QuantConnect.Data.Auxiliary
         /// <summary>
         /// Parses the contents as a MapFile, if error returns a new empty map file
         /// </summary>
-        private static MapFile SafeRead(string filename, IEnumerable<string> contents, string market)
+        private static MapFile SafeRead(string filename, IEnumerable<string> contents, string market, SecurityType securityType)
         {
             var permtick = Path.GetFileNameWithoutExtension(filename);
             try
             {
-                return new MapFile(permtick, contents.Select(s => MapFileRow.Parse(s, market)));
+                return new MapFile(permtick, contents.Select(s => MapFileRow.Parse(s, market, securityType)));
             }
             catch
             {

Common/Data/Auxiliary/MappingExtensions.cs

@@ -14,7 +14,7 @@
  *
 */
 
-using System;
+using QuantConnect.Interfaces;
 
 namespace QuantConnect.Data.Auxiliary
 {
@@ -28,15 +28,17 @@ namespace QuantConnect.Data.Auxiliary
         /// </summary>
         /// <remarks>This method is aware of the data type being added for <see cref="SecurityType.Base"/>
         /// to the <see cref="SecurityIdentifier.Symbol"/> value</remarks>
-        /// <param name="mapFileResolver">The map file resolver</param>
-        /// <param name="symbol">The symbol that we want to map</param>
-        /// <param name="dataType">The configuration data type <see cref="SubscriptionDataConfig.Type"/></param>
+        /// <param name="mapFileProvider">The map file provider</param>
+        /// <param name="dataConfig">The configuration to fetch the map file for</param>
         /// <returns>The mapping file to use</returns>
-        public static MapFile ResolveMapFile(this MapFileResolver mapFileResolver,
-            Symbol symbol,
-            Type dataType)
+        public static MapFile ResolveMapFile(this IMapFileProvider mapFileProvider, SubscriptionDataConfig dataConfig)
         {
-            return mapFileResolver.ResolveMapFile(symbol , dataType.Name);
+            var resolver = MapFileResolver.Empty;
+            if(dataConfig.TickerShouldBeMapped())
+            {
+                resolver = mapFileProvider.Get(CorporateActionsKey.Create(dataConfig.Symbol));
+            }
+            return resolver.ResolveMapFile(dataConfig.Symbol , dataConfig.Type.Name, dataConfig.DataMappingMode);
         }
 
         /// <summary>
@@ -47,22 +49,34 @@ namespace QuantConnect.Data.Auxiliary
         /// <param name="mapFileResolver">The map file resolver</param>
         /// <param name="symbol">The symbol that we want to map</param>
         /// <param name="dataType">The string data type name if any</param>
+        /// <param name="mappingMode">The mapping mode to use if any</param>
         /// <returns>The mapping file to use</returns>
         public static MapFile ResolveMapFile(this MapFileResolver mapFileResolver,
             Symbol symbol,
-            string dataType = null)
+            string dataType = null,
+            DataMappingMode? mappingMode = null)
         {
             // Load the symbol and date to complete the mapFile checks in one statement
             var symbolID = symbol.HasUnderlying ? symbol.Underlying.ID.Symbol : symbol.ID.Symbol;
-            var date = symbol.HasUnderlying ? symbol.Underlying.ID.Date : symbol.ID.Date;
-
             if (dataType == null && symbol.SecurityType == SecurityType.Base)
             {
                 symbol.ID.Symbol.TryGetCustomDataType(out dataType);
             }
-            return mapFileResolver.ResolveMapFile(
-                symbol.SecurityType == SecurityType.Base && dataType != null ? symbolID.RemoveFromEnd($".{dataType}") : symbolID,
-                date);
+            symbolID = symbol.SecurityType == SecurityType.Base && dataType != null ? symbolID.RemoveFromEnd($".{dataType}") : symbolID;
+
+            MapFile result;
+            if (ReferenceEquals(mapFileResolver, MapFileResolver.Empty))
+            {
+                result = mapFileResolver.ResolveMapFile(symbol.Value, Time.BeginningOfTime);
+            }
+            else
+            {
+                var date = symbol.HasUnderlying ? symbol.Underlying.ID.Date : symbol.ID.Date;
+                result = mapFileResolver.ResolveMapFile(symbolID, date);
+            }
+            result.DataMappingMode = mappingMode;
+
+            return result;
         }
     }
 }

Common/Data/BaseData.cs

@@ -198,7 +198,7 @@ namespace QuantConnect.Data
         /// <returns>True indicates mapping should be used</returns>
         public virtual bool RequiresMapping()
         {
-            return Symbol.SecurityType.RequiresMapping();
+            return Symbol.RequiresMapping();
         }
 
         /// <summary>

Common/Data/Market/SymbolChangedEvent.cs

@@ -69,5 +69,10 @@ namespace QuantConnect.Data.Market
         {
             return new SymbolChangedEvent(Symbol, Time, OldSymbol, NewSymbol);
         }
+
+        public override string ToString()
+        {
+            return $"{Time} {OldSymbol}->{NewSymbol}";
+        }
     }
 }

Common/Data/Shortable/LocalDiskShortableProvider.cs

@@ -59,28 +59,22 @@ namespace QuantConnect.Data.Shortable
             while (i <= 7)
             {
                 var shortableListFile = Path.Combine(_shortableDataDirectory.FullName, "dates", $"{localTime.AddDays(-i):yyyyMMdd}.csv");
-                var stream = _dataProvider.Fetch(shortableListFile);
 
-                if (stream != null)
+                foreach (var line in _dataProvider.ReadLines(shortableListFile))
                 {
-                    using (var streamReader = new StreamReader(stream))
-                    {
-                        foreach (var line in streamReader.ReadAllLines())
-                        {
-                            var csv = line.Split(',');
-                            var ticker = csv[0];
+                    var csv = line.Split(',');
+                    var ticker = csv[0];
 
-                            var symbol =
-                                new Symbol(
-                                    SecurityIdentifier.GenerateEquity(ticker, QuantConnect.Market.USA,
-                                        mappingResolveDate: localTime), ticker);
-                            var quantity = Parse.Long(csv[1]);
+                    var symbol = new Symbol(
+                            SecurityIdentifier.GenerateEquity(ticker, QuantConnect.Market.USA,
+                                mappingResolveDate: localTime), ticker);
+                    var quantity = Parse.Long(csv[1]);
 
-                            allSymbols[symbol] = quantity;
-                        }
-                    }
+                    allSymbols[symbol] = quantity;
+                }
 
-                    stream.Dispose();
+                if (allSymbols.Count > 0)
+                {
                     return allSymbols;
                 }
 
@@ -107,28 +101,16 @@ namespace QuantConnect.Data.Shortable
             // Implicitly trusts that Symbol.Value has been mapped and updated to the latest ticker
             var shortableSymbolFile = Path.Combine(_shortableDataDirectory.FullName, "symbols", $"{symbol.Value.ToLowerInvariant()}.csv");
 
-            using (var stream = _dataProvider.Fetch(shortableSymbolFile))
+            var localDate = localTime.Date;
+            foreach (var line in _dataProvider.ReadLines(shortableSymbolFile))
             {
-                if (stream == null)
-                {
-                    // Don't allow shorting if data is missing for the provided Symbol.
-                    return 0;
-                }
+                var csv = line.Split(',');
+                var date = Parse.DateTimeExact(csv[0], "yyyyMMdd");
 
-                var localDate = localTime.Date;
-                using (var streamReader = new StreamReader(stream))
+                if (localDate == date)
                 {
-                    foreach (var line in streamReader.ReadAllLines())
-                    {
-                        var csv = line.Split(',');
-                        var date = Parse.DateTimeExact(csv[0], "yyyyMMdd");
-                        
-                        if (localDate == date)
-                        {
-                            var quantity = Parse.Long(csv[1]);
-                            return quantity;
-                        }
-                    }
+                    var quantity = Parse.Long(csv[1]);
+                    return quantity;
                 }
             }
 

Common/Data/SubscriptionDataConfig.cs

@@ -14,13 +14,13 @@
 */
 
 using System;
-using System.Collections.Generic;
-using System.ComponentModel;
-using System.Runtime.CompilerServices;
 using NodaTime;
-using QuantConnect.Data.Consolidators;
-using QuantConnect.Securities;
 using QuantConnect.Util;
+using System.ComponentModel;
+using QuantConnect.Securities;
+using System.Collections.Generic;
+using QuantConnect.Data.Consolidators;
+using QuantConnect.Data.Market;
 using static QuantConnect.StringExtensions;
 
 namespace QuantConnect.Data
@@ -32,15 +32,20 @@ namespace QuantConnect.Data
     {
         private readonly SecurityIdentifier _sid;
 
+        /// <summary>
+        /// Event fired when there is a new symbol due to mapping
+        /// </summary>
+        public event EventHandler<NewSymbolEventArgs> NewSymbol;
+
         /// <summary>
         /// Type of data
         /// </summary>
-        public readonly Type Type;
+        public Type Type { get; }
 
         /// <summary>
         /// Security type of this data subscription
         /// </summary>
-        public readonly SecurityType SecurityType;
+        public SecurityType SecurityType => Symbol.SecurityType;
 
         /// <summary>
         /// Symbol of the asset we're requesting: this is really a perm tick!!
@@ -50,52 +55,64 @@ namespace QuantConnect.Data
         /// <summary>
         /// Trade, quote or open interest data
         /// </summary>
-        public readonly TickType TickType;
+        public TickType TickType { get; }
 
         /// <summary>
         /// Resolution of the asset we're requesting, second minute or tick
         /// </summary>
-        public readonly Resolution Resolution;
+        public Resolution Resolution { get; }
 
         /// <summary>
         /// Timespan increment between triggers of this data:
         /// </summary>
-        public readonly TimeSpan Increment;
+        public TimeSpan Increment { get; }
 
         /// <summary>
         /// True if wish to send old data when time gaps in data feed.
         /// </summary>
-        public readonly bool FillDataForward;
+        public bool FillDataForward { get; }
 
         /// <summary>
         /// Boolean Send Data from between 4am - 8am (Equities Setting Only)
         /// </summary>
-        public readonly bool ExtendedMarketHours;
+        public bool ExtendedMarketHours { get; }
 
         /// <summary>
         /// True if this subscription was added for the sole purpose of providing currency conversion rates via <see cref="CashBook.EnsureCurrencyDataFeeds"/>
         /// </summary>
-        public readonly bool IsInternalFeed;
+        public bool IsInternalFeed { get; }
 
         /// <summary>
         /// True if this subscription is for custom user data, false for QC data
         /// </summary>
-        public readonly bool IsCustomData;
+        public bool IsCustomData { get; }
 
         /// <summary>
         /// The sum of dividends accrued in this subscription, used for scaling total return prices
         /// </summary>
-        public decimal SumOfDividends;
+        public decimal SumOfDividends{ get; set; }
 
         /// <summary>
         /// Gets the normalization mode used for this subscription
         /// </summary>
-        public DataNormalizationMode DataNormalizationMode = DataNormalizationMode.Adjusted;
+        public DataNormalizationMode DataNormalizationMode { get; set; }
+
+        /// <summary>
+        /// Gets the securities mapping mode used for this subscription
+        /// </summary>
+        /// <remarks>This is particular useful when generating continuous futures</remarks>
+        public DataMappingMode DataMappingMode { get; }
+
+        /// <summary>
+        /// The continuous contract desired offset from the current front month.
+        /// For example, 0 (default) will use the front month, 1 will use the back month contract
+        /// </summary>
+        public uint ContractDepthOffset { get; }
 
         /// <summary>
         /// Price Scaling Factor:
         /// </summary>
-        public decimal PriceScaleFactor;
+        public decimal PriceScaleFactor { get; set; }
 
         /// <summary>
         /// Symbol Mapping: When symbols change over time (e.g. CHASE-> JPM) need to update the symbol requested.
@@ -104,40 +121,52 @@ namespace QuantConnect.Data
         {
             get
             {
-                return Symbol.ID.SecurityType.IsOption() && Symbol.HasUnderlying
-                    ? Symbol.Underlying.Value
-                    : Symbol.Value;
+                if (Symbol.HasUnderlying)
+                {
+                    if (SecurityType == SecurityType.Future)
+                    {
+                        return Symbol.Underlying.ID.ToString();
+                    }
+                    if (SecurityType.IsOption())
+                    {
+                        return Symbol.Underlying.Value;
+                    }
+                }
+                return Symbol.Value;
             }
             set
             {
-                Symbol = Symbol.UpdateMappedSymbol(value);
+                var oldSymbol = Symbol;
+                Symbol = Symbol.UpdateMappedSymbol(value, ContractDepthOffset);
+
+                NewSymbol?.Invoke(this, new NewSymbolEventArgs(Symbol, oldSymbol));
             }
         }
 
         /// <summary>
         /// Gets the market / scope of the symbol
         /// </summary>
-        public readonly string Market;
+        public string Market => Symbol.ID.Market;
 
         /// <summary>
         /// Gets the data time zone for this subscription
         /// </summary>
-        public readonly DateTimeZone DataTimeZone;
+        public DateTimeZone DataTimeZone { get; }
 
         /// <summary>
         /// Gets the exchange time zone for this subscription
         /// </summary>
-        public readonly DateTimeZone ExchangeTimeZone;
+        public DateTimeZone ExchangeTimeZone { get; }
 
         /// <summary>
         /// Consolidators that are registred with this subscription
         /// </summary>
-        public readonly ISet<IDataConsolidator> Consolidators;
+        public ISet<IDataConsolidator> Consolidators { get; }
 
         /// <summary>
         /// Gets whether or not this subscription should have filters applied to it (market hours/user filters from security)
         /// </summary>
-        public readonly bool IsFilteredSubscription;
+        public bool IsFilteredSubscription { get; }
 
         /// <summary>
         /// Constructor for Data Subscriptions
@@ -156,6 +185,9 @@ namespace QuantConnect.Data
         /// <param name="tickType">Specifies if trade or quote data is subscribed</param>
         /// <param name="isFilteredSubscription">True if this subscription should have filters applied to it (market hours/user filters from security), false otherwise</param>
         /// <param name="dataNormalizationMode">Specifies normalization mode used for this subscription</param>
+        /// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
+        /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
+        /// For example, 0 (default) will use the front month, 1 will use the back month contract</param>
         public SubscriptionDataConfig(Type objectType,
             Symbol symbol,
             Resolution resolution,
@@ -167,7 +199,9 @@ namespace QuantConnect.Data
             bool isCustom = false,
             TickType? tickType = null,
             bool isFilteredSubscription = true,
-            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted)
+            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted,
+            DataMappingMode dataMappingMode = DataMappingMode.OpenInterest,
+            uint contractDepthOffset = 0)
         {
             if (objectType == null) throw new ArgumentNullException(nameof(objectType));
             if (symbol == null) throw new ArgumentNullException(nameof(symbol));
@@ -175,7 +209,6 @@ namespace QuantConnect.Data
             if (exchangeTimeZone == null) throw new ArgumentNullException(nameof(exchangeTimeZone));
 
             Type = objectType;
-            SecurityType = symbol.ID.SecurityType;
             Resolution = resolution;
             _sid = symbol.ID;
             Symbol = symbol;
@@ -184,9 +217,10 @@ namespace QuantConnect.Data
             PriceScaleFactor = 1;
             IsInternalFeed = isInternalFeed;
             IsCustomData = isCustom;
-            Market = symbol.ID.Market;
             DataTimeZone = dataTimeZone;
+            DataMappingMode = dataMappingMode;
             ExchangeTimeZone = exchangeTimeZone;
+            ContractDepthOffset = contractDepthOffset;
             IsFilteredSubscription = isFilteredSubscription;
             Consolidators = new ConcurrentSet<IDataConsolidator>();
             DataNormalizationMode = dataNormalizationMode;
@@ -235,6 +269,9 @@ namespace QuantConnect.Data
         /// <param name="tickType">Specifies if trade or quote data is subscribed</param>
         /// <param name="isFilteredSubscription">True if this subscription should have filters applied to it (market hours/user filters from security), false otherwise</param>
         /// <param name="dataNormalizationMode">Specifies normalization mode used for this subscription</param>
+        /// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
+        /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
+        /// For example, 0 (default) will use the front month, 1 will use the back month contract</param>
         public SubscriptionDataConfig(SubscriptionDataConfig config,
             Type objectType = null,
             Symbol symbol = null,
@@ -247,7 +284,9 @@ namespace QuantConnect.Data
             bool? isCustom = null,
             TickType? tickType = null,
             bool? isFilteredSubscription = null,
-            DataNormalizationMode? dataNormalizationMode = null)
+            DataNormalizationMode? dataNormalizationMode = null,
+            DataMappingMode? dataMappingMode = null,
+            uint? contractDepthOffset = null)
             : this(
             objectType ?? config.Type,
             symbol ?? config.Symbol,
@@ -260,7 +299,9 @@ namespace QuantConnect.Data
             isCustom ?? config.IsCustomData,
             tickType ?? config.TickType,
             isFilteredSubscription ?? config.IsFilteredSubscription,
-            dataNormalizationMode ?? config.DataNormalizationMode
+            dataNormalizationMode ?? config.DataNormalizationMode,
+            dataMappingMode ?? config.DataMappingMode,
+            contractDepthOffset ?? config.ContractDepthOffset
             )
         {
             PriceScaleFactor = config.PriceScaleFactor;
@@ -287,7 +328,9 @@ namespace QuantConnect.Data
                 && IsInternalFeed == other.IsInternalFeed
                 && IsCustomData == other.IsCustomData
                 && DataTimeZone.Equals(other.DataTimeZone)
+                && DataMappingMode == other.DataMappingMode
                 && ExchangeTimeZone.Equals(other.ExchangeTimeZone)
+                && ContractDepthOffset == other.ContractDepthOffset
                 && IsFilteredSubscription == other.IsFilteredSubscription;
         }
 
@@ -324,8 +367,10 @@ namespace QuantConnect.Data
                 hashCode = (hashCode*397) ^ ExtendedMarketHours.GetHashCode();
                 hashCode = (hashCode*397) ^ IsInternalFeed.GetHashCode();
                 hashCode = (hashCode*397) ^ IsCustomData.GetHashCode();
+                hashCode = (hashCode*397) ^ DataMappingMode.GetHashCode();
                 hashCode = (hashCode*397) ^ DataTimeZone.Id.GetHashCode();// timezone hash is expensive, use id instead
                 hashCode = (hashCode*397) ^ ExchangeTimeZone.Id.GetHashCode();// timezone hash is expensive, use id instead
+                hashCode = (hashCode*397) ^ ContractDepthOffset.GetHashCode();
                 hashCode = (hashCode*397) ^ IsFilteredSubscription.GetHashCode();
                 return hashCode;
             }
@@ -356,7 +401,26 @@ namespace QuantConnect.Data
         /// <filterpriority>2</filterpriority>
         public override string ToString()
         {
-            return Invariant($"{Symbol.Value},{MappedSymbol},{Resolution},{Type.Name},{TickType},{DataNormalizationMode}{(IsInternalFeed ? ",Internal" : string.Empty)}");
+            return Invariant($"{Symbol.Value},#{ContractDepthOffset},{MappedSymbol},{Resolution},{Type.Name},{TickType},{DataNormalizationMode},{DataMappingMode}{(IsInternalFeed ? ",Internal" : string.Empty)}");
+        }
+
+        public class NewSymbolEventArgs : EventArgs
+        {
+            /// <summary>
+            /// The old symbol instance
+            /// </summary>
+            public Symbol Old { get; }
+
+            /// <summary>
+            /// The new symbol instance
+            /// </summary>
+            public Symbol New { get; }
+
+            public NewSymbolEventArgs(Symbol @new, Symbol old)
+            {
+                New = @new;
+                Old = old;
+            }
         }
     }
 }

Common/Data/SubscriptionDataConfigExtensions.cs

@@ -120,6 +120,18 @@ namespace QuantConnect.Data
             return config.GetBaseDataInstance().RequiresMapping();
         }
 
+        /// <summary>
+        /// Will determine if price scaling should be used for this subscription configuration
+        /// </summary>
+        /// <param name="config">The subscription data configuration we are processing</param>
+        /// <remarks>One of the objectives of this method is to normalize the 'use price scale'
+        /// check and void code duplication and related issues</remarks>
+        /// <returns>True if ticker prices should be scaled</returns>
+        public static bool PricesShouldBeScaled(this SubscriptionDataConfig config)
+        {
+            return !config.IsCustomData && !config.Symbol.Value.Contains("UNIVERSE") && config.SecurityType == SecurityType.Equity;
+        }
+
         /// <summary>
         /// Initializes a new instance of the <see cref="BaseData"/> type defined in <paramref name="config"/> with the symbol properly set
         /// </summary>

Common/Data/UniverseSelection/Universe.cs

@@ -251,7 +251,8 @@ namespace QuantConnect.Data.UniverseSelection
                 UniverseSettings.FillForward,
                 UniverseSettings.ExtendedMarketHours,
                 dataNormalizationMode: UniverseSettings.DataNormalizationMode,
-                subscriptionDataTypes: UniverseSettings.SubscriptionDataTypes);
+                subscriptionDataTypes: UniverseSettings.SubscriptionDataTypes,
+                dataMappingMode: UniverseSettings.DataMappingMode);
             return result.Select(config => new SubscriptionRequest(isUniverseSubscription: false,
                 universe: this,
                 security: security,

Common/Data/UniverseSelection/UniverseSettings.cs

@@ -57,6 +57,18 @@ namespace QuantConnect.Data.UniverseSelection
         /// </summary>
         public DataNormalizationMode DataNormalizationMode;
 
+        /// <summary>
+        /// Defines how universe data is mapped together
+        /// </summary>
+        /// <remarks>This is particular useful when generating continuous futures</remarks>
+        public DataMappingMode DataMappingMode;
+
+        /// <summary>
+        /// The continuous contract desired offset from the current front month.
+        /// For example, 0 (default) will use the front month, 1 will use the back month contra
+        /// </summary>
+        public int ContractDepthOffset;
+
         /// <summary>
         /// Allows a universe to specify which data types to add for a selected symbol
         /// </summary>
@@ -71,11 +83,17 @@ namespace QuantConnect.Data.UniverseSelection
         /// <param name="extendedMarketHours">True to allow extended market hours data, false otherwise</param>
         /// <param name="minimumTimeInUniverse">Defines the minimum amount of time a security must remain in the universe before being removed</param>
         /// <param name="dataNormalizationMode">Defines how universe data is normalized before being send into the algorithm</param>
-        public UniverseSettings(Resolution resolution, decimal leverage, bool fillForward, bool extendedMarketHours, TimeSpan minimumTimeInUniverse, DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted)
+        /// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
+        /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
+        /// For example, 0 (default) will use the front month, 1 will use the back month contract</param>
+        public UniverseSettings(Resolution resolution, decimal leverage, bool fillForward, bool extendedMarketHours, TimeSpan minimumTimeInUniverse, DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted,
+            DataMappingMode dataMappingMode = DataMappingMode.OpenInterest, int contractDepthOffset = 0)
         {
             Resolution = resolution;
             Leverage = leverage;
             FillForward = fillForward;
+            DataMappingMode = dataMappingMode;
+            ContractDepthOffset = contractDepthOffset;
             ExtendedMarketHours = extendedMarketHours;
             MinimumTimeInUniverse = minimumTimeInUniverse;
             DataNormalizationMode = dataNormalizationMode;
@@ -89,6 +107,8 @@ namespace QuantConnect.Data.UniverseSelection
             Resolution = universeSettings.Resolution;
             Leverage = universeSettings.Leverage;
             FillForward = universeSettings.FillForward;
+            DataMappingMode = universeSettings.DataMappingMode;
+            ContractDepthOffset = universeSettings.ContractDepthOffset;
             ExtendedMarketHours = universeSettings.ExtendedMarketHours;
             MinimumTimeInUniverse = universeSettings.MinimumTimeInUniverse;
             DataNormalizationMode = universeSettings.DataNormalizationMode;

Common/Exchange.cs

@@ -183,6 +183,12 @@ namespace QuantConnect
         public static Exchange ISE_MERCURY { get; }
             = new("ISE_MERCURY", "J", "International Securities Options Exchange MERCURY", QuantConnect.Market.USA, SecurityType.Option);
 
+        /// <summary>
+        /// The Chicago Mercantile Exchange (CME), is an organized exchange for the trading of futures and options.
+        /// </summary>
+        public static Exchange CME { get; }
+            = new("CME", "CME", "Futures and Options Chicago Mercantile Exchange", QuantConnect.Market.CME, SecurityType.Future, SecurityType.FutureOption);
+
         /// <summary>
         /// Exchange description
         /// </summary>

Common/Extensions.cs

@@ -55,6 +55,7 @@ using NodaTime.TimeZones;
 using QuantConnect.Configuration;
 using QuantConnect.Data.Auxiliary;
 using QuantConnect.Exceptions;
+using QuantConnect.Securities.Future;
 using QuantConnect.Securities.FutureOption;
 using QuantConnect.Securities.Option;
 
@@ -171,7 +172,26 @@ namespace QuantConnect
                 catch (WebException ex)
                 {
                     Log.Error(ex, $"DownloadData(): failed for: '{url}'");
-                    // If server returned an error most likely on this day there is no data we are going to the next cycle
+                    return null;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Helper method to download a provided url as a byte array
+        /// </summary>
+        /// <param name="url">The url to download data from</param>
+        public static byte[] DownloadByteArray(this string url)
+        {
+            using (var wc = new HttpClient())
+            {
+                try
+                {
+                    return wc.GetByteArrayAsync(url).Result;
+                }
+                catch (Exception ex)
+                {
+                    Log.Error(ex, $"DownloadByteArray(): failed for: '{url}'");
                     return null;
                 }
             }
@@ -2210,6 +2230,31 @@ namespace QuantConnect
             }
         }
 
+        /// <summary>
+        /// Converts the specified string to its corresponding DataMappingMode
+        /// </summary>
+        /// <remarks>This method provides faster performance than enum parse</remarks>
+        /// <param name="dataMappingMode">The dataMappingMode string value</param>
+        /// <returns>The DataMappingMode value</returns>
+        public static DataMappingMode? ParseDataMappingMode(this string dataMappingMode)
+        {
+            if (string.IsNullOrEmpty(dataMappingMode))
+            {
+                return null;
+            }
+            switch (dataMappingMode.LazyToLower())
+            {
+                case "openinterest":
+                    return DataMappingMode.OpenInterest;
+                case "firstdaymonth":
+                    return DataMappingMode.FirstDayMonth;
+                case "lasttradingday":
+                    return DataMappingMode.LastTradingDay;
+                default:
+                    throw new ArgumentException($"Unexpected DataMappingMode: {dataMappingMode}");
+            }
+        }
+
         /// <summary>
         /// Converts the specified <paramref name="securityType"/> value to its corresponding lower-case string representation
         /// </summary>
@@ -2927,7 +2972,7 @@ namespace QuantConnect
             switch (symbol.ID.SecurityType)
             {
                 case SecurityType.Future:
-                    return symbol.ID.Date;
+                    return symbol.ID.Date == SecurityIdentifier.DefaultDate ? Time.EndOfTime : symbol.ID.Date;
                 case SecurityType.Option:
                     return OptionSymbol.GetLastDayOfTrading(symbol);
                 case SecurityType.FutureOption:
@@ -2949,6 +2994,62 @@ namespace QuantConnect
                 && type.Equals(typeof(T).Name, StringComparison.InvariantCultureIgnoreCase);
         }
 
+        /// <summary>
+        /// Helper method that will return a back month, with future expiration, future contract based on the given offset
+        /// </summary>
+        /// <param name="symbol">The none canonical future symbol</param>
+        /// <param name="offset">The quantity of contracts to move into the future expiration chain</param>
+        /// <returns>A new future expiration symbol instance</returns>
+        public static Symbol AdjustSymbolByOffset(this Symbol symbol, uint offset)
+        {
+            if (symbol.SecurityType != SecurityType.Future || symbol.IsCanonical())
+            {
+                throw new InvalidOperationException("Adjusting a symbol by an offset is currently only supported for non canonical futures");
+            }
+
+            var expiration = symbol.ID.Date;
+            for (var i = 0; i < offset; i++)
+            {
+                var expiryFunction = FuturesExpiryFunctions.FuturesExpiryFunction(symbol);
+                DateTime newExpiration;
+                // for the current expiration we add a month to get the next one
+                var monthOffset = 0;
+                do
+                {
+                    monthOffset++;
+                    newExpiration = expiryFunction(expiration.AddMonths(monthOffset));
+                } while (newExpiration <= expiration);
+
+                expiration = newExpiration;
+                symbol = Symbol.CreateFuture(symbol.ID.Symbol, symbol.ID.Market, newExpiration);
+            }
+
+            return symbol;
+        }
+
+        /// <summary>
+        /// Helper method to stream read lines from a file
+        /// </summary>
+        /// <param name="dataProvider">The data provider to use</param>
+        /// <param name="file">The file path to read from</param>
+        /// <returns>Enumeration of lines in file</returns>
+        public static IEnumerable<string> ReadLines(this IDataProvider dataProvider, string file)
+        {
+            var stream = dataProvider.Fetch(file);
+            if (stream == null)
+            {
+                yield break;
+            }
+
+            using (var streamReader = new StreamReader(stream))
+            {
+                while (!streamReader.EndOfStream)
+                {
+                    yield return streamReader.ReadLine();
+                }
+            }
+        }
+
         /// <summary>
         /// Returns the delisted liquidation time for a given delisting warning and exchange hours
         /// </summary>
@@ -3453,29 +3554,19 @@ namespace QuantConnect
             }
         }
 
-        /// <summary>
-        /// Read all lines from a stream reader
-        /// </summary>
-        /// <param name="reader">Stream reader to read from</param>
-        /// <returns>Enumerable of lines in stream</returns>
-        public static IEnumerable<string> ReadAllLines(this StreamReader reader)
-        {
-            string line;
-            while ((line = reader.ReadLine()) != null)
-            {
-                yield return line;
-            }
-        }
-
         /// <summary>
         /// Determine if this SecurityType requires mapping
         /// </summary>
-        /// <param name="securityType">Type to check</param>
+        /// <param name="symbol">Type to check</param>
         /// <returns>True if it needs to be mapped</returns>
-        public static bool RequiresMapping(this SecurityType securityType)
+        public static bool RequiresMapping(this Symbol symbol)
         {
-            switch (securityType)
+            switch (symbol.SecurityType)
             {
+                case SecurityType.Base:
+                    return symbol.HasUnderlying && symbol.Underlying.RequiresMapping();
+                case SecurityType.Future:
+                    return symbol.IsCanonical();
                 case SecurityType.Equity:
                 case SecurityType.Option:
                     return true;

Common/Global.cs

@@ -692,7 +692,44 @@ namespace QuantConnect
         /// <summary>
         /// The split adjusted price plus dividends
         /// </summary>
-        TotalReturn
+        TotalReturn,
+        /// <summary>
+        /// Eliminates price jumps between two consecutive contracts, adding a factor based on the difference of their prices.
+        /// </summary>
+        /// <remarks>First contract is the true one, factor 1</remarks>
+        ForwardPanamaCanal,
+        /// <summary>
+        /// Eliminates price jumps between two consecutive contracts, adding a factor based on the difference of their prices.
+        /// </summary>
+        /// <remarks>Last contract is the true one, factor 1</remarks>
+        BackwardsPanamaCanal,
+        /// <summary>
+        /// Eliminates price jumps between two consecutive contracts, multiplying the prices by their ratio.
+        /// </summary>
+        /// <remarks>Last contract is the true one, factor 1</remarks>
+        BackwardsRatio
+    }
+
+    /// <summary>
+    /// Continuous contracts mapping modes
+    /// </summary>
+    public enum DataMappingMode
+    {
+        /// <summary>
+        /// The contract maps on the previous day of expiration of the front month.
+        /// </summary>
+        LastTradingDay,
+        /// <summary>
+        /// The contract maps on the first date of the delivery month of the front month. If the contract expires prior to this date,
+        /// then it rolls on the contract's last trading date instead.
+        /// </summary>
+        /// <remarks>For example Crude Oil WTI (CL) 'DEC 2021 CLZ1' contract expires on Nov 19 2021, so mapping date will be it's expiration date</remarks>
+        /// <remarks>Another example Corn 'DEC 2021 ZCZ1' contract expires on Dec 14 2021, so mapping date will be Dec 1st</remarks>
+        FirstDayMonth,
+        /// <summary>
+        /// The contract maps when the back month contract has a higher volume that the current front month.
+        /// </summary>
+        OpenInterest
     }
 
     /// <summary>
@@ -852,6 +889,16 @@ namespace QuantConnect
                         return Exchange.UNKNOWN;
                 }
             }
+            else if (securityType == SecurityType.Future || securityType == SecurityType.FutureOption)
+            {
+                switch (exchange.LazyToUpper())
+                {
+                    case "CME":
+                        return Exchange.CME;
+                    default:
+                        return Exchange.UNKNOWN;
+                }
+            }
             return Exchange.UNKNOWN;
         }
     }

Common/Interfaces/IAlgorithm.cs

@@ -560,7 +560,10 @@ namespace QuantConnect.Interfaces
         /// <param name="fillDataForward">If true, returns the last available data even if none in that timeslice.</param>
         /// <param name="leverage">leverage for this security</param>
         /// <param name="extendedMarketHours">ExtendedMarketHours send in data from 4am - 8pm, not used for FOREX</param>
-        Security AddSecurity(SecurityType securityType, string symbol, Resolution? resolution, string market, bool fillDataForward, decimal leverage, bool extendedMarketHours);
+        /// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
+        /// <param name="dataNormalizationMode">The price scaling mode to use for the security</param>
+        Security AddSecurity(SecurityType securityType, string symbol, Resolution? resolution, string market, bool fillDataForward, decimal leverage, bool extendedMarketHours,
+            DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null);
 
         /// <summary>
         /// Creates and adds a new single <see cref="Future"/> contract to the algorithm

Common/Interfaces/IMapFileProvider.cs

@@ -35,8 +35,8 @@ namespace QuantConnect.Interfaces
         /// Gets a <see cref="MapFileResolver"/> representing all the map
         /// files for the specified market
         /// </summary>
-        /// <param name="market">The equity market, for example, 'usa'</param>
+        /// <param name="corporateActionsKey">Key used to fetch a map file resolver. Specifying market and security type</param>
         /// <returns>A <see cref="MapFileResolver"/> containing all map files for the specified market</returns>
-        MapFileResolver Get(string market);
+        MapFileResolver Get(CorporateActionsKey corporateActionsKey);
     }
 }

Common/Interfaces/ISubscriptionDataConfigService.cs

@@ -40,7 +40,9 @@ namespace QuantConnect.Interfaces
             bool isFilteredSubscription = true,
             bool isInternalFeed = false,
             bool isCustomData = false,
-            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted
+            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted,
+            DataMappingMode dataMappingMode = DataMappingMode.OpenInterest,
+            uint contractDepthOffset = 0
             );
 
         /// <summary>
@@ -57,7 +59,9 @@ namespace QuantConnect.Interfaces
             bool isInternalFeed = false,
             bool isCustomData = false,
             List<Tuple<Type, TickType>> subscriptionDataTypes = null,
-            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted
+            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted,
+            DataMappingMode dataMappingMode = DataMappingMode.OpenInterest,
+            uint contractDepthOffset = 0
             );
 
         /// <summary>

Common/Securities/Future/Future.cs

@@ -79,7 +79,7 @@ namespace QuantConnect.Securities.Future
             SettlementType = SettlementType.Cash;
             Holdings = new FutureHolding(this, currencyConverter);
             _symbolProperties = symbolProperties;
-            SetFilter(TimeSpan.Zero, TimeSpan.FromDays(35));
+            SetFilter(TimeSpan.Zero, TimeSpan.Zero);
         }
 
         /// <summary>
@@ -126,7 +126,7 @@ namespace QuantConnect.Securities.Future
             SettlementType = SettlementType.Cash;
             Holdings = new FutureHolding(this, currencyConverter);
             _symbolProperties = symbolProperties;
-            SetFilter(TimeSpan.Zero, TimeSpan.FromDays(35));
+            SetFilter(TimeSpan.Zero, TimeSpan.Zero);
             Underlying = underlying;
         }
 

Common/Securities/Future/FutureExpirationCycles.cs

@@ -40,6 +40,16 @@ namespace QuantConnect.Securities
         /// </summary>
         public static readonly int[] AllYear = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 };
 
+        /// <summary>
+        /// GJMQVZ Cycle
+        /// </summary>
+        public static readonly int[] GJMQVZ = { 2, 4, 6, 8, 10, 12 };
+
+        /// <summary>
+        /// GJKMNQVZ Cycle
+        /// </summary>
+        public static readonly int[] GJKMNQVZ = { 2, 4, 5, 6, 7, 8, 10, 12 };
+
         /// <summary>
         /// HMUZ Cycle
         /// </summary>
@@ -50,6 +60,26 @@ namespace QuantConnect.Securities
         /// </summary>
         public static readonly int[] HKNUZ = { 3, 5, 7, 9, 12 };
 
+        /// <summary>
+        /// HKNV Cycle
+        /// </summary>
+        public static readonly int[] HKNV = { 3, 5, 7, 10 };
+
+        /// <summary>
+        /// HKNVZ Cycle
+        /// </summary>
+        public static readonly int[] HKNVZ = { 3, 5, 7, 10, 12 };
+
+        /// <summary>
+        /// FHKNUX Cycle
+        /// </summary>
+        public static readonly int[] FHKNUX = { 1, 3, 5, 7, 9, 11 };
+
+        /// <summary>
+        /// FHJKQUVX Cycle
+        /// </summary>
+        public static readonly int[] FHJKQUVX = { 1, 3, 4, 5, 8, 9, 10, 11 };
+
         /// <summary>
         /// HKNUVZ Cycle
         /// </summary>
@@ -58,7 +88,12 @@ namespace QuantConnect.Securities
         /// <summary>
         /// FHKNQUVZ Cycle
         /// </summary>
-        public static readonly int[] FHKNQUVZ = { 1, 3, 5, 7, 9, 10, 12 };
+        public static readonly int[] FHKNUVZ = { 1, 3, 5, 7, 9, 10, 12 };
+
+        /// <summary>
+        /// FHKMQUVZ Cycle
+        /// </summary>
+        public static readonly int[] FHKNQUVZ = { 1, 3, 5, 7, 8, 9, 10, 12 };
 
         /// <summary>
         /// FHKNQUX Cycle

Common/Securities/Future/FutureMarginModel.cs

@@ -239,32 +239,25 @@ namespace QuantConnect.Securities.Future
         {
             lock (DataFolderSymbolLock)
             {
-                var stream = _dataProvider.Fetch(file);
-                if (stream == null)
+                // skip the first header line, also skip #'s as these are comment lines
+                var marginRequirementsEntries = _dataProvider.ReadLines(file)
+                    .Where(x => !x.StartsWith("#") && !string.IsNullOrWhiteSpace(x))
+                    .Skip(1)
+                    .Select(FromCsvLine)
+                    .OrderBy(x => x.Date)
+                    .ToArray();
+
+                if(marginRequirementsEntries.Length == 0)
                 {
                     Log.Trace($"Unable to locate future margin requirements file. Defaulting to zero margin for this symbol. File: {file}");
 
                     return new[] {
-                                new MarginRequirementsEntry
-                                {
-                                  Date = DateTime.MinValue
-                                }
-                            };
+                        new MarginRequirementsEntry
+                        {
+                            Date = DateTime.MinValue
+                        }
+                    };
                 }
-
-                MarginRequirementsEntry[] marginRequirementsEntries;
-                using (var streamReader = new StreamReader(stream))
-                {
-                    // skip the first header line, also skip #'s as these are comment lines
-                    marginRequirementsEntries = streamReader.ReadAllLines()
-                        .Where(x => !x.StartsWith("#") && !string.IsNullOrWhiteSpace(x))
-                        .Skip(1)
-                        .Select(FromCsvLine)
-                        .OrderBy(x => x.Date)
-                        .ToArray();
-                }
-                
-                stream.Dispose();
                 return marginRequirementsEntries;
             }
         }

Common/Securities/SecurityDefinitionSymbolResolver.cs

@@ -22,6 +22,7 @@ using QuantConnect.Logging;
 using QuantConnect.Interfaces;
 using QuantConnect.Configuration;
 using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
 
 namespace QuantConnect.Securities
 {
@@ -161,8 +162,7 @@ namespace QuantConnect.Securities
                 return null;
             }
 
-            var market = securityDefinition.SecurityIdentifier.Market;
-            var mapFileResolver = _mapFileProvider.Get(market);
+            var mapFileResolver = _mapFileProvider.Get(CorporateActionsKey.Create(securityDefinition.SecurityIdentifier));
             
             // Get the first ticker the symbol traded under, and then lookup the
             // trading date to get the ticker on the trading date.

Common/SecurityIdentifier.cs

@@ -21,6 +21,7 @@ using System.Numerics;
 using Newtonsoft.Json;
 using ProtoBuf;
 using QuantConnect.Configuration;
+using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Interfaces;
 using QuantConnect.Logging;
@@ -430,7 +431,7 @@ namespace QuantConnect
             var firstDate = DefaultDate;
             if (mapSymbol)
             {
-                var firstTickerDate = GetFirstTickerAndDate(mapFileProvider ?? MapFileProvider.Value, symbol, market, mappingResolveDate: mappingResolveDate);
+                var firstTickerDate = GetFirstTickerAndDate(mapFileProvider ?? MapFileProvider.Value, symbol, market, SecurityType.Equity, mappingResolveDate: mappingResolveDate);
                 firstDate = firstTickerDate.Item2;
                 symbol = firstTickerDate.Item1;
             }
@@ -497,7 +498,7 @@ namespace QuantConnect
 
             if (mapSymbol)
             {
-                var firstTickerDate = GetFirstTickerAndDate(MapFileProvider.Value, symbol, market);
+                var firstTickerDate = GetFirstTickerAndDate(MapFileProvider.Value, symbol, market, SecurityType.Equity);
                 firstDate = firstTickerDate.Item2;
                 symbol = firstTickerDate.Item1;
             }
@@ -636,11 +637,12 @@ namespace QuantConnect
         /// <param name="mapFileProvider">The IMapFileProvider instance used for resolving map files</param>
         /// <param name="tickerToday">The security's ticker as it trades today</param>
         /// <param name="market">The market the security exists in</param>
+        /// <param name="securityType">The securityType the security exists in</param>
         /// <param name="mappingResolveDate">The date to use to resolve the map file. Default value is <see cref="DateTime.Today"/></param>
         /// <returns>The security's first ticker/date if mapping data available, otherwise, the provided ticker and DefaultDate are returned</returns>
-        private static Tuple<string, DateTime> GetFirstTickerAndDate(IMapFileProvider mapFileProvider, string tickerToday, string market, DateTime? mappingResolveDate = null)
+        private static Tuple<string, DateTime> GetFirstTickerAndDate(IMapFileProvider mapFileProvider, string tickerToday, string market, SecurityType securityType, DateTime? mappingResolveDate = null)
         {
-            var resolver = mapFileProvider.Get(market);
+            var resolver = mapFileProvider.Get(new CorporateActionsKey(market, securityType));
             var mapFile = resolver.ResolveMapFile(tickerToday, mappingResolveDate ?? DateTime.Today);
 
             // if we have mapping data, use the first ticker/date from there, otherwise use provided ticker and DefaultDate

Common/Symbol.cs

@@ -17,6 +17,7 @@
 using System;
 using Newtonsoft.Json;
 using ProtoBuf;
+using QuantConnect.Securities.Future;
 using static QuantConnect.StringExtensions;
 
 namespace QuantConnect
@@ -330,7 +331,7 @@ namespace QuantConnect
         /// Creates new symbol with updated mapped symbol. Symbol Mapping: When symbols change over time (e.g. CHASE-> JPM) need to update the symbol requested.
         /// Method returns newly created symbol
         /// </summary>
-        public Symbol UpdateMappedSymbol(string mappedSymbol)
+        public Symbol UpdateMappedSymbol(string mappedSymbol, uint contractDepthOffset = 0)
         {
             // Throw for any option SecurityType that is not for equities, we don't support mapping for them (FOPs and Index Options)
             if (ID.SecurityType.IsOption() && SecurityType != SecurityType.Option)
@@ -338,6 +339,21 @@ namespace QuantConnect
                 throw new ArgumentException($"SecurityType {ID.SecurityType} can not be mapped.");
             }
 
+            if(ID.SecurityType == SecurityType.Future)
+            {
+                if (mappedSymbol == Value)
+                {
+                    // futures with no real continuous mapping
+                    return this;
+                }
+                var id = SecurityIdentifier.Parse(mappedSymbol);
+                var underlying = new Symbol(id, mappedSymbol);
+                underlying = underlying.AdjustSymbolByOffset(contractDepthOffset);
+
+                // we map the underlying
+                return new Symbol(ID, underlying.Value, underlying);
+            }
+
             // Avoid updating the current instance's underlying Symbol.
             var underlyingSymbol = Underlying;
 
@@ -348,14 +364,14 @@ namespace QuantConnect
             // This will ensure that we map all of the underlying Symbol(s) that also require mapping updates.
             if (HasUnderlying)
             {
-                underlyingSymbol = Underlying.UpdateMappedSymbol(mappedSymbol);
+                underlyingSymbol = Underlying.UpdateMappedSymbol(mappedSymbol, contractDepthOffset);
             }
 
             // If this Symbol is not a custom data type, and the security type does not support mapping,
             // then we know for a fact that this Symbol should not be mapped.
             // Custom data types should be mapped, especially if this method is called on them because
             // they can have an underlying that is also mapped.
-            if (SecurityType != SecurityType.Base && !SecurityType.RequiresMapping())
+            if (SecurityType != SecurityType.Base && !this.RequiresMapping())
             {
                 return new Symbol(ID, Value, underlyingSymbol);
             }

Common/Util/LeanData.cs

@@ -660,6 +660,10 @@ namespace QuantConnect.Util
                         ) + ".csv";
 
                 case SecurityType.Future:
+                    if (symbol.HasUnderlying)
+                    {
+                        symbol = symbol.Underlying;
+                    }
                     var expiryDate = symbol.ID.Date;
                     var monthsToAdd = FuturesExpiryUtilityFunctions.GetDeltaBetweenContractMonthAndContractExpiry(symbol.ID.Symbol, expiryDate.Date);
                     var contractYearMonth = expiryDate.AddMonths(monthsToAdd).ToStringInvariant(DateFormat.YearMonth);

Data/future/cme/map_files/es.csv

@@ -0,0 +1,163 @@
+18991230,es,CME
+20090228,es ub39dohg8w75,CME,FirstDayMonth
+20090320,es ub39dohg8w75,CME,LastTradingDay
+20090320,es ub39dohg8w75,CME,OpenInterest
+20090531,es udkv26rt7jsx,CME,FirstDayMonth
+20090611,es udkv26rt7jsx,CME,OpenInterest
+20090619,es udkv26rt7jsx,CME,LastTradingDay
+20090831,es ug2gqp2667ep,CME,FirstDayMonth
+20090910,es ug2gqp2667ep,CME,OpenInterest
+20090918,es ug2gqp2667ep,CME,LastTradingDay
+20091130,es uik2f7cj4v0h,CME,FirstDayMonth
+20091211,es uik2f7cj4v0h,CME,OpenInterest
+20091218,es uik2f7cj4v0h,CME,LastTradingDay
+20100228,es ul1o3pmw3im9,CME,FirstDayMonth
+20100312,es ul1o3pmw3im9,CME,OpenInterest
+20100319,es ul1o3pmw3im9,CME,LastTradingDay
+20100531,es unj9s7x92681,CME,FirstDayMonth
+20100611,es unj9s7x92681,CME,OpenInterest
+20100618,es unj9s7x92681,CME,LastTradingDay
+20100831,es uq0vgq7m0ttt,CME,FirstDayMonth
+20100910,es uq0vgq7m0ttt,CME,OpenInterest
+20100917,es uq0vgq7m0ttt,CME,LastTradingDay
+20101130,es usih58hyzhfl,CME,FirstDayMonth
+20101210,es usih58hyzhfl,CME,OpenInterest
+20101217,es usih58hyzhfl,CME,LastTradingDay
+20110228,es uv02tqsby51d,CME,FirstDayMonth
+20110311,es uv02tqsby51d,CME,OpenInterest
+20110318,es uv02tqsby51d,CME,LastTradingDay
+20110531,es uxhoi92owsn5,CME,FirstDayMonth
+20110610,es uxhoi92owsn5,CME,OpenInterest
+20110617,es uxhoi92owsn5,CME,LastTradingDay
+20110831,es uzza6rd1vg8x,CME,FirstDayMonth
+20110909,es uzza6rd1vg8x,CME,OpenInterest
+20110916,es uzza6rd1vg8x,CME,LastTradingDay
+20111130,es v2gvv9neu3up,CME,FirstDayMonth
+20111209,es v2gvv9neu3up,CME,OpenInterest
+20111216,es v2gvv9neu3up,CME,LastTradingDay
+20120229,es v4yhjrxrsrgh,CME,FirstDayMonth
+20120309,es v4yhjrxrsrgh,CME,OpenInterest
+20120316,es v4yhjrxrsrgh,CME,LastTradingDay
+20120531,es v7g38a84rf29,CME,FirstDayMonth
+20120608,es v7g38a84rf29,CME,OpenInterest
+20120615,es v7g38a84rf29,CME,LastTradingDay
+20120831,es va4l1g2o9csh,CME,FirstDayMonth
+20120914,es va4l1g2o9csh,CME,OpenInterest
+20120921,es va4l1g2o9csh,CME,LastTradingDay
+20121130,es vcm6pyd180e9,CME,FirstDayMonth
+20121214,es vcm6pyd180e9,CME,OpenInterest
+20121221,es vcm6pyd180e9,CME,LastTradingDay
+20130228,es veww9t37ndvl,CME,FirstDayMonth
+20130308,es veww9t37ndvl,CME,OpenInterest
+20130315,es veww9t37ndvl,CME,LastTradingDay
+20130531,es vhle2yxr5blt,CME,FirstDayMonth
+20130614,es vhle2yxr5blt,CME,OpenInterest
+20130621,es vhle2yxr5blt,CME,LastTradingDay
+20130831,es vk2zrh843z7l,CME,FirstDayMonth
+20130913,es vk2zrh843z7l,CME,OpenInterest
+20130920,es vk2zrh843z7l,CME,LastTradingDay
+20131130,es vmklfzih2mtd,CME,FirstDayMonth
+20131213,es vmklfzih2mtd,CME,OpenInterest
+20131220,es vmklfzih2mtd,CME,LastTradingDay
+20140228,es vp274hsu1af5,CME,FirstDayMonth
+20140314,es vp274hsu1af5,CME,OpenInterest
+20140321,es vp274hsu1af5,CME,LastTradingDay
+20140531,es vrjst036zy0x,CME,FirstDayMonth
+20140613,es vrjst036zy0x,CME,OpenInterest
+20140620,es vrjst036zy0x,CME,LastTradingDay
+20140831,es vu1ehidjylmp,CME,FirstDayMonth
+20140912,es vu1ehidjylmp,CME,OpenInterest
+20140919,es vu1ehidjylmp,CME,LastTradingDay
+20141130,es vwj060nwx98h,CME,FirstDayMonth
+20141212,es vwj060nwx98h,CME,OpenInterest
+20141219,es vwj060nwx98h,CME,LastTradingDay
+20150228,es vz0luiy9vwu9,CME,FirstDayMonth
+20150313,es vz0luiy9vwu9,CME,OpenInterest
+20150320,es vz0luiy9vwu9,CME,LastTradingDay
+20150531,es w1i7j18mukg1,CME,FirstDayMonth
+20150612,es w1i7j18mukg1,CME,OpenInterest
+20150619,es w1i7j18mukg1,CME,LastTradingDay
+20150831,es w3zt7jizt81t,CME,FirstDayMonth
+20150911,es w3zt7jizt81t,CME,OpenInterest
+20150918,es w3zt7jizt81t,CME,LastTradingDay
+20151130,es w6hew1tcrvnl,CME,FirstDayMonth
+20151211,es w6hew1tcrvnl,CME,OpenInterest
+20151218,es w6hew1tcrvnl,CME,LastTradingDay
+20160229,es w8z0kk3pqj9d,CME,FirstDayMonth
+20160311,es w8z0kk3pqj9d,CME,OpenInterest
+20160318,es w8z0kk3pqj9d,CME,LastTradingDay
+20160531,es wbgm92e2p6v5,CME,FirstDayMonth
+20160610,es wbgm92e2p6v5,CME,OpenInterest
+20160617,es wbgm92e2p6v5,CME,LastTradingDay
+20160831,es wdy7xkofnugx,CME,FirstDayMonth
+20160909,es wdy7xkofnugx,CME,OpenInterest
+20160916,es wdy7xkofnugx,CME,LastTradingDay
+20161130,es wgftm2ysmi2p,CME,FirstDayMonth
+20161209,es wgftm2ysmi2p,CME,OpenInterest
+20161216,es wgftm2ysmi2p,CME,LastTradingDay
+20170228,es wixfal95l5oh,CME,FirstDayMonth
+20170310,es wixfal95l5oh,CME,OpenInterest
+20170317,es wixfal95l5oh,CME,LastTradingDay
+20170531,es wlf0z3jijta9,CME,FirstDayMonth
+20170609,es wlf0z3jijta9,CME,OpenInterest
+20170616,es wlf0z3jijta9,CME,LastTradingDay
+20170831,es wnwmnltvigw1,CME,FirstDayMonth
+20170908,es wnwmnltvigw1,CME,OpenInterest
+20170915,es wnwmnltvigw1,CME,LastTradingDay
+20171130,es wqe8c448h4ht,CME,FirstDayMonth
+20171208,es wqe8c448h4ht,CME,OpenInterest
+20171215,es wqe8c448h4ht,CME,LastTradingDay
+20180228,es wsvu0melfs3l,CME,FirstDayMonth
+20180311,es wsvu0melfs3l,CME,OpenInterest
+20180316,es wsvu0melfs3l,CME,LastTradingDay
+20180531,es wvdfp4oyefpd,CME,FirstDayMonth
+20180610,es wvdfp4oyefpd,CME,OpenInterest
+20180615,es wvdfp4oyefpd,CME,LastTradingDay
+20180831,es wy1xiajhwdfl,CME,FirstDayMonth
+20180914,es wy1xiajhwdfl,CME,OpenInterest
+20180921,es wy1xiajhwdfl,CME,LastTradingDay
+20181130,es x0jj6stuv11d,CME,FirstDayMonth
+20181214,es x0jj6stuv11d,CME,OpenInterest
+20181221,es x0jj6stuv11d,CME,LastTradingDay
+20190228,es x2u8qnk1aeip,CME,FirstDayMonth
+20190308,es x2u8qnk1aeip,CME,OpenInterest
+20190315,es x2u8qnk1aeip,CME,LastTradingDay
+20190531,es x5iqjteksc8x,CME,FirstDayMonth
+20190614,es x5iqjteksc8x,CME,OpenInterest
+20190621,es x5iqjteksc8x,CME,LastTradingDay
+20190831,es x80c8boxqzup,CME,FirstDayMonth
+20190913,es x80c8boxqzup,CME,OpenInterest
+20190920,es x80c8boxqzup,CME,LastTradingDay
+20191130,es xahxwtzapngh,CME,FirstDayMonth
+20191213,es xahxwtzapngh,CME,OpenInterest
+20191220,es xahxwtzapngh,CME,LastTradingDay
+20200229,es xczjlc9nob29,CME,FirstDayMonth
+20200315,es xczjlc9nob29,CME,OpenInterest
+20200320,es xczjlc9nob29,CME,LastTradingDay
+20200531,es xfh59uk0myo1,CME,FirstDayMonth
+20200614,es xfh59uk0myo1,CME,OpenInterest
+20200619,es xfh59uk0myo1,CME,LastTradingDay
+20200831,es xhyqycudlm9t,CME,FirstDayMonth
+20200911,es xhyqycudlm9t,CME,OpenInterest
+20200918,es xhyqycudlm9t,CME,LastTradingDay
+20201130,es xkgcmv4qk9vl,CME,FirstDayMonth
+20201211,es xkgcmv4qk9vl,CME,OpenInterest
+20201218,es xkgcmv4qk9vl,CME,LastTradingDay
+20210228,es xmxybdf3ixhd,CME,FirstDayMonth
+20210312,es xmxybdf3ixhd,CME,OpenInterest
+20210319,es xmxybdf3ixhd,CME,LastTradingDay
+20210531,es xpfjzvpghl35,CME,FirstDayMonth
+20210611,es xpfjzvpghl35,CME,OpenInterest
+20210618,es xpfjzvpghl35,CME,LastTradingDay
+20210831,es xrx5odztg8ox,CME,FirstDayMonth
+20210910,es xrx5odztg8ox,CME,OpenInterest
+20210917,es xrx5odztg8ox,CME,LastTradingDay
+20211130,es xuercwa6ewap,CME,FirstDayMonth
+20211217,es xuercwa6ewap,CME,LastTradingDay
+20220228,es xwwd1ekjdjwh,CME,FirstDayMonth
+20220318,es xwwd1ekjdjwh,CME,LastTradingDay
+20220531,es xzdypwuwc7i9,CME,FirstDayMonth
+20220617,es xzdypwuwc7i9,CME,LastTradingDay
+20220831,es y1vkef59av41,CME,FirstDayMonth
+20220916,es y1vkef59av41,CME,LastTradingDay
+20501231,es xuercwa6ewap,CME,OpenInterest

Engine/DataFeeds/BacktestingOptionChainProvider.cs

@@ -17,6 +17,7 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using QuantConnect.Configuration;
+using QuantConnect.Data.Auxiliary;
 using QuantConnect.Interfaces;
 using QuantConnect.Logging;
 using QuantConnect.Util;
@@ -59,10 +60,10 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             // Resolve any mapping before requesting option contract list for equities
             // Needs to be done in order for the data file key to be accurate
             Symbol mappedSymbol;
-            if (underlyingSymbol.SecurityType.RequiresMapping())
+            if (underlyingSymbol.RequiresMapping())
             {
-                var mapFileResolver = _mapFileProvider.Get(underlyingSymbol.ID.Market);
-                var mapFile = mapFileResolver.ResolveMapFile(underlyingSymbol.ID.Symbol, date);
+                var mapFileResolver = _mapFileProvider.Get(CorporateActionsKey.Create(underlyingSymbol));
+                var mapFile = mapFileResolver.ResolveMapFile(underlyingSymbol);
                 var ticker = mapFile.GetMappedSymbol(date, underlyingSymbol.Value);
                 mappedSymbol = underlyingSymbol.UpdateMappedSymbol(ticker);
             }

Engine/DataFeeds/DataManager.cs

@@ -389,11 +389,14 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             bool isFilteredSubscription = true,
             bool isInternalFeed = false,
             bool isCustomData = false,
-            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted
+            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted,
+            DataMappingMode dataMappingMode = DataMappingMode.OpenInterest,
+            uint contractDepthOffset = 0
             )
         {
             return Add(symbol, resolution, fillForward, extendedMarketHours, isFilteredSubscription, isInternalFeed, isCustomData,
-                new List<Tuple<Type, TickType>> { new Tuple<Type, TickType>(dataType, LeanData.GetCommonTickTypeForCommonDataTypes(dataType, symbol.SecurityType))}, dataNormalizationMode)
+                new List<Tuple<Type, TickType>> { new Tuple<Type, TickType>(dataType, LeanData.GetCommonTickTypeForCommonDataTypes(dataType, symbol.SecurityType))},
+                dataNormalizationMode, dataMappingMode, contractDepthOffset)
                 .First();
         }
 
@@ -411,7 +414,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             bool isInternalFeed = false,
             bool isCustomData = false,
             List<Tuple<Type, TickType>> subscriptionDataTypes = null,
-            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted
+            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted,
+            DataMappingMode dataMappingMode = DataMappingMode.OpenInterest,
+            uint contractDepthOffset = 0
             )
         {
             var dataTypes = subscriptionDataTypes ??
@@ -496,7 +501,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                     isCustomData,
                     isFilteredSubscription: isFilteredSubscription,
                     tickType: tickType,
-                    dataNormalizationMode: dataNormalizationMode)).ToList();
+                    dataNormalizationMode: dataNormalizationMode,
+                    dataMappingMode: dataMappingMode,
+                    contractDepthOffset: contractDepthOffset)).ToList();
 
             for (int i = 0; i < result.Count; i++)
             {

Engine/DataFeeds/Enumerators/AuxiliaryDataEnumerator.cs

@@ -15,10 +15,11 @@
 */
 
 using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
+using QuantConnect.Util;
 using QuantConnect.Data;
+using System.Collections;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
 using QuantConnect.Data.Auxiliary;
 
 namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
@@ -32,66 +33,46 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
     {
         private readonly Queue<BaseData> _auxiliaryData;
         private bool _initialized;
+        private DateTime _startTime;
+        private IMapFileProvider _mapFileProvider;
+        private IFactorFileProvider _factorFileProvider;
+        private ITradableDateEventProvider[] _tradableDateEventProviders;
+
+        /// <summary>
+        /// The associated data configuration
+        /// </summary>
+        protected SubscriptionDataConfig Config { get; }
 
         /// <summary>
         /// Creates a new instance
         /// </summary>
         /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
-        /// <param name="factorFile">The factor file to use</param>
-        /// <param name="mapFile">The <see cref="MapFile"/> to use</param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param>
         /// <param name="tradableDateEventProviders">The tradable dates event providers</param>
         /// <param name="tradableDayNotifier">Tradable dates provider</param>
         /// <param name="startTime">Start date for the data request</param>
         public AuxiliaryDataEnumerator(
             SubscriptionDataConfig config,
-            Lazy<FactorFile> factorFile,
-            Lazy<MapFile> mapFile,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
             ITradableDateEventProvider []tradableDateEventProviders,
             ITradableDatesNotifier tradableDayNotifier,
             DateTime startTime)
         {
+            Config = config;
+            _startTime = startTime;
+            _mapFileProvider = mapFileProvider;
             _auxiliaryData = new Queue<BaseData>();
+            _factorFileProvider = factorFileProvider;
+            _tradableDateEventProviders = tradableDateEventProviders;
 
-            tradableDayNotifier.NewTradableDate += (sender, eventArgs) =>
+            if (tradableDayNotifier != null)
             {
-                if (!_initialized)
-                {
-                    Initialize(config, factorFile, mapFile, tradableDateEventProviders, startTime);
-                }
-
-                foreach (var tradableDateEventProvider in tradableDateEventProviders)
-                {
-                    var newEvents = tradableDateEventProvider.GetEvents(eventArgs).ToList();
-                    foreach (var newEvent in newEvents)
-                    {
-                        _auxiliaryData.Enqueue(newEvent);
-                    }
-                }
-            };
-        }
-
-        /// <summary>
-        /// Late initialization so it is performed in the data feed stack
-        /// and not in the algorithm thread
-        /// </summary>
-        private void Initialize(SubscriptionDataConfig config,
-            Lazy<FactorFile> factorFile,
-            Lazy<MapFile> mapFile,
-            ITradableDateEventProvider[] tradableDateEventProviders,
-            DateTime startTime)
-        {
-            foreach (var tradableDateEventProvider in tradableDateEventProviders)
-            {
-                tradableDateEventProvider.Initialize(
-                    config,
-                    factorFile?.Value,
-                    mapFile?.Value,
-                    startTime);
+                tradableDayNotifier.NewTradableDate += NewTradableDate;
             }
-            _initialized = true;
         }
 
-
         /// <summary>
         /// Advances the enumerator to the next element.
         /// </summary>
@@ -102,11 +83,47 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
             return true;
         }
 
+        /// <summary>
+        /// Handle a new tradable date, drives the <see cref="ITradableDateEventProvider"/> instances
+        /// </summary>
+        protected void NewTradableDate(object sender, NewTradableDateEventArgs eventArgs)
+        {
+            Initialize();
+            for (var i = 0; i < _tradableDateEventProviders.Length; i++)
+            {
+                foreach (var newEvent in _tradableDateEventProviders[i].GetEvents(eventArgs))
+                {
+                    _auxiliaryData.Enqueue(newEvent);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Initializes the underlying tradable data event providers
+        /// </summary>
+        protected void Initialize()
+        {
+            if (!_initialized)
+            {
+                _initialized = true;
+                // Late initialization so it is performed in the data feed stack
+                for (var i = 0; i < _tradableDateEventProviders.Length; i++)
+                {
+                    _tradableDateEventProviders[i].Initialize(Config, _factorFileProvider, _mapFileProvider, _startTime);
+                }
+            }
+        }
+
         /// <summary>
         /// Dispose of the Stream Reader and close out the source stream and file connections.
         /// </summary>
         public void Dispose()
         {
+            for (var i = 0; i < _tradableDateEventProviders.Length; i++)
+            {
+                var disposable =_tradableDateEventProviders[i] as IDisposable;
+                disposable?.DisposeSafely();
+            }
         }
 
         /// <summary>

Engine/DataFeeds/Enumerators/DelistingEventProvider.cs

@@ -19,6 +19,7 @@ using System.Collections.Generic;
 using QuantConnect.Data;
 using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Market;
+using QuantConnect.Interfaces;
 using QuantConnect.Util;
 
 namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
@@ -45,16 +46,17 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// Initializes this instance
         /// </summary>
         /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
-        /// <param name="factorFile">The factor file to use</param>
-        /// <param name="mapFile">The <see cref="MapFile"/> to use</param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
         /// <param name="startTime">Start date for the data request</param>
         public virtual void Initialize(
             SubscriptionDataConfig config,
-            FactorFile factorFile,
-            MapFile mapFile,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
             DateTime startTime)
         {
             _config = config;
+            var mapFile = mapFileProvider.ResolveMapFile(_config);
             DelistingDate = new ReferenceWrapper<DateTime>(config.Symbol.GetDelistingDate(mapFile));
         }
 

Engine/DataFeeds/Enumerators/DividendEventProvider.cs

@@ -19,6 +19,7 @@ using System.Collections.Generic;
 using QuantConnect.Data;
 using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Market;
+using QuantConnect.Interfaces;
 
 namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
 {
@@ -39,18 +40,18 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// Initializes this instance
         /// </summary>
         /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
-        /// <param name="factorFile">The factor file to use</param>
-        /// <param name="mapFile">The <see cref="MapFile"/> to use</param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
         /// <param name="startTime">Start date for the data request</param>
         public void Initialize(
             SubscriptionDataConfig config,
-            FactorFile factorFile,
-            MapFile mapFile,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
             DateTime startTime)
         {
-            _mapFile = mapFile;
-            _factorFile = factorFile;
             _config = config;
+            _mapFile = mapFileProvider.ResolveMapFile(_config);
+            _factorFile = factorFileProvider.Get(_config.Symbol);
         }
 
         /// <summary>
@@ -61,6 +62,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         public IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
         {
             if (_config.Symbol == eventArgs.Symbol
+                && _factorFile != null
                 && _mapFile.HasData(eventArgs.Date))
             {
                 if (_priceFactorRatio != null)

Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactory.cs

@@ -30,22 +30,22 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
     public class BaseDataSubscriptionEnumeratorFactory : ISubscriptionEnumeratorFactory
     {
         private readonly Func<SubscriptionRequest, IEnumerable<DateTime>> _tradableDaysProvider;
-        private readonly MapFileResolver _mapFileResolver;
+        private readonly IMapFileProvider _mapFileProvider;
         private readonly bool _isLiveMode;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="BaseDataSubscriptionEnumeratorFactory"/> class
         /// </summary>
         /// <param name="isLiveMode">True for live mode, false otherwise</param>
-        /// <param name="mapFileResolver">Used for resolving the correct map files</param>
+        /// <param name="mapFileProvider">Used for resolving the correct map files</param>
         /// <param name="factorFileProvider">Used for getting factor files</param>
         /// <param name="tradableDaysProvider">Function used to provide the tradable dates to be enumerator.
         /// Specify null to default to <see cref="SubscriptionRequest.TradableDays"/></param>
-        public BaseDataSubscriptionEnumeratorFactory(bool isLiveMode, MapFileResolver mapFileResolver, IFactorFileProvider factorFileProvider, Func<SubscriptionRequest, IEnumerable<DateTime>> tradableDaysProvider = null)
+        public BaseDataSubscriptionEnumeratorFactory(bool isLiveMode, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, Func<SubscriptionRequest, IEnumerable<DateTime>> tradableDaysProvider = null)
         {
             _isLiveMode = isLiveMode;
             _tradableDaysProvider = tradableDaysProvider ?? (request => request.TradableDays);
-            _mapFileResolver = mapFileResolver;
+            _mapFileProvider = mapFileProvider;
         }
 
         /// <summary>
@@ -77,7 +77,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
             {
                 foreach (var date in _tradableDaysProvider(request))
                 {
-                    if (sourceFactory.RequiresMapping())
+                    if (sourceFactory.RequiresMapping() && _mapFileProvider != null)
                     {
                         request.Configuration.MappedSymbol = GetMappedSymbol(request.Configuration, date);
                     }
@@ -95,8 +95,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
 
         private string GetMappedSymbol(SubscriptionDataConfig config, DateTime date)
         {
-            return _mapFileResolver.ResolveMapFile(config.Symbol, config.Type)
-                .GetMappedSymbol(date, config.MappedSymbol);
+            return _mapFileProvider.ResolveMapFile(config).GetMappedSymbol(date, config.MappedSymbol);
         }
     }
 }

Engine/DataFeeds/Enumerators/Factories/CorporateEventEnumeratorFactory.cs

@@ -40,7 +40,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
         /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
         /// <param name="factorFileProvider">Used for getting factor files</param>
         /// <param name="tradableDayNotifier">Tradable dates provider</param>
-        /// <param name="mapFileResolver">Used for resolving the correct map files</param>
+        /// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param>
         /// <param name="startTime">Start date for the data request</param>
         /// <param name="enablePriceScaling">Applies price factor</param>
         /// <returns>The new auxiliary data enumerator</returns>
@@ -49,24 +49,20 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
             SubscriptionDataConfig config,
             IFactorFileProvider factorFileProvider,
             ITradableDatesNotifier tradableDayNotifier,
-            MapFileResolver mapFileResolver,
+            IMapFileProvider mapFileProvider,
             DateTime startTime,
             bool enablePriceScaling = true)
         {
-            var lazyFactorFile =
-                new Lazy<FactorFile>(() => SubscriptionUtils.GetFactorFileToUse(config, factorFileProvider));
 
             var tradableEventProviders = new List<ITradableDateEventProvider>();
 
-            if (config.Symbol.SecurityType == SecurityType.Equity)
+            if (config.PricesShouldBeScaled())
             {
                 tradableEventProviders.Add(new SplitEventProvider());
                 tradableEventProviders.Add(new DividendEventProvider());
             }
 
-            if (config.Symbol.SecurityType == SecurityType.Equity
-                || config.Symbol.SecurityType == SecurityType.Base
-                || config.Symbol.SecurityType == SecurityType.Option)
+            if (config.TickerShouldBeMapped())
             {
                 tradableEventProviders.Add(new MappingEventProvider());
             }
@@ -75,8 +71,8 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
 
             var enumerator = new AuxiliaryDataEnumerator(
                 config,
-                lazyFactorFile,
-                new Lazy<MapFile>(() => GetMapFileToUse(config, mapFileResolver)),
+                factorFileProvider,
+                mapFileProvider,
                 tradableEventProviders.ToArray(),
                 tradableDayNotifier,
                 startTime);
@@ -84,44 +80,15 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
             // avoid price scaling for backtesting; calculate it directly in worker
             // and allow subscription to extract the the data depending on config data mode
             var dataEnumerator = rawDataEnumerator;
-            if (enablePriceScaling)
+            if (enablePriceScaling && config.PricesShouldBeScaled())
             {
                 dataEnumerator = new PriceScaleFactorEnumerator(
                     rawDataEnumerator,
                     config,
-                    lazyFactorFile);
+                    factorFileProvider);
             }
 
             return new SynchronizingEnumerator(dataEnumerator, enumerator);
         }
-
-        private static MapFile GetMapFileToUse(
-            SubscriptionDataConfig config,
-            MapFileResolver mapFileResolver)
-        {
-            var mapFileToUse = new MapFile(config.Symbol.Value, new List<MapFileRow>());
-
-            // load up the map and factor files for equities, options, and custom data
-            if (config.TickerShouldBeMapped())
-            {
-                try
-                {
-                    var mapFile = mapFileResolver.ResolveMapFile(config.Symbol, config.Type);
-
-                    // only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
-                    if (mapFile.Any())
-                    {
-                        mapFileToUse = mapFile;
-                    }
-                }
-                catch (Exception err)
-                {
-                    Log.Error(err, "CorporateEventEnumeratorFactory.GetMapFileToUse():" +
-                        " Map File: " + config.Symbol.ID + ": ");
-                }
-            }
-
-            return mapFileToUse;
-        }
     }
 }

Engine/DataFeeds/Enumerators/Factories/SubscriptionDataReaderSubscriptionEnumeratorFactory.cs

@@ -15,15 +15,14 @@
 */
 
 using System;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
 using System.Linq;
 using QuantConnect.Data;
-using QuantConnect.Data.Auxiliary;
-using QuantConnect.Data.UniverseSelection;
-using QuantConnect.Interfaces;
-using QuantConnect.Lean.Engine.Results;
 using QuantConnect.Util;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using QuantConnect.Lean.Engine.Results;
+using QuantConnect.Data.UniverseSelection;
 
 namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
 {
@@ -44,7 +43,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
         private readonly Func<SubscriptionRequest, IEnumerable<DateTime>> _tradableDaysProvider;
         private readonly IMapFileProvider _mapFileProvider;
         private readonly bool _enablePriceScaling;
-        
+
         /// <summary>
         /// Initializes a new instance of the <see cref="SubscriptionDataReaderSubscriptionEnumeratorFactory"/> class
         /// </summary>
@@ -82,14 +81,10 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
         /// <returns>An enumerator reading the subscription request</returns>
         public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
         {
-            var mapFileResolver = request.Configuration.TickerShouldBeMapped()
-                                    ? _mapFileProvider.Get(request.Security.Symbol.ID.Market)
-                                    : MapFileResolver.Empty;
-
             var dataReader = new SubscriptionDataReader(request.Configuration,
                 request.StartTimeLocal,
                 request.EndTimeLocal,
-                mapFileResolver,
+                _mapFileProvider,
                 _factorFileProvider,
                 _tradableDaysProvider(request),
                 _isLiveMode,
@@ -122,7 +117,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
                 request.Configuration,
                 _factorFileProvider,
                 dataReader,
-                mapFileResolver,
+                _mapFileProvider,
                 request.StartTimeLocal,
                 _enablePriceScaling);
 

Engine/DataFeeds/Enumerators/ITradableDateEventProvider.cs

@@ -15,8 +15,9 @@
 */
 
 using System;
-using System.Collections.Generic;
 using QuantConnect.Data;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
 using QuantConnect.Data.Auxiliary;
 
 namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
@@ -37,12 +38,12 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// Initializes the event provider instance
         /// </summary>
         /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
-        /// <param name="factorFile">The factor file to use</param>
-        /// <param name="mapFile">The <see cref="MapFile"/> to use</param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param>
         /// <param name="startTime">Start date for the data request</param>
         void Initialize(SubscriptionDataConfig config,
-            FactorFile factorFile,
-            MapFile mapFile,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
             DateTime startTime);
     }
 }

Engine/DataFeeds/Enumerators/LiveAuxiliaryDataEnumerator.cs

@@ -0,0 +1,105 @@
+/*
+* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Auxiliary data enumerator that will trigger new tradable dates event accordingly
+    /// </summary>
+    public class LiveAuxiliaryDataEnumerator : AuxiliaryDataEnumerator
+    {
+        private DateTime _lastTime;
+        private ITimeProvider _timeProvider;
+        private SecurityCache _securityCache;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param>
+        /// <param name="tradableDateEventProviders">The tradable dates event providers</param>
+        /// <param name="startTime">Start date for the data request</param>
+        /// <param name="timeProvider">The time provider to use</param>
+        /// <param name="securityCache">The security cache</param>
+        public LiveAuxiliaryDataEnumerator(SubscriptionDataConfig config, IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider, ITradableDateEventProvider[] tradableDateEventProviders,
+            DateTime startTime,
+            ITimeProvider timeProvider,
+            SecurityCache securityCache)
+            // tradableDayNotifier: null -> we are going to trigger the new tradables events for the base implementation
+            : base(config, factorFileProvider, mapFileProvider, tradableDateEventProviders, tradableDayNotifier:null, startTime)
+        {
+            _securityCache = securityCache;
+            _timeProvider = timeProvider;
+
+            // initialize providers right away so mapping happens before we subscribe
+            Initialize();
+        }
+
+        public override bool MoveNext()
+        {
+            var currentDate = _timeProvider.GetUtcNow().ConvertFromUtc(Config.ExchangeTimeZone).Date;
+            if (currentDate != _lastTime)
+            {
+                // when the date changes for the security we trigger a new tradable date event
+                var newDayEvent = new NewTradableDateEventArgs(currentDate, _securityCache.GetData(), Config.Symbol, null);
+
+                NewTradableDate(this, newDayEvent);
+                // update last time
+                _lastTime = currentDate;
+            }
+
+            return base.MoveNext();
+        }
+
+        /// <summary>
+        /// Helper method to create a new instance.
+        /// Knows which security types should create one and determines the appropriate delisting event provider to use
+        /// </summary>
+        public static bool TryCreate(SubscriptionDataConfig dataConfig, ITimeProvider timeProvider, IDataQueueHandler dataQueueHandler,
+            SecurityCache securityCache, IMapFileProvider mapFileProvider, IFactorFileProvider fileProvider, DateTime startTime,
+            out IEnumerator<BaseData> enumerator)
+        {
+            enumerator = null;
+            var securityType = dataConfig.SecurityType;
+            if (securityType.IsOption() || securityType == SecurityType.Future || securityType == SecurityType.Equity)
+            {
+                var providers = new List<ITradableDateEventProvider>
+                {
+                    securityType == SecurityType.Equity
+                        ? new LiveDataBasedDelistingEventProvider(dataConfig, dataQueueHandler)
+                        : new DelistingEventProvider()
+                };
+
+                if (dataConfig.TickerShouldBeMapped())
+                {
+                    providers.Add(new LiveMappingEventProvider());
+                }
+
+                enumerator = new LiveAuxiliaryDataEnumerator(dataConfig, fileProvider, mapFileProvider,
+                    providers.ToArray(), startTime, timeProvider, securityCache);
+            }
+            return enumerator != null;
+        }
+    }
+}

Engine/DataFeeds/Enumerators/LiveAuxiliaryDataSynchronizingEnumerator.cs

@@ -41,11 +41,11 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// <param name="exchangeTimeZone">The time zone the raw data is time stamped in</param>
         /// <param name="tradeBarAggregator">The trade bar aggregator enumerator</param>
         /// <param name="auxDataEnumerators">The auxiliary data enumerators</param>
-        public LiveAuxiliaryDataSynchronizingEnumerator(ITimeProvider timeProvider, DateTimeZone exchangeTimeZone, IEnumerator<BaseData> tradeBarAggregator, params IEnumerator<BaseData>[] auxDataEnumerators)
+        public LiveAuxiliaryDataSynchronizingEnumerator(ITimeProvider timeProvider, DateTimeZone exchangeTimeZone, IEnumerator<BaseData> tradeBarAggregator, List<IEnumerator<BaseData>> auxDataEnumerators)
         {
             _timeProvider = timeProvider;
             _exchangeTimeZone = exchangeTimeZone;
-            _auxDataEnumerators = auxDataEnumerators.ToList();
+            _auxDataEnumerators = auxDataEnumerators;
             _tradeBarAggregator = tradeBarAggregator;
         }
 

Engine/DataFeeds/Enumerators/LiveDataBasedDelistingEventProvider.cs

@@ -49,16 +49,16 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// Initializes this instance
         /// </summary>
         /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
-        /// <param name="factorFile">The factor file to use</param>
-        /// <param name="mapFile">The <see cref="MapFile"/> to use</param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
         /// <param name="startTime">Start date for the data request</param>
         public override void Initialize(
             SubscriptionDataConfig config,
-            FactorFile factorFile,
-            MapFile mapFile,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
             DateTime startTime)
         {
-            base.Initialize(config, factorFile, mapFile, startTime);
+            base.Initialize(config, factorFileProvider, mapFileProvider, startTime);
 
             _delistingEnumerator = _dataQueueHandler.Subscribe(_dataConfig, (sender, args) =>
             {

Engine/DataFeeds/Enumerators/LiveDelistingEventProviderEnumerator.cs

@@ -1,141 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using QuantConnect.Data;
-using System.Collections;
-using QuantConnect.Interfaces;
-using QuantConnect.Securities;
-using System.Collections.Generic;
-using QuantConnect.Data.Auxiliary;
-using QuantConnect.Util;
-
-namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
-{
-    /// <summary>
-    /// A live trading delisting event provider which uses a <see cref="DelistingEventProvider"/> internally to emit events
-    /// based on the current frontier time
-    /// </summary>
-    /// <remarks>We create the events dynamically so that we can set the current security price in the event</remarks>
-    public class LiveDelistingEventProviderEnumerator : IEnumerator<BaseData>
-    {
-        private DateTime _lastTime;
-        private readonly ITimeProvider _timeProvider;
-        private readonly Queue<BaseData> _dataToEmit;
-        private readonly SecurityCache _securityCache;
-        private readonly SubscriptionDataConfig _dataConfig;
-        private readonly DelistingEventProvider _delistingEventProvider;
-
-        /// <summary>
-        /// Gets the element in the collection at the current position of the enumerator.
-        /// </summary>
-        /// <returns>The element in the collection at the current position of the enumerator.</returns>
-        public BaseData Current { get; private set; }
-
-        /// <summary>
-        /// Gets the current element in the collection.
-        /// </summary>
-        /// <returns>The current element in the collection.</returns>
-        object IEnumerator.Current => Current;
-
-        /// <summary>
-        /// Creates a new instance
-        /// </summary>
-        private LiveDelistingEventProviderEnumerator(ITimeProvider timeProvider, SubscriptionDataConfig dataConfig,
-            SecurityCache securityCache, DelistingEventProvider delistingEventProvider, MapFile mapFile)
-        {
-            _dataConfig = dataConfig;
-            _timeProvider = timeProvider;
-            _securityCache = securityCache;
-            _dataToEmit = new Queue<BaseData>();
-            _delistingEventProvider = delistingEventProvider;
-            _delistingEventProvider.Initialize(dataConfig, null, mapFile, DateTime.UtcNow);
-        }
-
-        /// <summary>
-        /// Advances the enumerator to the next element of the collection.
-        /// </summary>
-        /// <returns> true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.</returns>
-        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created.</exception>
-        public bool MoveNext()
-        {
-            var currentDate = _timeProvider.GetUtcNow().ConvertFromUtc(_dataConfig.ExchangeTimeZone).Date;
-            if (currentDate != _lastTime)
-            {
-                // when the date changes for the security we trigger a new tradable date event
-                var newDayEvent = new NewTradableDateEventArgs(currentDate, _securityCache.GetData(), _dataConfig.Symbol, null);
-                foreach (var delistingEvent in _delistingEventProvider.GetEvents(newDayEvent))
-                {
-                    _dataToEmit.Enqueue(delistingEvent);
-                }
-
-                // update last time
-                _lastTime = currentDate;
-            }
-
-            if (_dataToEmit.Count > 0)
-            {
-                // emit event if any
-                Current = _dataToEmit.Dequeue();
-                return true;
-            }
-
-            Current = null;
-            return false;
-        }
-
-        /// <summary>
-        /// Sets the enumerator to its initial position, which is before the first element in the collection.
-        /// </summary>
-        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created.</exception>
-        public void Reset()
-        {
-            _dataToEmit.Clear();
-        }
-
-        /// <summary>
-        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
-        /// </summary>
-        public void Dispose()
-        {
-            var liveProvider = _delistingEventProvider as LiveDataBasedDelistingEventProvider;
-            liveProvider.DisposeSafely();
-        }
-
-        /// <summary>
-        /// Helper method to create a new instance.
-        /// Knows which security types should create one and determines the appropriate delisting event provider to use
-        /// </summary>
-        public static bool TryCreate(SubscriptionDataConfig dataConfig, ITimeProvider timeProvider, IDataQueueHandler dataQueueHandler,
-            SecurityCache securityCache, IMapFileProvider mapFileProvider, out IEnumerator<BaseData> enumerator)
-        {
-            enumerator = null;
-            var securityType = dataConfig.SecurityType;
-            if (securityType.IsOption() || securityType == SecurityType.Future || securityType == SecurityType.Equity)
-            {
-                var delistingEventProvider = new DelistingEventProvider();
-                MapFile mapFile = null;
-                if (securityType == SecurityType.Equity)
-                {
-                    delistingEventProvider = new LiveDataBasedDelistingEventProvider(dataConfig, dataQueueHandler);
-                    mapFile = mapFileProvider.Get(dataConfig.Symbol.ID.Market).ResolveMapFile(dataConfig.Symbol, dataConfig.Type);
-                }
-                enumerator = new LiveDelistingEventProviderEnumerator(timeProvider, dataConfig, securityCache, delistingEventProvider, mapFile);
-                return true;
-            }
-            return false;
-        }
-    }
-}

Engine/DataFeeds/Enumerators/LiveMappingEventProvider.cs

@@ -0,0 +1,37 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Event provider who will emit <see cref="SymbolChangedEvent"/> events
+    /// </summary>
+    /// <remarks>Only special behavior is that it will refresh map file on each new tradable date event</remarks>
+    public class LiveMappingEventProvider : MappingEventProvider
+    {
+        public override IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        {
+            // refresh map file instance
+            InitializeMapFile();
+
+            return base.GetEvents(eventArgs);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/MappingEventProvider.cs

@@ -15,10 +15,11 @@
 */
 
 using System;
-using System.Collections.Generic;
 using QuantConnect.Data;
-using QuantConnect.Data.Auxiliary;
+using QuantConnect.Interfaces;
 using QuantConnect.Data.Market;
+using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
 
 namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
 {
@@ -27,24 +28,27 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
     /// </summary>
     public class MappingEventProvider : ITradableDateEventProvider
     {
-        private MapFile _mapFile;
+        private IMapFileProvider _mapFileProvider;
         private SubscriptionDataConfig _config;
+        private MapFile _mapFile;
 
         /// <summary>
         /// Initializes this instance
         /// </summary>
         /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
-        /// <param name="factorFile">The factor file to use</param>
-        /// <param name="mapFile">The <see cref="MapFile"/> to use</param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
         /// <param name="startTime">Start date for the data request</param>
-        public void Initialize(
+        public virtual void Initialize(
             SubscriptionDataConfig config,
-            FactorFile factorFile,
-            MapFile mapFile,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
             DateTime startTime)
         {
-            _mapFile = mapFile;
+            _mapFileProvider = mapFileProvider;
             _config = config;
+            InitializeMapFile();
+
             if (_mapFile.HasData(startTime.Date))
             {
                 // initialize mapped symbol using request start date
@@ -57,7 +61,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// </summary>
         /// <param name="eventArgs">The new tradable day event arguments</param>
         /// <returns>New mapping event if any</returns>
-        public IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        public virtual IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
         {
             if (_config.Symbol == eventArgs.Symbol
                 && _mapFile.HasData(eventArgs.Date))
@@ -76,5 +80,13 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
                 }
             }
         }
+
+        /// <summary>
+        /// Initializes the map file to use
+        /// </summary>
+        protected void InitializeMapFile()
+        {
+            _mapFile = _mapFileProvider.ResolveMapFile(_config);
+        }
     }
 }

Engine/DataFeeds/Enumerators/PriceScaleFactorEnumerator.cs

@@ -19,6 +19,7 @@ using System.Collections;
 using System.Collections.Generic;
 using QuantConnect.Data;
 using QuantConnect.Data.Auxiliary;
+using QuantConnect.Interfaces;
 
 namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
 {
@@ -31,8 +32,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
     {
         private readonly IEnumerator<BaseData> _rawDataEnumerator;
         private readonly SubscriptionDataConfig _config;
-        private readonly Lazy<FactorFile> _factorFile;
+        private readonly IFactorFileProvider _factorFileProvider;
         private DateTime _lastTradableDate;
+        private FactorFile _factorFile;
 
         /// <summary>
         /// Explicit interface implementation for <see cref="Current"/>
@@ -54,16 +56,16 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// <param name="rawDataEnumerator">The underlying raw data enumerator</param>
         /// <param name="config">The <see cref="SubscriptionDataConfig"/> to enumerate for.
         /// Will determine the <see cref="DataNormalizationMode"/> to use.</param>
-        /// <param name="factorFile">The <see cref="FactorFile"/> instance to use</param>
+        /// <param name="factorFileProvider">The <see cref="IFactorFileProvider"/> instance to use</param>
         public PriceScaleFactorEnumerator(
             IEnumerator<BaseData> rawDataEnumerator,
             SubscriptionDataConfig config,
-            Lazy<FactorFile> factorFile)
+            IFactorFileProvider factorFileProvider)
         {
             _lastTradableDate = DateTime.MinValue;
             _config = config;
             _rawDataEnumerator = rawDataEnumerator;
-            _factorFile = factorFile;
+            _factorFileProvider = factorFileProvider;
         }
 
         /// <summary>
@@ -88,13 +90,14 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
 
             if (underlyingReturnValue
                 && Current != null
-                && _factorFile != null
+                && _factorFileProvider != null
                 && _config.DataNormalizationMode != DataNormalizationMode.Raw)
             {
                 if (Current.Time.Date > _lastTradableDate)
                 {
+                    _factorFile = _factorFileProvider.Get(_config.Symbol);
                     _lastTradableDate = Current.Time.Date;
-                    UpdateScaleFactor(_lastTradableDate);
+                    UpdateScaleFactor(_factorFile, _lastTradableDate);
                 }
 
                 Current = Current.Normalize(_config);
@@ -112,8 +115,13 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
             throw new NotImplementedException("Reset method not implemented. Assumes loop will only be used once.");
         }
 
-        private void UpdateScaleFactor(DateTime date)
+        private void UpdateScaleFactor(FactorFile factorFile, DateTime date)
         {
+            if (factorFile == null)
+            {
+                return;
+            }
+
             switch (_config.DataNormalizationMode)
             {
                 case DataNormalizationMode.Raw:
@@ -121,11 +129,11 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
 
                 case DataNormalizationMode.TotalReturn:
                 case DataNormalizationMode.SplitAdjusted:
-                    _config.PriceScaleFactor = _factorFile.Value.GetSplitFactor(date);
+                    _config.PriceScaleFactor = _factorFile.GetSplitFactor(date);
                     break;
 
                 case DataNormalizationMode.Adjusted:
-                    _config.PriceScaleFactor = _factorFile.Value.GetPriceScaleFactor(date);
+                    _config.PriceScaleFactor = _factorFile.GetPriceScaleFactor(date);
                     break;
 
                 default:

Engine/DataFeeds/Enumerators/SplitEventProvider.cs

@@ -19,6 +19,7 @@ using System.Collections.Generic;
 using QuantConnect.Data;
 using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Market;
+using QuantConnect.Interfaces;
 
 namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
 {
@@ -39,18 +40,18 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// Initializes this instance
         /// </summary>
         /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
-        /// <param name="factorFile">The factor file to use</param>
-        /// <param name="mapFile">The <see cref="MapFile"/> to use</param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
         /// <param name="startTime">Start date for the data request</param>
         public void Initialize(
             SubscriptionDataConfig config,
-            FactorFile factorFile,
-            MapFile mapFile,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
             DateTime startTime)
         {
-            _mapFile = mapFile;
-            _factorFile = factorFile;
             _config = config;
+            _mapFile = mapFileProvider.ResolveMapFile(_config);
+            _factorFile = factorFileProvider.Get(_config.Symbol);
         }
 
         /// <summary>
@@ -61,6 +62,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         public IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
         {
             if (_config.Symbol == eventArgs.Symbol
+                && _factorFile != null
                 && _mapFile.HasData(eventArgs.Date))
             {
                 var factor = _splitFactor;

Engine/DataFeeds/FileSystemDataFeed.cs

@@ -18,6 +18,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using QuantConnect.Data;
+using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Market;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Interfaces;
@@ -149,11 +150,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             {
                 factory = new OptionChainUniverseSubscriptionEnumeratorFactory((req) =>
                 {
-                    var mapFileResolver = req.Security.Symbol.SecurityType == SecurityType.Option
-                        ? _mapFileProvider.Get(req.Configuration.Market)
-                        : null;
-
-                    var underlyingFactory = new BaseDataSubscriptionEnumeratorFactory(false, mapFileResolver, _factorFileProvider);
+                    var underlyingFactory = new BaseDataSubscriptionEnumeratorFactory(false, _mapFileProvider, _factorFileProvider);
                     return ConfigureEnumerator(req, true, underlyingFactory.CreateEnumerator(req, _dataProvider));
                 });
             }

Engine/DataFeeds/LiveTradingDataFeed.cs

@@ -20,6 +20,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using QuantConnect.Configuration;
 using QuantConnect.Data;
+using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Custom;
 using QuantConnect.Data.Custom.Tiingo;
 using QuantConnect.Data.Market;
@@ -50,6 +51,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         private IDataQueueHandler _dataQueueHandler;
         private BaseDataExchange _customExchange;
         private SubscriptionCollection _subscriptions;
+        private IFactorFileProvider _factorFileProvider;
         private CancellationTokenSource _cancellationTokenSource = new CancellationTokenSource();
         private IDataChannelProvider _channelProvider;
 
@@ -85,6 +87,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             _timeProvider = dataFeedTimeProvider.TimeProvider;
             _dataProvider = dataProvider;
             _mapFileProvider = mapFileProvider;
+            _factorFileProvider = factorFileProvider;
             _channelProvider = dataChannelProvider;
             _frontierTimeProvider = dataFeedTimeProvider.FrontierTimeProvider;
             _customExchange = new BaseDataExchange("CustomDataExchange") {SleepInterval = 10};
@@ -217,27 +220,26 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                 }
                 else
                 {
-                    EventHandler handler = (sender, args) => subscription?.OnNewDataAvailable();
-                    enumerator = _dataQueueHandler.Subscribe(request.Configuration, handler);
-
-                    var securityType = request.Configuration.SecurityType;
                     var auxEnumerators = new List<IEnumerator<BaseData>>();
 
-                    if (securityType == SecurityType.Equity)
+                    if (LiveAuxiliaryDataEnumerator.TryCreate(request.Configuration, _timeProvider, _dataQueueHandler,
+                        request.Security.Cache, _mapFileProvider, _factorFileProvider, request.StartTimeLocal, out var auxDataEnumator))
+                    {
+                        auxEnumerators.Add(auxDataEnumator);
+                    }
+
+                    EventHandler handler = (_, _) => subscription?.OnNewDataAvailable();
+                    enumerator = _dataQueueHandler.Subscribe(request.Configuration, handler);
+
+                    if (request.Configuration.PricesShouldBeScaled())
                     {
                         auxEnumerators.Add(_dataQueueHandler.Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Dividend)), handler));
                         auxEnumerators.Add(_dataQueueHandler.Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Split)), handler));
                     }
 
-                    IEnumerator<BaseData> delistingEnumerator;
-                    if (LiveDelistingEventProviderEnumerator.TryCreate(request.Configuration, _timeProvider, _dataQueueHandler, request.Security.Cache, _mapFileProvider, out delistingEnumerator))
-                    {
-                        auxEnumerators.Add(delistingEnumerator);
-                    }
-
                     if (auxEnumerators.Count > 0)
                     {
-                        enumerator = new LiveAuxiliaryDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, auxEnumerators.ToArray());
+                        enumerator = new LiveAuxiliaryDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, auxEnumerators);
                     }
                 }
 

Engine/DataFeeds/NullDataFeed.cs

@@ -27,11 +27,20 @@ namespace QuantConnect.Lean.Engine.DataFeeds
     /// </summary>
     public class NullDataFeed : IDataFeed
     {
+        /// <summary>
+        /// Allows specifying if this implementation should throw always or not
+        /// </summary>
+        public bool ShouldThrow { get; set; } = true;
+
         /// <inheritdoc />
         public bool IsActive
         {
             get
             {
+                if (!ShouldThrow)
+                {
+                    return true;
+                }
                 throw new NotImplementedException("Unexpected usage of null data feed implementation.");
             }
         }
@@ -49,24 +58,40 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             IDataChannelProvider channelProvider
             )
         {
+            if (!ShouldThrow)
+            {
+                return;
+            }
             throw new NotImplementedException("Unexpected usage of null data feed implementation.");
         }
 
         /// <inheritdoc />
         public Subscription CreateSubscription(SubscriptionRequest request)
         {
+            if (!ShouldThrow)
+            {
+                return null;
+            }
             throw new NotImplementedException("Unexpected usage of null data feed implementation.");
         }
 
         /// <inheritdoc />
         public void RemoveSubscription(Subscription subscription)
         {
+            if (!ShouldThrow)
+            {
+                return;
+            }
             throw new NotImplementedException("Unexpected usage of null data feed implementation.");
         }
 
         /// <inheritdoc />
         public void Exit()
         {
+            if (!ShouldThrow)
+            {
+                return;
+            }
             throw new NotImplementedException("Unexpected usage of null data feed implementation.");
         }
     }

Engine/DataFeeds/Queues/FakeDataQueue.cs

@@ -31,7 +31,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Queues
     /// <summary>
     /// This is an implementation of <see cref="IDataQueueHandler"/> used for testing
     /// </summary>
-    public class FakeDataQueue : IDataQueueHandler
+    public class FakeDataQueue : IDataQueueHandler, IDataQueueUniverseProvider
     {
         private int _count;
         private readonly Random _random = new Random();
@@ -205,5 +205,15 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Queues
             }
             return offsetProvider;
         }
+
+        public IEnumerable<Symbol> LookupSymbols(Symbol symbol, bool includeExpired, string securityCurrency = null)
+        {
+            yield break;
+        }
+
+        public bool CanPerformSelection()
+        {
+            return true;
+        }
     }
 }

Engine/DataFeeds/SubscriptionDataReader.cs

@@ -61,7 +61,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         private DateTime _periodStart;
         private readonly DateTime _periodFinish;
 
-        private readonly MapFileResolver _mapFileResolver;
+        private readonly IMapFileProvider _mapFileProvider;
         private readonly IFactorFileProvider _factorFileProvider;
         private FactorFile _factorFile;
         private MapFile _mapFile;
@@ -141,7 +141,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         public SubscriptionDataReader(SubscriptionDataConfig config,
             DateTime periodStart,
             DateTime periodFinish,
-            MapFileResolver mapFileResolver,
+            IMapFileProvider mapFileProvider,
             IFactorFileProvider factorFileProvider,
             IEnumerable<DateTime> tradeableDates,
             bool isLiveMode,
@@ -154,7 +154,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             //Save Start and End Dates:
             _periodStart = periodStart;
             _periodFinish = periodFinish;
-            _mapFileResolver = mapFileResolver;
+            _mapFileProvider = mapFileProvider;
             _factorFileProvider = factorFileProvider;
             _dataCacheProvider = dataCacheProvider;
 
@@ -208,16 +208,13 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                 }
             }
 
-            _factorFile = new FactorFile(_config.Symbol.Value, new List<FactorFileRow>());
-            _mapFile = new MapFile(_config.Symbol.Value, new List<MapFileRow>());
-
             // load up the map files for equities, options, and custom data if it supports it.
             // Only load up factor files for equities
             if (_dataFactory.RequiresMapping())
             {
                 try
                 {
-                    var mapFile = _mapFileResolver.ResolveMapFile(_config.Symbol, _config.Type);
+                    var mapFile = _mapFileProvider.ResolveMapFile(_config);
 
                     // only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
                     if (mapFile.Any()) _mapFile = mapFile;
@@ -261,6 +258,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                 }
             }
 
+            _factorFile ??= new FactorFile(_config.Symbol.Value, Enumerable.Empty<FactorFileRow>());
+            _mapFile ??= new MapFile(_config.Symbol.Value, Enumerable.Empty<MapFileRow>());
+
             _delistingDate = _config.Symbol.GetDelistingDate(_mapFile);
 
             // adding a day so we stop at EOD

Engine/DataFeeds/SubscriptionUtils.cs

@@ -54,7 +54,6 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             return new Subscription(request, dataEnumerator, timeZoneOffsetProvider);
         }
 
-
         /// <summary>
         /// Setups a new <see cref="Subscription"/> which will consume a blocking <see cref="EnqueueableEnumerator{T}"/>
         /// that will be feed by a worker task
@@ -70,12 +69,12 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             IFactorFileProvider factorFileProvider,
             bool enablePriceScale)
         {
-            var factorFile = GetFactorFileToUse(request.Configuration, factorFileProvider);
             var exchangeHours = request.Security.Exchange.Hours;
             var enqueueable = new EnqueueableEnumerator<SubscriptionData>(true);
             var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
             var subscription = new Subscription(request, enqueueable, timeZoneOffsetProvider);
             var config = subscription.Configuration;
+            enablePriceScale = enablePriceScale && config.PricesShouldBeScaled();
             var lastTradableDate = DateTime.MinValue;
             decimal? currentScale = null;
 
@@ -115,6 +114,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                         // We don't take into account auxiliary data because we don't scale it and because the underlying price data could be fill forwarded
                         if (enablePriceScale && data?.Time.Date > lastTradableDate && data.DataType != MarketDataType.Auxiliary && (!data.IsFillForward || lastTradableDate == DateTime.MinValue))
                         {
+                            var factorFile = factorFileProvider.Get(request.Configuration.Symbol);
                             lastTradableDate = data.Time.Date;
                             currentScale = GetScaleFactor(factorFile, mode, data.Time.Date);
                         }
@@ -166,41 +166,12 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             return subscription;
         }
 
-        /// <summary>
-        /// Gets <see cref="FactorFile"/> for configuration
-        /// </summary>
-        /// <param name="config">Subscription configuration</param>
-        /// <param name="factorFileProvider">The factor file provider</param>
-        /// <returns></returns>
-        public static FactorFile GetFactorFileToUse(
-            SubscriptionDataConfig config,
-            IFactorFileProvider factorFileProvider)
-        {
-            var factorFileToUse = new FactorFile(config.Symbol.Value, new List<FactorFileRow>());
-
-            if (!config.IsCustomData
-                && config.SecurityType == SecurityType.Equity)
-            {
-                try
-                {
-                    var factorFile = factorFileProvider.Get(config.Symbol);
-                    if (factorFile != null)
-                    {
-                        factorFileToUse = factorFile;
-                    }
-                }
-                catch (Exception err)
-                {
-                    Log.Error(err, "SubscriptionUtils.GetFactorFileToUse(): Factors File: "
-                        + config.Symbol.ID + ": ");
-                }
-            }
-
-            return factorFileToUse;
-        }
-
         private static decimal GetScaleFactor(FactorFile factorFile, DataNormalizationMode mode, DateTime date)
         {
+            if (factorFile == null)
+            {
+                return 1;
+            }
             switch (mode)
             {
                 case DataNormalizationMode.Raw:

Engine/DataFeeds/TimeSliceFactory.cs

@@ -282,8 +282,8 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                             }
                         }
 
-                        // special handling of futures data to build the futures chain
-                        if (symbol.SecurityType == SecurityType.Future)
+                        // special handling of futures data to build the futures chain. Don't push canonical continuous contract
+                        if (symbol.SecurityType == SecurityType.Future && !symbol.IsCanonical())
                         {
                             // internal feeds, like open interest, will not create the chain but will update it if it exists
                             // this is because the open interest could arrive at some closed market hours in which there is no other data and we don't

Engine/DataFeeds/UniverseSelection.cs

@@ -516,7 +516,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                     universeSettings.FillForward,
                     universeSettings.ExtendedMarketHours,
                     dataNormalizationMode: universeSettings.DataNormalizationMode,
-                    subscriptionDataTypes: universeSettings.SubscriptionDataTypes);
+                    subscriptionDataTypes: universeSettings.SubscriptionDataTypes,
+                    dataMappingMode: universeSettings.DataMappingMode,
+                    contractDepthOffset: (uint)Math.Abs(universeSettings.ContractDepthOffset));
 
                 security = _securityService.CreateSecurity(symbol, configs, universeSettings.Leverage, symbol.ID.SecurityType.IsOption(), underlying);
 

Engine/HistoricalData/SubscriptionDataReaderHistoryProvider.cs

@@ -111,21 +111,13 @@ namespace QuantConnect.Lean.Engine.HistoricalData
                 new SecurityCache()
             );
 
-            var mapFileResolver = MapFileResolver.Empty;
-            if (config.TickerShouldBeMapped())
-            {
-                mapFileResolver = _mapFileProvider.Get(config.Market);
-                var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date);
-                config.MappedSymbol = mapFile.GetMappedSymbol(startTimeLocal, config.MappedSymbol);
-            }
-
             // Tradable dates are defined with the data time zone to access the right source
             var tradableDates = Time.EachTradeableDayInTimeZone(request.ExchangeHours, startTimeLocal, endTimeLocal, request.DataTimeZone, request.IncludeExtendedMarketHours);
 
             var dataReader = new SubscriptionDataReader(config,
                 startTimeLocal,
                 endTimeLocal,
-                mapFileResolver,
+                _mapFileProvider,
                 _factorFileProvider,
                 tradableDates,
                 false,
@@ -152,7 +144,7 @@ namespace QuantConnect.Lean.Engine.HistoricalData
                 config,
                 _factorFileProvider,
                 dataReader,
-                mapFileResolver,
+                _mapFileProvider,
                 startTimeLocal);
 
             // optionally apply fill forward behavior

Tests/Algorithm/AlgorithmAddDataTests.cs

@@ -191,12 +191,12 @@ namespace QuantConnect.Tests.Algorithm
 
         [TestCase("EURUSD", typeof(IndexedLinkedData), SecurityType.Cfd, false, true)]
         [TestCase("BTCUSD", typeof(IndexedLinkedData), SecurityType.Crypto, false, true)]
-        [TestCase("CL", typeof(IndexedLinkedData), SecurityType.Future, false, true)]
+        [TestCase("CL", typeof(IndexedLinkedData), SecurityType.Future, true, true)]
         [TestCase("EURUSD", typeof(IndexedLinkedData), SecurityType.Forex, false, true)]
         [TestCase("AAPL", typeof(IndexedLinkedData), SecurityType.Equity, true, true)]
         [TestCase("EURUSD", typeof(UnlinkedData), SecurityType.Cfd, false, false)]
         [TestCase("BTCUSD", typeof(UnlinkedData), SecurityType.Crypto, false, false)]
-        [TestCase("CL", typeof(UnlinkedData), SecurityType.Future, false, false)]
+        [TestCase("CL", typeof(UnlinkedData), SecurityType.Future, true, false)]
         [TestCase("AAPL", typeof(UnlinkedData), SecurityType.Equity, true, false)]
         [TestCase("EURUSD", typeof(UnlinkedData), SecurityType.Forex, false, false)]
         public void AddDataSecuritySymbolWithUnderlying(string ticker, Type customDataType, SecurityType securityType, bool securityShouldBeMapped, bool customDataShouldBeMapped)
@@ -328,7 +328,7 @@ namespace QuantConnect.Tests.Algorithm
 
         [TestCase("EURUSD", typeof(UnlinkedData), SecurityType.Cfd, false, false)]
         [TestCase("BTCUSD", typeof(UnlinkedData), SecurityType.Crypto, false, false)]
-        [TestCase("CL", typeof(UnlinkedData), SecurityType.Future, false, false)]
+        [TestCase("CL", typeof(UnlinkedData), SecurityType.Future, true, false)]
         [TestCase("AAPL", typeof(UnlinkedData), SecurityType.Equity, true, false)]
         [TestCase("EURUSD", typeof(UnlinkedData), SecurityType.Forex, false, false)]
         public void AddDataSecurityTickerNoUnderlying(string ticker, Type customDataType, SecurityType securityType, bool securityShouldBeMapped, bool customDataShouldBeMapped)

Tests/Algorithm/AlgorithmAddSecurityTests.cs

@@ -18,6 +18,7 @@ using System;
 using NUnit.Framework;
 using QuantConnect.Algorithm;
 using QuantConnect.Data.Custom.AlphaStreams;
+using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Securities.Cfd;
 using QuantConnect.Securities.Crypto;
 using QuantConnect.Securities.Equity;
@@ -33,6 +34,7 @@ namespace QuantConnect.Tests.Algorithm
     public class AlgorithmAddSecurityTests
     {
         private QCAlgorithm _algo;
+        private NullDataFeed _dataFeed;
 
         /// <summary>
         /// Instatiate a new algorithm before each test.
@@ -42,7 +44,11 @@ namespace QuantConnect.Tests.Algorithm
         public void Setup()
         {
             _algo = new QCAlgorithm();
-            _algo.SubscriptionManager.SetDataManager(new DataManagerStub(_algo));
+            _dataFeed = new NullDataFeed
+            {
+                ShouldThrow = false
+            };
+            _algo.SubscriptionManager.SetDataManager(new DataManagerStub(_dataFeed, _algo));
         }
 
         [Test, TestCaseSource(nameof(TestAddSecurityWithSymbol))]
@@ -86,9 +92,7 @@ namespace QuantConnect.Tests.Algorithm
 
             if (symbol.IsCanonical())
             {
-                // Throws NotImplementedException because we are using NullDataFeed
-                // We need to call this to add the pending universe additions
-                Assert.Throws<NotImplementedException>(() => _algo.OnEndOfTimeStep());
+                Assert.DoesNotThrow(() => _algo.OnEndOfTimeStep());
 
                 Assert.IsTrue(_algo.UniverseManager.ContainsKey(symbol));
             }

Tests/Algorithm/Framework/Execution/ImmediateExecutionModelTests.cs

@@ -46,7 +46,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((OrderTicket)null)
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
 
             var algorithm = new QCAlgorithm();
             algorithm.SetPandasConverter();
@@ -113,7 +113,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((SubmitOrderRequest request) => new OrderTicket(algorithm.Transactions, request))
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
             orderProcessor.Setup(m => m.GetOpenOrders(It.IsAny<Func<Order, bool>>()))
                 .Returns(new List<Order> { new MarketOrder(Symbols.AAPL, openOrdersQuantity, DateTime.MinValue) });
             orderProcessor.Setup(m => m.GetOpenOrderTickets(It.IsAny<Func<OrderTicket, bool>>()))
@@ -167,7 +167,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((SubmitOrderRequest request) => new OrderTicket(algorithm.Transactions, request))
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
             orderProcessor.Setup(m => m.GetOpenOrders(It.IsAny<Func<Order, bool>>()))
                 .Returns(new List<Order> { new MarketOrder(Symbols.AAPL, 100, DateTime.MinValue) });
             orderProcessor.Setup(m => m.GetOpenOrderTickets(It.IsAny<Func<OrderTicket, bool>>()))
@@ -211,7 +211,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((SubmitOrderRequest request) => new OrderTicket(algorithm.Transactions, request))
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
             algorithm.Transactions.SetOrderProcessor(orderProcessor.Object);
 
             var model = GetExecutionModel(language);

Tests/Algorithm/Framework/Execution/SpreadExecutionModelTests.cs

@@ -43,7 +43,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((OrderTicket)null)
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
 
             var algorithm = new QCAlgorithm();
             algorithm.SetPandasConverter();
@@ -97,7 +97,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((SubmitOrderRequest request) => new OrderTicket(algorithm.Transactions, request))
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
             orderProcessor.Setup(m => m.GetOpenOrders(It.IsAny<Func<Order, bool>>()))
                 .Returns(new List<Order>());
             algorithm.Transactions.SetOrderProcessor(orderProcessor.Object);
@@ -149,7 +149,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((SubmitOrderRequest request) => new OrderTicket(algorithm.Transactions, request))
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
             orderProcessor.Setup(m => m.GetOpenOrders(It.IsAny<Func<Order, bool>>()))
                 .Returns(new List<Order>());
             algorithm.Transactions.SetOrderProcessor(orderProcessor.Object);

Tests/Algorithm/Framework/Execution/StandardDeviationExecutionModelTests.cs

@@ -45,7 +45,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((OrderTicket)null)
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
 
             var algorithm = new QCAlgorithm();
             algorithm.SetPandasConverter();
@@ -109,7 +109,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((SubmitOrderRequest request) => new OrderTicket(algorithm.Transactions, request))
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
             orderProcessor.Setup(m => m.GetOpenOrders(It.IsAny<Func<Order, bool>>()))
                 .Returns(new List<Order>());
             algorithm.Transactions.SetOrderProcessor(orderProcessor.Object);

Tests/Algorithm/Framework/Execution/VolumeWeightedAveragePriceExecutionModelTests.cs

@@ -46,7 +46,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((OrderTicket)null)
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
 
             var algorithm = new QCAlgorithm();
             algorithm.SetPandasConverter();
@@ -113,7 +113,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((SubmitOrderRequest request) => new OrderTicket(algorithm.Transactions, request))
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
             orderProcessor.Setup(m => m.GetOpenOrders(It.IsAny<Func<Order, bool>>()))
                 .Returns(new List<Order>());
             algorithm.Transactions.SetOrderProcessor(orderProcessor.Object);

Tests/Algorithm/Framework/Portfolio/EqualWeightingAlphaStreamsPortfolioConstructionModelTests.cs

@@ -20,7 +20,6 @@ using QuantConnect.Data;
 using QuantConnect.Util;
 using QuantConnect.Algorithm;
 using System.Collections.Generic;
-using QuantConnect.Data.Auxiliary;
 using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Tests.Engine.DataFeeds;
@@ -35,22 +34,16 @@ namespace QuantConnect.Tests.Algorithm.Framework.Portfolio
     public class EqualWeightingAlphaStreamsPortfolioConstructionModelTests
     {
         private ZipDataCacheProvider _cacheProvider;
-        private DefaultDataProvider _dataProvider;
         private QCAlgorithm _algorithm;
 
         [SetUp]
         public virtual void SetUp()
         {
             _algorithm = new QCAlgorithm();
-            _dataProvider = new DefaultDataProvider();
-            var mapFileProvider = new LocalDiskMapFileProvider();
-            mapFileProvider.Initialize(_dataProvider);
-            var factorFileProvider = new LocalZipFactorFileProvider();
-            factorFileProvider.Initialize(mapFileProvider, _dataProvider);
             var historyProvider = new SubscriptionDataReaderHistoryProvider();
-            _cacheProvider = new ZipDataCacheProvider(_dataProvider);
+            _cacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider);
             historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null,
-                _dataProvider, _cacheProvider, mapFileProvider, factorFileProvider,
+                TestGlobals.DataProvider, _cacheProvider, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider,
                 null, true, new DataPermissionManager()));
             _algorithm.SetHistoryProvider(historyProvider);
             _algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm));
@@ -62,7 +55,6 @@ namespace QuantConnect.Tests.Algorithm.Framework.Portfolio
         public virtual void TearDown()
         {
             _cacheProvider.DisposeSafely();
-            _dataProvider.DisposeSafely();
         }
 
         [TestCase(Language.CSharp)]

Tests/AlgorithmRunner.cs

@@ -275,7 +275,8 @@ namespace QuantConnect.Tests
             public override IEnumerable<Slice> GetHistory(IEnumerable<HistoryRequest> requests, DateTimeZone sliceTimeZone)
             {
                 requests = requests.ToList();
-                if (requests.Any(r => RegressionSetupHandlerWrapper.Algorithm.UniverseManager.ContainsKey(r.Symbol)))
+                if (requests.Any(r => RegressionSetupHandlerWrapper.Algorithm.UniverseManager.ContainsKey(r.Symbol)
+                    && (r.Symbol.SecurityType != SecurityType.Future || !r.Symbol.IsCanonical())))
                 {
                     throw new Exception("History requests should not be submitted for universe symbols");
                 }

Tests/Brokerages/InteractiveBrokers/InteractiveBrokersSymbolMapperTests.cs

@@ -16,10 +16,9 @@
 using System;
 using IBApi;
 using NUnit.Framework;
-using QuantConnect.Brokerages.InteractiveBrokers;
-using QuantConnect.Data.Auxiliary;
-using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Securities;
+using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Brokerages.InteractiveBrokers;
 using IB = QuantConnect.Brokerages.InteractiveBrokers.Client;
 
 namespace QuantConnect.Tests.Brokerages.InteractiveBrokers
@@ -71,7 +70,7 @@ namespace QuantConnect.Tests.Brokerages.InteractiveBrokers
         [TestCase("NB", "BAC")]
         public void MapCorrectBrokerageSymbol(string ticker, string ibSymbol)
         {
-            var mapper = new InteractiveBrokersSymbolMapper(new LocalDiskMapFileProvider());
+            var mapper = new InteractiveBrokersSymbolMapper(TestGlobals.MapFileProvider);
 
             var symbol = Symbol.Create(ticker, SecurityType.Equity, Market.USA);
             var brokerageSymbol = mapper.GetBrokerageSymbol(symbol);
@@ -81,7 +80,7 @@ namespace QuantConnect.Tests.Brokerages.InteractiveBrokers
         [Test]
         public void ThrowsOnNullOrEmptyOrInvalidSymbol()
         {
-            var mapper = new InteractiveBrokersSymbolMapper(new LocalDiskMapFileProvider());
+            var mapper = new InteractiveBrokersSymbolMapper(TestGlobals.MapFileProvider);
 
             Assert.Throws<ArgumentException>(() => mapper.GetLeanSymbol(null, SecurityType.Forex, Market.FXCM));
 
@@ -161,7 +160,7 @@ namespace QuantConnect.Tests.Brokerages.InteractiveBrokers
                 Currency = "USD"
             };
 
-            var mapper = new InteractiveBrokersSymbolMapper(new LocalDiskMapFileProvider());
+            var mapper = new InteractiveBrokersSymbolMapper(TestGlobals.MapFileProvider);
             var actualContract = mapper.ParseMalformedContractFutureSymbol(malformedContract, SymbolPropertiesDatabase.FromDataFolder());
 
             Assert.AreEqual(expectedContract.Symbol, actualContract.Symbol);

Tests/Common/Data/Auxiliary/LocalDiskMapFileProviderTests.cs

@@ -26,7 +26,8 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
         public void RetrievesFromDisk()
         {
             var provider = new LocalDiskMapFileProvider();
-            var mapFiles = provider.Get(QuantConnect.Market.USA);
+            provider.Initialize(TestGlobals.DataProvider);
+            var mapFiles = provider.Get(CorporateActionsKey.EquityUsa);
             Assert.IsNotEmpty(mapFiles);
         }
 
@@ -34,8 +35,9 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
         public void CachesValueAndReturnsSameReference()
         {
             var provider = new LocalDiskMapFileProvider();
-            var mapFiles1 = provider.Get(QuantConnect.Market.USA);
-            var mapFiles2 = provider.Get(QuantConnect.Market.USA);
+            provider.Initialize(TestGlobals.DataProvider);
+            var mapFiles1 = provider.Get(CorporateActionsKey.EquityUsa);
+            var mapFiles2 = provider.Get(CorporateActionsKey.EquityUsa);
             Assert.IsTrue(ReferenceEquals(mapFiles1, mapFiles2));
         }
     }

Tests/Common/Data/Auxiliary/LocalZipMapFileProviderTests.cs

@@ -59,7 +59,7 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
             var dataProviderTest = new DefaultDataProviderTest();
             fileProviderTest.Initialize(dataProviderTest);
 
-            var mapFileResolver = fileProviderTest.Get(QuantConnect.Market.USA);
+            var mapFileResolver = fileProviderTest.Get(CorporateActionsKey.EquityUsa);
 
             fileProviderTest.Enabled = false;
             dataProviderTest.DisposeSafely();
@@ -74,15 +74,15 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
             var dataProviderTest = new DefaultDataProviderTest();
             fileProviderTest.Initialize(dataProviderTest);
 
-            fileProviderTest.Get(QuantConnect.Market.USA);
+            fileProviderTest.Get(CorporateActionsKey.EquityUsa);
             Assert.AreEqual(1, dataProviderTest.FetchCount);
             Thread.Sleep(50);
-            fileProviderTest.Get(QuantConnect.Market.USA);
+            fileProviderTest.Get(CorporateActionsKey.EquityUsa);
             Assert.AreEqual(1, dataProviderTest.FetchCount);
 
             Thread.Sleep(1000);
 
-            fileProviderTest.Get(QuantConnect.Market.USA);
+            fileProviderTest.Get(CorporateActionsKey.EquityUsa);
             Assert.AreEqual(2, dataProviderTest.FetchCount);
 
             fileProviderTest.Enabled = false;

Tests/Common/Data/Auxiliary/MapFileTests.cs

@@ -106,7 +106,7 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
                 rowParts[1],
                 rowParts[2]);
             // Act
-            var actualMapFileRow = MapFileRow.Parse(mapFileRow, QuantConnect.Market.USA);
+            var actualMapFileRow = MapFileRow.Parse(mapFileRow, QuantConnect.Market.USA, SecurityType.Equity);
             // Assert
             Assert.AreEqual(expectedExchange, actualMapFileRow.PrimaryExchange);
             Assert.AreEqual(expectedMapFileRow, actualMapFileRow);
@@ -122,7 +122,7 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
                 DateTime.ParseExact(rowParts[0], DateFormat.EightCharacter, CultureInfo.InvariantCulture),
                 rowParts[1]);
             // Act
-            var actualMapFileRow = MapFileRow.Parse(mapFileRow, QuantConnect.Market.USA);
+            var actualMapFileRow = MapFileRow.Parse(mapFileRow, QuantConnect.Market.USA, SecurityType.Equity);
             // Assert
             Assert.AreEqual(Exchange.UNKNOWN, actualMapFileRow.PrimaryExchange);
             Assert.AreEqual(expectedMapFileRow, actualMapFileRow);

Tests/Common/Securities/Futures/FuturesExpiryFunctionsTests.cs

@@ -21,6 +21,7 @@ using System.Linq;
 using System.Xml;
 using System.Xml.Serialization;
 using System.IO;
+using System.Threading.Tasks;
 using QuantConnect.Brokerages;
 using QuantConnect.Securities;
 

Tests/Common/Securities/SubscriptionDataConfigTests.cs

@@ -55,5 +55,17 @@ namespace QuantConnect.Tests.Common.Securities
                 Assert.DoesNotThrow(() => config.Consolidators.Remove(consolidator));
             }
         }
+
+        [TestCase(1, 0, DataMappingMode.OpenInterest, DataMappingMode.OpenInterest)]
+        [TestCase(0, 0, DataMappingMode.OpenInterest, DataMappingMode.FirstDayMonth)]
+        public void NotEqualsMappingAndOffset(int offsetA, int offsetB, DataMappingMode mappingModeA, DataMappingMode mappingModeB)
+        {
+            var configA = new SubscriptionDataConfig(typeof(TradeBar), Symbols.SPY, Resolution.Minute, TimeZones.NewYork,
+                TimeZones.NewYork, false, false, false, dataMappingMode: mappingModeA, contractDepthOffset: (uint)offsetA);
+            var configB = new SubscriptionDataConfig(configA, dataMappingMode: mappingModeB, contractDepthOffset: (uint)offsetB);
+
+            Assert.AreNotEqual(configA, configB);
+            Assert.AreNotEqual(configA.GetHashCode(), configB.GetHashCode());
+        }
     }
-}
+}

Tests/Common/SymbolTests.cs

@@ -523,7 +523,7 @@ namespace QuantConnect.Tests.Common
             do
             {
                 Console.WriteLine(symbolChain.ToString() + "; Value: " + symbolChain.Value);
-                if (symbolChain.SecurityType == SecurityType.Base || symbolChain.SecurityType.RequiresMapping())
+                if (symbolChain.SecurityType == SecurityType.Base || symbolChain.RequiresMapping())
                 {
                     Assert.AreEqual(mappedTicker, symbolChain.Value);
                 }

Tests/Common/Util/ExtensionsTests.cs

@@ -42,6 +42,29 @@ namespace QuantConnect.Tests.Common.Util
     [TestFixture]
     public class ExtensionsTests
     {
+        [TestCase("CL XTN6UA1G9QKH")]
+        [TestCase("ES VU1EHIDJYLMP")]
+        [TestCase("ES VRJST036ZY0X")]
+        [TestCase("GE YYBCLAZG1NGH")]
+        [TestCase("GE YTC58AEQ4C8X")]
+        [TestCase("BTC XTU2YXLMT1XD")]
+        [TestCase("UB XUIP59QUPVS5")]
+        [TestCase("NQ XUERCWA6EWAP")]
+        [TestCase("PL XVJ4OQA3JSN5")]
+        public void AdjustSymbolByOffsetTest(string future)
+        {
+            var sid = SecurityIdentifier.Parse(future);
+            var symbol = new Symbol(sid, sid.Symbol);
+
+            Assert.AreEqual(symbol.ID.Date, symbol.AdjustSymbolByOffset(0).ID.Date);
+
+            var nextExpiration = symbol.AdjustSymbolByOffset(1);
+            Assert.Greater(nextExpiration.ID.Date, symbol.ID.Date);
+
+            var nextNextExpiration = symbol.AdjustSymbolByOffset(2);
+            Assert.Greater(nextNextExpiration.ID.Date, nextExpiration.ID.Date);
+        }
+
         [TestCase("A", "a")]
         [TestCase("", "")]
         [TestCase(null, null)]
@@ -1220,7 +1243,7 @@ actualDictionary.update({'IBM': 5})
                         null
                     ),
                     new DataPermissionManager(),
-                    new DefaultDataProvider()
+                    TestGlobals.DataProvider
                 ),
                 algo,
                 new TimeKeeper(DateTime.UtcNow),
@@ -1230,7 +1253,7 @@ actualDictionary.update({'IBM': 5})
                 new DataPermissionManager()
             ));
 
-            using (var zipDataCacheProvider = new ZipDataCacheProvider(new DefaultDataProvider()))
+            using (var zipDataCacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider))
             {
                 algo.HistoryProvider = new SubscriptionDataReaderHistoryProvider();
                 algo.HistoryProvider.Initialize(
@@ -1239,8 +1262,8 @@ actualDictionary.update({'IBM': 5})
                         null,
                         null,
                         zipDataCacheProvider,
-                        new LocalDiskMapFileProvider(),
-                        new LocalDiskFactorFileProvider(),
+                        TestGlobals.MapFileProvider,
+                        TestGlobals.FactorFileProvider,
                         (_) => {},
                         false,
                         new DataPermissionManager()));

Tests/Engine/DataFeeds/Auxiliary/MapFileResolverTests.cs

@@ -32,8 +32,8 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Auxiliary
         [Test]
         public void ChecksFirstDate()
         {
-            var mapFileProvider = new LocalDiskMapFileProvider();
-            var mapFileResolver = mapFileProvider.Get(Market.USA);
+            var mapFileProvider = TestGlobals.MapFileProvider;
+            var mapFileResolver = mapFileProvider.Get(CorporateActionsKey.EquityUsa);
             // QQQ started trading on 19990310
             var mapFile = mapFileResolver.ResolveMapFile("QQQ", new DateTime(1999, 3, 9));
             Assert.IsTrue(mapFile.IsNullOrEmpty());
@@ -45,8 +45,8 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Auxiliary
         [Test]
         public void ResolvesCorrectlyReUsedTicker()
         {
-            var mapFileProvider = new LocalDiskMapFileProvider();
-            var mapFileResolver = mapFileProvider.Get(Market.USA);
+            var mapFileProvider = TestGlobals.MapFileProvider;
+            var mapFileResolver = mapFileProvider.Get(CorporateActionsKey.EquityUsa);
 
             // FB.1 started trading on 19990929 and ended on 20030328
             var mapFile = mapFileResolver.ResolveMapFile("FB", new DateTime(1999, 9, 28));
@@ -66,9 +66,9 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Auxiliary
         [Test]
         public void InitializationSpeedTest()
         {
-            var mapFileProvider = new LocalDiskMapFileProvider();
+            var mapFileProvider = TestGlobals.MapFileProvider;
             var sw = Stopwatch.StartNew();
-            var mapFileresolver = mapFileProvider.Get(Market.USA);
+            var mapFileresolver = mapFileProvider.Get(CorporateActionsKey.EquityUsa);
             sw.Stop();
             Log.Trace($"elapsed: {sw.Elapsed.TotalSeconds} seconds");
         }
@@ -202,4 +202,4 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Auxiliary
             });
         }
     }
-}
+}

Tests/Engine/DataFeeds/Enumerators/AuxiliaryDataEnumeratorTests.cs

@@ -20,6 +20,7 @@ using NUnit.Framework;
 using QuantConnect.Data;
 using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Market;
+using QuantConnect.Interfaces;
 using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
 using QuantConnect.Tests.Common.Securities;
 
@@ -82,7 +83,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
             yield return Data.Dequeue();
         }
 
-        public void Initialize(SubscriptionDataConfig config, FactorFile factorFile, MapFile mapFile, DateTime startTime)
+        public void Initialize(SubscriptionDataConfig config, IFactorFileProvider factorFileProvider, IMapFileProvider mapFileProvider, DateTime startTime)
         {
         }
     }

Tests/Engine/DataFeeds/Enumerators/DividendEventProviderTests.cs

@@ -20,7 +20,7 @@ using QuantConnect.Data;
 using System.Globalization;
 using QuantConnect.Data.Market;
 using QuantConnect.Data.Auxiliary;
-using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Interfaces;
 using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
 
 namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
@@ -58,11 +58,8 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
             var config = new SubscriptionDataConfig(typeof(TradeBar), Symbols.AAPL, Resolution.Second, TimeZones.NewYork, TimeZones.NewYork,
                 false, false, false);
 
-            var mapFile = TestGlobals.MapFileProvider;
-            var factorFile = TestGlobals.FactorFileProvider;
-
             var start = new DateTime(1998, 01, 02);
-            dividendProvider.Initialize(config, factorFile.Get(Symbols.AAPL), mapFile.Get(Market.USA).ResolveMapFile(Symbols.AAPL, start), start);
+            dividendProvider.Initialize(config, TestGlobals.FactorFileProvider, TestGlobals.MapFileProvider, start);
 
             var exDividendDate = DateTime.ParseExact(exDividendDateStr, DateFormat.EightCharacter, CultureInfo.InvariantCulture);
 
@@ -94,17 +91,19 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
             var row2 = new DateTime(2001, 01, 02);
             var row3 = new DateTime(2002, 01, 02);
 
-            var factorFile = new FactorFile("AAPL", new []
+            var factorFileProvider = new TestFactorFileProvider
             {
-                new FactorFileRow(row1, 0.693m, 1),
-                new FactorFileRow(row2, 0.77m, 1),
-                new FactorFileRow(row3, 0.85555m, 1)
-            }, start);
-            var mapFile = new LocalDiskMapFileProvider();
+                FactorFile = new FactorFile("AAPL", new []
+                {
+                    new FactorFileRow(row1, 0.693m, 1),
+                    new FactorFileRow(row2, 0.77m, 1),
+                    new FactorFileRow(row3, 0.85555m, 1)
+                }, start)
+            };
 
-            dividendProvider.Initialize(config, factorFile, mapFile.Get(Market.USA).ResolveMapFile(Symbols.AAPL, start), start);
+            dividendProvider.Initialize(config, factorFileProvider, TestGlobals.MapFileProvider, start);
 
-            foreach (var row in factorFile.Take(1))
+            foreach (var row in factorFileProvider.FactorFile.Take(1))
             {
                 var lastRawPrice = 100;
                 var events = dividendProvider
@@ -121,5 +120,18 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
                 });
             }
         }
+
+        private class TestFactorFileProvider : IFactorFileProvider
+        {
+            public FactorFile FactorFile { get; set; }
+            public void Initialize(IMapFileProvider mapFileProvider, IDataProvider dataProvider)
+            {
+            }
+
+            public FactorFile Get(Symbol symbol)
+            {
+                return FactorFile;
+            }
+        }
     }
 }

Tests/Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactoryTests.cs

@@ -46,12 +46,8 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
                 new SecurityCache()
             );
 
-            var mapFileProvider = TestGlobals.MapFileProvider;
             var fileProvider = TestGlobals.DataProvider;
-            var factorFileProvider = TestGlobals.FactorFileProvider;
-            var mapFileResolver = mapFileProvider.Get(security.Symbol.ID.Market);
-
-            var factory = new BaseDataSubscriptionEnumeratorFactory(false, mapFileResolver, factorFileProvider);
+            var factory = new BaseDataSubscriptionEnumeratorFactory(false, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider);
 
             GC.Collect();
             var ramUsageBeforeLoop = OS.TotalPhysicalMemoryUsed;

Tests/Engine/DataFeeds/Enumerators/Factories/OptionChainUniverseSubscriptionEnumeratorFactoryTests.cs

@@ -71,7 +71,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
             );
 
             var dataProvider = TestGlobals.DataProvider;
-            var enumeratorFactory = new BaseDataSubscriptionEnumeratorFactory(false, MapFileResolver.Create(Globals.DataFolder, Market.USA), TestGlobals.FactorFileProvider);
+            var enumeratorFactory = new BaseDataSubscriptionEnumeratorFactory(false, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider);
             var fillForwardResolution = Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan());
             Func<SubscriptionRequest, IEnumerator<BaseData>> underlyingEnumeratorFunc = (req) =>
                 {
@@ -90,7 +90,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
             var factory = new OptionChainUniverseSubscriptionEnumeratorFactory(underlyingEnumeratorFunc);
 
             var request = new SubscriptionRequest(true, null, option, config, startTime, endTime);
-            var enumerator = factory.CreateEnumerator(request, new DefaultDataProvider());
+            var enumerator = factory.CreateEnumerator(request, dataProvider);
 
             var emittedCount = 0;
             foreach (var data in enumerator.AsEnumerable())
@@ -165,7 +165,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
             var universeSettings = new UniverseSettings(Resolution.Minute, 0, true, false, TimeSpan.Zero);
             var universe = new OptionChainUniverse(option, universeSettings, true);
             var request = new SubscriptionRequest(true, universe, option, config, startTime, Time.EndOfTime);
-            var enumerator = (DataQueueOptionChainUniverseDataCollectionEnumerator) factory.CreateEnumerator(request, new DefaultDataProvider());
+            var enumerator = (DataQueueOptionChainUniverseDataCollectionEnumerator) factory.CreateEnumerator(request, TestGlobals.DataProvider);
 
             // 2018-10-19 10:00 AM UTC
             underlyingEnumerator.Enqueue(new Tick { Symbol = Symbols.SPY, Value = 280m });

Tests/Engine/DataFeeds/Enumerators/LiveAuxiliaryDataEnumeratorTests.cs

@@ -0,0 +1,212 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Linq;
+using NUnit.Framework;
+using QuantConnect.Data;
+using System.Globalization;
+using QuantConnect.Securities;
+using QuantConnect.Interfaces;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
+using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
+
+namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
+{
+    [TestFixture]
+    public class LiveAuxiliaryDataEnumeratorTests
+    {
+        [TestCase(DataMappingMode.OpenInterest, "20130616", false)]
+        [TestCase(DataMappingMode.FirstDayMonth, "20130602", false)]
+        [TestCase(DataMappingMode.LastTradingDay, "20130623", false)]
+        [TestCase(DataMappingMode.OpenInterest, "20130616", true)]
+        [TestCase(DataMappingMode.FirstDayMonth, "20130602", true)]
+        [TestCase(DataMappingMode.LastTradingDay, "20130623", true)]
+        public void EmitsMappingEventsBasedOnCurrentMapFileAndTime(DataMappingMode dataMappingMode, string mappingDate, bool delayed)
+        {
+            var config = new SubscriptionDataConfig(typeof(TradeBar),
+                Symbols.ES_Future_Chain,
+                Resolution.Daily,
+                TimeZones.NewYork,
+                TimeZones.NewYork,
+                true,
+                true,
+                false,
+                dataMappingMode: dataMappingMode);
+            var symbolMaps = new List<SubscriptionDataConfig.NewSymbolEventArgs>();
+            config.NewSymbol += (sender, args) => symbolMaps.Add(args);
+            var time = new DateTime(2013, 05, 28);
+            var cache = new SecurityCache();
+
+            cache.AddData(new Tick(time, config.Symbol, 20, 10));
+            var timeProvider = new ManualTimeProvider(time);
+
+            var futureTicker1 = "es vhle2yxr5blt";
+            TestMapFileResolver.MapFile = new MapFile(Futures.Indices.SP500EMini, new []
+            {
+                new MapFileRow(Time.BeginningOfTime, Futures.Indices.SP500EMini, Exchange.CME),
+                new MapFileRow(new DateTime(2013,06,01), futureTicker1, Exchange.CME, DataMappingMode.FirstDayMonth),
+                new MapFileRow(new DateTime(2013,06,15), futureTicker1, Exchange.CME, DataMappingMode.OpenInterest),
+                new MapFileRow(new DateTime(2013,06,22), futureTicker1, Exchange.CME, DataMappingMode.LastTradingDay),
+            });
+
+            IEnumerator<BaseData> enumerator;
+            Assert.IsTrue(LiveAuxiliaryDataEnumerator.TryCreate(config, timeProvider, null, cache, new TestMapFileProvider(), TestGlobals.FactorFileProvider, time, out enumerator));
+
+            // get's mapped right away!
+            Assert.AreEqual(futureTicker1.ToUpper(), config.MappedSymbol);
+
+            Assert.AreEqual(1, symbolMaps.Count);
+            Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[0].Old);
+            Assert.AreEqual(Futures.Indices.SP500EMini, symbolMaps[0].Old.ID.Symbol);
+            Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[0].New);
+            Assert.AreEqual(futureTicker1.ToUpper(), symbolMaps[0].New.Underlying.ID.ToString());
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            var expectedMappingDate = DateTime.ParseExact(mappingDate, DateFormat.EightCharacter, CultureInfo.InvariantCulture);
+            if (delayed)
+            {
+                // we advance to the mapping date, without any new mapFile!
+                timeProvider.Advance(expectedMappingDate.ConvertToUtc(config.ExchangeTimeZone) - timeProvider.GetUtcNow());
+            }
+            else
+            {
+                // just advance a day to show nothing happens until mapping time
+                timeProvider.Advance(TimeSpan.FromDays(1));
+            }
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            var futureTicker2 = "es vk2zrh843z7l";
+            TestMapFileResolver.MapFile = new MapFile(Futures.Indices.SP500EMini, TestMapFileResolver.MapFile.Concat(
+                new []
+                {
+                    new MapFileRow(new DateTime(2013,09,01), futureTicker2, Exchange.CME, DataMappingMode.FirstDayMonth),
+                    new MapFileRow(new DateTime(2013,09,14), futureTicker2, Exchange.CME, DataMappingMode.OpenInterest),
+                    new MapFileRow(new DateTime(2013,09,21), futureTicker2, Exchange.CME, DataMappingMode.LastTradingDay),
+                }));
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            if (delayed)
+            {
+                // we got a new mapFile! advance the date and expect mapping to have happened
+                timeProvider.Advance(TimeSpan.FromDays(1));
+            }
+            else
+            {
+                // we advance to the mapping date
+                timeProvider.Advance(expectedMappingDate.ConvertToUtc(config.ExchangeTimeZone) - timeProvider.GetUtcNow());
+            }
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNotNull(enumerator.Current);
+
+            Assert.AreEqual(2, symbolMaps.Count);
+            Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[1].Old);
+            Assert.AreEqual(futureTicker1.ToUpper(), symbolMaps[1].Old.Underlying.ID.ToString());
+            Assert.AreEqual(Symbols.ES_Future_Chain, symbolMaps[1].New);
+            Assert.AreEqual(futureTicker2.ToUpper(), symbolMaps[1].New.Underlying.ID.ToString());
+
+            Assert.AreEqual(futureTicker2.ToUpper(), config.MappedSymbol);
+
+            Assert.AreEqual(futureTicker2.ToUpper(), (enumerator.Current as SymbolChangedEvent).NewSymbol);
+            Assert.AreEqual(futureTicker1.ToUpper(), (enumerator.Current as SymbolChangedEvent).OldSymbol);
+            Assert.AreEqual(config.Symbol, (enumerator.Current as SymbolChangedEvent).Symbol);
+            Assert.AreEqual(timeProvider.GetUtcNow().Date, (enumerator.Current as SymbolChangedEvent).Time);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+        }
+
+        [Test]
+        public void EmitsDelistingEventsBasedOnCurrentTime()
+        {
+            var config = new SubscriptionDataConfig(typeof(TradeBar),
+                Symbols.SPY_C_192_Feb19_2016,
+                Resolution.Daily,
+                TimeZones.NewYork,
+                TimeZones.NewYork,
+                true,
+                true,
+                false);
+            var delistingDate = config.Symbol.GetDelistingDate();
+            var time = delistingDate.AddDays(-10);
+            var cache = new SecurityCache();
+            cache.AddData(new Tick(DateTime.UtcNow, config.Symbol, 20, 10));
+            var timeProvider = new ManualTimeProvider(time);
+
+            IEnumerator<BaseData> enumerator;
+            Assert.IsTrue(LiveAuxiliaryDataEnumerator.TryCreate(config, timeProvider, null, cache, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, config.Symbol.ID.Date, out enumerator));
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            // advance until delisting date, take into account 5 hour offset of NY
+            timeProvider.Advance(TimeSpan.FromDays(10));
+            timeProvider.Advance(TimeSpan.FromHours(5));
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.AreEqual(DelistingType.Warning, (enumerator.Current as Delisting).Type);
+            Assert.AreEqual(config.Symbol, (enumerator.Current as Delisting).Symbol);
+            Assert.AreEqual(delistingDate, (enumerator.Current as Delisting).Time);
+            Assert.AreEqual(15, (enumerator.Current as Delisting).Price);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+
+            // when the day ends the delisted event will pass through
+            timeProvider.Advance(TimeSpan.FromDays(1));
+            cache.AddData(new Tick(DateTime.UtcNow, config.Symbol, 40, 20));
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.AreEqual(DelistingType.Delisted, (enumerator.Current as Delisting).Type);
+            Assert.AreEqual(config.Symbol, (enumerator.Current as Delisting).Symbol);
+            Assert.AreEqual(delistingDate.AddDays(1), (enumerator.Current as Delisting).Time);
+            Assert.AreEqual(30, (enumerator.Current as Delisting).Price);
+
+            Assert.IsTrue(enumerator.MoveNext());
+            Assert.IsNull(enumerator.Current);
+        }
+
+        private class TestMapFileProvider : IMapFileProvider
+        {
+            public void Initialize(IDataProvider dataProvider)
+            {
+            }
+
+            public MapFileResolver Get(CorporateActionsKey corporateActionsKey)
+            {
+                return new TestMapFileResolver();
+            }
+        }
+
+        private class TestMapFileResolver : MapFileResolver
+        {
+            public static MapFile MapFile { get; set; }
+            public TestMapFileResolver()
+                : base(new [] { MapFile })
+            {
+            }
+        }
+    }
+}

Tests/Engine/DataFeeds/Enumerators/LiveDataBasedDelistingEventProviderTests.cs

@@ -57,8 +57,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
                 return Enumerable.Empty<BaseData>();
             }, timeProvider);
             var provider = new LiveDataBasedDelistingEventProvider(config, dataQueueHandler);
-            var mapFile = TestGlobals.MapFileProvider.Get(config.Symbol.ID.Market).ResolveMapFile(config.Symbol, config.Type);
-            provider.Initialize(config, null, mapFile, time);
+            provider.Initialize(config, TestGlobals.FactorFileProvider, TestGlobals.MapFileProvider, time);
             Assert.IsTrue(autoResetEvent.WaitOne(TimeSpan.FromMilliseconds(100)));
 
             var events = provider.GetEvents(new NewTradableDateEventArgs(time, null, Symbols.AAPL, null)).ToList();

Tests/Engine/DataFeeds/Enumerators/LiveDelistingEventProviderEnumeratorTests.cs

@@ -1,80 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using NUnit.Framework;
-using QuantConnect.Data;
-using QuantConnect.Securities;
-using QuantConnect.Data.Market;
-using System.Collections.Generic;
-using QuantConnect.Lean.Engine.DataFeeds;
-using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
-
-namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
-{
-    [TestFixture]
-    public class LiveDelistingEventProviderEnumeratorTests
-    {
-        [Test]
-        public void EmitsDelistingEventsBasedOnCurrentTime()
-        {
-            var config = new SubscriptionDataConfig(typeof(TradeBar),
-                Symbols.SPY_C_192_Feb19_2016,
-                Resolution.Daily,
-                TimeZones.NewYork,
-                TimeZones.NewYork,
-                true,
-                true,
-                false);
-            var delistingDate = config.Symbol.GetDelistingDate();
-            var time = delistingDate.AddDays(-10);
-            var cache = new SecurityCache();
-            cache.AddData(new Tick(DateTime.UtcNow, config.Symbol, 20, 10));
-            var timeProvider = new ManualTimeProvider(time);
-
-            IEnumerator<BaseData> enumerator;
-            Assert.IsTrue(LiveDelistingEventProviderEnumerator.TryCreate(config, timeProvider, null, cache, TestGlobals.MapFileProvider, out enumerator));
-
-            Assert.IsFalse(enumerator.MoveNext());
-            Assert.IsNull(enumerator.Current);
-
-            // advance until delisting date, take into account 5 hour offset of NY
-            timeProvider.Advance(TimeSpan.FromDays(10));
-            timeProvider.Advance(TimeSpan.FromHours(5));
-
-            Assert.IsTrue(enumerator.MoveNext());
-            Assert.AreEqual(DelistingType.Warning, (enumerator.Current as Delisting).Type);
-            Assert.AreEqual(config.Symbol, (enumerator.Current as Delisting).Symbol);
-            Assert.AreEqual(delistingDate, (enumerator.Current as Delisting).Time);
-            Assert.AreEqual(15, (enumerator.Current as Delisting).Price);
-
-            Assert.IsFalse(enumerator.MoveNext());
-            Assert.IsNull(enumerator.Current);
-
-            // when the day ends the delisted event will pass through
-            timeProvider.Advance(TimeSpan.FromDays(1));
-            cache.AddData(new Tick(DateTime.UtcNow, config.Symbol, 40, 20));
-
-            Assert.IsTrue(enumerator.MoveNext());
-            Assert.AreEqual(DelistingType.Delisted, (enumerator.Current as Delisting).Type);
-            Assert.AreEqual(config.Symbol, (enumerator.Current as Delisting).Symbol);
-            Assert.AreEqual(delistingDate.AddDays(1), (enumerator.Current as Delisting).Time);
-            Assert.AreEqual(30, (enumerator.Current as Delisting).Price);
-
-            Assert.IsFalse(enumerator.MoveNext());
-            Assert.IsNull(enumerator.Current);
-        }
-    }
-}

Tests/Engine/DataFeeds/Enumerators/LiveEquityDataSynchronizingEnumeratorTests.cs

@@ -15,9 +15,11 @@
 */
 
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using NodaTime;
 using NUnit.Framework;
+using QuantConnect.Data;
 using QuantConnect.Data.Market;
 using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
 using QuantConnect.Logging;
@@ -45,7 +47,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
             var count1 = 0;
             var count2 = 0;
             var previous = DateTime.MinValue;
-            var synchronizer = new LiveAuxiliaryDataSynchronizingEnumerator(new RealTimeProvider(), DateTimeZone.Utc, stream1, stream2);
+            var synchronizer = new LiveAuxiliaryDataSynchronizingEnumerator(new RealTimeProvider(), DateTimeZone.Utc, stream1, new List<IEnumerator<BaseData>> { stream2 });
             while (synchronizer.MoveNext() && DateTime.UtcNow < end)
             {
                 if (synchronizer.Current != null)

Tests/Engine/DataFeeds/Enumerators/MappingEventProviderTests.cs

@@ -18,7 +18,6 @@ using System;
 using System.Linq;
 using NUnit.Framework;
 using QuantConnect.Data;
-using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Market;
 using QuantConnect.Lean.Engine.DataFeeds.Enumerators;
 
@@ -53,7 +52,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
 
             provider.Initialize(_config,
                 null,
-                MapFile.Read("TFCFA", Market.USA),
+                TestGlobals.MapFileProvider,
                 new DateTime(2006, 1, 1));
 
             Assert.AreEqual("NWSA", _config.MappedSymbol);
@@ -65,7 +64,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
             var provider = new MappingEventProvider();
             provider.Initialize(_config,
                 null,
-                MapFile.Read("TFCFA", Market.USA),
+                TestGlobals.MapFileProvider,
                 new DateTime(2006, 1, 1));
 
             Assert.AreEqual("NWSA", _config.MappedSymbol);

Tests/Engine/DataFeeds/Enumerators/PriceScaleFactorEnumeratorTests.cs

@@ -31,7 +31,6 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
     public class PriceScaleFactorEnumeratorTests
     {
         private SubscriptionDataConfig _config;
-        private FactorFile _factorFile;
         private RawDataEnumerator _rawDataEnumerator;
 
         [SetUp]
@@ -46,7 +45,6 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
                 true,
                 false);
 
-            _factorFile = TestGlobals.FactorFileProvider.Get(_config.Symbol);
             _rawDataEnumerator = new RawDataEnumerator();
         }
 
@@ -56,7 +54,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
             var enumerator = new PriceScaleFactorEnumerator(
                 _rawDataEnumerator,
                 _config,
-                new Lazy<FactorFile>(() => _factorFile));
+                TestGlobals.FactorFileProvider);
             _rawDataEnumerator.CurrentValue = new TradeBar(
                 new DateTime(2018, 1, 1),
                 _config.Symbol,
@@ -86,7 +84,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
             var enumerator = new PriceScaleFactorEnumerator(
                 _rawDataEnumerator,
                 _config,
-                new Lazy<FactorFile>(() => _factorFile));
+                TestGlobals.FactorFileProvider);
             _rawDataEnumerator.CurrentValue = new QuoteBar(
                 new DateTime(2018, 1, 1),
                 _config.Symbol,
@@ -126,7 +124,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
             var enumerator = new PriceScaleFactorEnumerator(
                 _rawDataEnumerator,
                 _config,
-                new Lazy<FactorFile>(() => _factorFile));
+                TestGlobals.FactorFileProvider);
             _rawDataEnumerator.CurrentValue = new Tick(
                 new DateTime(2018, 1, 1),
                 _config.Symbol,
@@ -171,7 +169,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
             var enumerator = new PriceScaleFactorEnumerator(
                 _rawDataEnumerator,
                 _config,
-                new Lazy<FactorFile>(() => _factorFile));
+                TestGlobals.FactorFileProvider);
             _rawDataEnumerator.CurrentValue = new Tick(
                 new DateTime(2018, 1, 1),
                 _config.Symbol,
@@ -191,7 +189,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
             var enumerator = new PriceScaleFactorEnumerator(
                 _rawDataEnumerator,
                 _config,
-                new Lazy<FactorFile>(() => _factorFile));
+                TestGlobals.FactorFileProvider);
             _rawDataEnumerator.CurrentValue = null;
             Assert.IsTrue(enumerator.MoveNext());
             Assert.IsNull(enumerator.Current);
@@ -209,7 +207,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
             var enumerator = new PriceScaleFactorEnumerator(
                 _rawDataEnumerator,
                 _config,
-                new Lazy<FactorFile>(() => _factorFile));
+                TestGlobals.FactorFileProvider);
 
             // Before factor file update date (2018, 3, 15)
             _rawDataEnumerator.CurrentValue = new Tick(
@@ -220,7 +218,8 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
                 10);
 
             Assert.IsTrue(enumerator.MoveNext());
-            var expectedFactor = _factorFile.GetPriceScaleFactor(dateBeforeUpadate);
+            var factorFile = TestGlobals.FactorFileProvider.Get(_config.Symbol);
+            var expectedFactor = factorFile.GetPriceScaleFactor(dateBeforeUpadate);
             var tick = enumerator.Current as Tick;
             Assert.AreEqual(expectedFactor, _config.PriceScaleFactor);
             Assert.AreEqual(10 * expectedFactor, tick.Price);
@@ -234,7 +233,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
                 10,
                 10);
             Assert.IsTrue(enumerator.MoveNext());
-            var expectedFactor2 = _factorFile.GetPriceScaleFactor(dateAtUpadate);
+            var expectedFactor2 = factorFile.GetPriceScaleFactor(dateAtUpadate);
             var tick2 = enumerator.Current as Tick;
             Assert.AreEqual(expectedFactor2, _config.PriceScaleFactor);
             Assert.AreEqual(10 * expectedFactor2, tick2.Price);
@@ -248,7 +247,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
                 10,
                 10);
             Assert.IsTrue(enumerator.MoveNext());
-            var expectedFactor3 = _factorFile.GetPriceScaleFactor(dateAfterUpadate);
+            var expectedFactor3 = factorFile.GetPriceScaleFactor(dateAfterUpadate);
             var tick3 = enumerator.Current as Tick;
             Assert.AreEqual(expectedFactor3, _config.PriceScaleFactor);
             Assert.AreEqual(10 * expectedFactor3, tick3.Price);

Tests/Engine/DataFeeds/LiveTradingDataFeedTests.cs

@@ -2134,9 +2134,8 @@ namespace QuantConnect.Tests.Engine.DataFeeds
                 new SecurityCacheProvider(algorithm.Portfolio));
             algorithm.Securities.SetSecurityService(securityService);
             var dataPermissionManager = new DataPermissionManager();
-            var dataProvider = new DefaultDataProvider();
             var dataManager = new DataManager(_feed,
-                new UniverseSelection(algorithm, securityService, dataPermissionManager, dataProvider),
+                new UniverseSelection(algorithm, securityService, dataPermissionManager, TestGlobals.DataProvider),
                 algorithm,
                 algorithm.TimeKeeper,
                 marketHoursDatabase,
@@ -2175,7 +2174,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds
             }
 
             _feed.Initialize(algorithm, new LiveNodePacket(), new BacktestingResultHandler(),
-                TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, dataProvider,
+                TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider,
                 dataManager, _synchronizer, new TestDataChannelProvider());
 
             var cancellationTokenSource = new CancellationTokenSource();
@@ -2241,7 +2240,8 @@ namespace QuantConnect.Tests.Engine.DataFeeds
                 {
                     if (securityType == SecurityType.Future)
                     {
-                        Assert.AreEqual(futureSymbols.Count, futureContractCount);
+                        // -1 to remove canonical since it's not part of the chain
+                        Assert.AreEqual(futureSymbols.Count - 1, futureContractCount);
 
                         foreach (var symbol in futureSymbols)
                         {
@@ -2322,7 +2322,8 @@ namespace QuantConnect.Tests.Engine.DataFeeds
 
             if (securityType == SecurityType.Future)
             {
-                Assert.AreEqual(2, futureSymbols.Count, "Future symbols count mismatch");
+                // we add 2 symbols + 1 continuous future
+                Assert.AreEqual(3, futureSymbols.Count, "Future symbols count mismatch");
             }
             else if (securityType == SecurityType.Option)
             {

Tests/Engine/DataFeeds/SubscriptionDataReaderTests.cs

@@ -46,9 +46,6 @@ namespace QuantConnect.Tests.Engine.DataFeeds
             var start = new DateTime(2019, 12, 9);
             var end = new DateTime(2019, 12, 12);
 
-            var mapFileProvider = TestGlobals.MapFileProvider;
-            var mapFileResolver = new MapFileResolver(mapFileProvider.Get(Market.USA));
-
             var dataReader = new SubscriptionDataReader(
                 new SubscriptionDataConfig(typeof(TradeBar),
                     Symbols.SPY,
@@ -60,7 +57,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds
                     false),
                 start,
                 end,
-                mapFileResolver,
+                TestGlobals.MapFileProvider,
                 TestGlobals.FactorFileProvider,
                 LinqExtensions.Range(start, end, time => time + TimeSpan.FromDays(1)),
                 false,

Tests/Engine/HistoricalData/SubscriptionDataReaderHistoryProviderTests.cs

@@ -35,7 +35,7 @@ namespace QuantConnect.Tests.Engine.HistoricalData
         public void OptionsAreMappedCorrectly()
         {
             var historyProvider = new SubscriptionDataReaderHistoryProvider();
-            var zipCache = new ZipDataCacheProvider(new DefaultDataProvider());
+            var zipCache = new ZipDataCacheProvider(TestGlobals.DataProvider);
 
             historyProvider.Initialize(new HistoryProviderInitializeParameters(
                 null,
@@ -90,14 +90,14 @@ namespace QuantConnect.Tests.Engine.HistoricalData
         public void EquitiesAreMappedCorrectly()
         {
             var historyProvider = new SubscriptionDataReaderHistoryProvider();
-            var zipCache = new ZipDataCacheProvider(new DefaultDataProvider());
+            var zipCache = new ZipDataCacheProvider(TestGlobals.DataProvider);
             historyProvider.Initialize(new HistoryProviderInitializeParameters(
                 null,
                 null,
-                new DefaultDataProvider(),
+                TestGlobals.DataProvider,
                 zipCache,
-                new LocalDiskMapFileProvider(),
-                new LocalDiskFactorFileProvider(),
+               TestGlobals.MapFileProvider,
+                TestGlobals.FactorFileProvider,
                 null,
                 false,
                 new DataPermissionManager()));

Tests/QuantConnect.Tests.csproj

@@ -46,7 +46,7 @@
     </PackageReference>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.9.4" />
     <PackageReference Include="Microsoft.TestPlatform.ObjectModel" Version="16.9.4" />
-    <PackageReference Include="Moq" Version="4.7.63" />
+    <PackageReference Include="Moq" Version="4.16.1" />
     <PackageReference Include="NetMQ" Version="4.0.1.6" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
     <PackageReference Include="NodaTime" Version="3.0.5" />

ToolBox/CoarseUniverseGenerator/CoarseUniverseGeneratorProgram.cs

@@ -113,7 +113,7 @@ namespace QuantConnect.ToolBox.CoarseUniverseGenerator
             var dailyFilesNotFound = 0;
             var coarseFilesGenerated = 0;
 
-            var mapFileResolver = _mapFileProvider.Get(_market);
+            var mapFileResolver = _mapFileProvider.Get(new CorporateActionsKey(_market, SecurityType.Equity));
 
             var blackListedTickers = new HashSet<string>();
             if (_blackListedTickersFile.Exists)