Refactor price factors scaling

- Add support for live mapping, refreshing mapfiles
- Fix future expiration functions
- Adding unit tests

Algorithm.CSharp/AdjustedVolumeRegressionAlgorithm.cs

@@ -30,7 +30,7 @@ namespace QuantConnect.Algorithm.CSharp
     {
         private Symbol _aapl;
         private const string Ticker = "AAPL";
-        private FactorFile _factorFile;
+        private CorporateFactorProvider _factorFile;
         private readonly IEnumerator<decimal> _expectedAdjustedVolume = new List<decimal> { 6164842, 3044047, 3680347, 3468303, 2169943, 2652523,
             1499707, 1518215, 1655219, 1510487 }.GetEnumerator();
         private readonly IEnumerator<decimal> _expectedAdjustedAskSize = new List<decimal> { 215600, 5600, 25200, 8400, 5600, 5600, 2800,
@@ -56,7 +56,7 @@ namespace QuantConnect.Algorithm.CSharp
             factorFileProvider.Initialize(mapFileProvider, dataProvider);
 
 
-            _factorFile = factorFileProvider.Get(_aapl);
+            _factorFile = factorFileProvider.Get(_aapl) as CorporateFactorProvider;
         }
 
         /// <summary>
@@ -83,7 +83,7 @@ namespace QuantConnect.Algorithm.CSharp
                 if (_expectedAdjustedVolume.MoveNext() && _expectedAdjustedVolume.Current != aaplData.Volume)
                 {
                     // Our values don't match lets try and give a reason why
-                    var dayFactor = _factorFile.GetSplitFactor(aaplData.Time);
+                    var dayFactor = _factorFile.GetPriceScale(aaplData.Time, DataNormalizationMode.SplitAdjusted);
                     var probableAdjustedVolume = aaplData.Volume / dayFactor;
 
                     if (_expectedAdjustedVolume.Current == probableAdjustedVolume)
@@ -107,7 +107,7 @@ namespace QuantConnect.Algorithm.CSharp
                 if (_expectedAdjustedAskSize.MoveNext() && _expectedAdjustedAskSize.Current != aaplQuoteData.LastAskSize)
                 {
                     // Our values don't match lets try and give a reason why
-                    var dayFactor = _factorFile.GetSplitFactor(aaplQuoteData.Time);
+                    var dayFactor = _factorFile.GetPriceScale(aaplQuoteData.Time, DataNormalizationMode.SplitAdjusted);
                     var probableAdjustedAskSize = aaplQuoteData.LastAskSize / dayFactor;
 
                     if (_expectedAdjustedAskSize.Current == probableAdjustedAskSize)
@@ -126,7 +126,7 @@ namespace QuantConnect.Algorithm.CSharp
                 if (_expectedAdjustedBidSize.MoveNext() && _expectedAdjustedBidSize.Current != aaplQuoteData.LastBidSize)
                 {
                     // Our values don't match lets try and give a reason why
-                    var dayFactor = _factorFile.GetSplitFactor(aaplQuoteData.Time);
+                    var dayFactor = _factorFile.GetPriceScale(aaplQuoteData.Time, DataNormalizationMode.SplitAdjusted);
                     var probableAdjustedBidSize = aaplQuoteData.LastBidSize / dayFactor;
 
                     if (_expectedAdjustedBidSize.Current == probableAdjustedBidSize)

Algorithm.CSharp/BasicTemplateFuturesAlgorithm.cs

@@ -131,30 +131,30 @@ namespace QuantConnect.Algorithm.CSharp
             {"Average Win", "0.00%"},
             {"Average Loss", "0.00%"},
             {"Compounding Annual Return", "-100.000%"},
-            {"Drawdown", "13.500%"},
+            {"Drawdown", "13.400%"},
             {"Expectancy", "-0.818"},
-            {"Net Profit", "-13.517%"},
-            {"Sharpe Ratio", "-98.781"},
+            {"Net Profit", "-13.418%"},
+            {"Sharpe Ratio", "-321.172"},
             {"Probabilistic Sharpe Ratio", "0%"},
             {"Loss Rate", "89%"},
             {"Win Rate", "11%"},
             {"Profit-Loss Ratio", "0.69"},
-            {"Alpha", "-1.676"},
-            {"Beta", "0.042"},
-            {"Annual Standard Deviation", "0.01"},
+            {"Alpha", "-1.208"},
+            {"Beta", "0.013"},
+            {"Annual Standard Deviation", "0.003"},
             {"Annual Variance", "0"},
-            {"Information Ratio", "-73.981"},
-            {"Tracking Error", "0.233"},
-            {"Treynor Ratio", "-23.975"},
+            {"Information Ratio", "-71.816"},
+            {"Tracking Error", "0.24"},
+            {"Treynor Ratio", "-77.951"},
             {"Total Fees", "$15207.00"},
             {"Estimated Strategy Capacity", "$8000.00"},
             {"Lowest Capacity Asset", "GC VOFJUCDY9XNH"},
-            {"Fitness Score", "0.033"},
+            {"Fitness Score", "0.031"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "-8.62"},
-            {"Return Over Maximum Drawdown", "-7.81"},
-            {"Portfolio Turnover", "302.321"},
+            {"Sortino Ratio", "-9.206"},
+            {"Return Over Maximum Drawdown", "-7.871"},
+            {"Portfolio Turnover", "302.123"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
             {"Total Insights Analysis Completed", "0"},
@@ -168,7 +168,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "35b3f4b7a225468d42ca085386a2383e"}
+            {"OrderListHash", "9e50b7d8e41033110f927658e731f4c6"}
         };
     }
 }

Algorithm.CSharp/ContinuousBackMonthRawFutureRegressionAlgorithm.cs

@@ -0,0 +1,184 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Orders;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+using QuantConnect.Securities.Future;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Continuous Back Month Raw Futures Regression algorithm. Asserting and showcasing the behavior of adding a continuous future
+    /// </summary>
+    public class ContinuousBackMonthRawFutureRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        private List<SymbolChangedEvent> _mappings = new();
+        private Future _continuousContract;
+        private DateTime _lastDateLog;
+
+        /// <summary>
+        /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
+        /// </summary>
+        public override void Initialize()
+        {
+            SetStartDate(2013, 7, 1);
+            SetEndDate(2014, 1, 1);
+
+            _continuousContract = AddFuture(Futures.Indices.SP500EMini,
+                dataNormalizationMode: DataNormalizationMode.Raw,
+                dataMappingMode: DataMappingMode.FirstDayMonth,
+                contractDepthOffset: 1
+            );
+        }
+
+        /// <summary>
+        /// OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
+        /// </summary>
+        /// <param name="data">Slice object keyed by symbol containing the stock data</param>
+        public override void OnData(Slice data)
+        {
+            if (data.Keys.Count != 1)
+            {
+                throw new Exception($"We are getting data for more than one symbols! {string.Join(",", data.Keys.Select(symbol => symbol))}");
+            }
+
+            foreach (var changedEvent in data.SymbolChangedEvents.Values)
+            {
+                if (changedEvent.Symbol == _continuousContract.Symbol)
+                {
+                    _mappings.Add(changedEvent);
+                    Log($"SymbolChanged event: {changedEvent}");
+
+                    var currentExpiration = changedEvent.Symbol.Underlying.ID.Date;
+                    // +4 months cause we are actually using the back month, es is quarterly contract
+                    var frontMonthExpiration = FuturesExpiryFunctions.FuturesExpiryFunction(_continuousContract.Symbol)(Time.AddMonths(1 + 4));
+
+                    if (currentExpiration != frontMonthExpiration.Date)
+                    {
+                        throw new Exception($"Unexpected current mapped contract expiration {currentExpiration}" +
+                            $" @ {Time} it should be AT front month expiration {frontMonthExpiration}");
+                    }
+                }
+            }
+
+            if (_lastDateLog.Month != Time.Month)
+            {
+                _lastDateLog = Time;
+
+                Log($"{Time}- {Securities[_continuousContract.Symbol].GetLastData()}");
+                if (Portfolio.Invested)
+                {
+                    Liquidate();
+                }
+                else if(_continuousContract.HasData)
+                {
+                    // This works because we set this contract as tradable, even if it's a canonical security
+                    Buy(_continuousContract.Symbol, 1);
+                }
+
+                if(Time.Month == 1 && Time.Year == 2013)
+                {
+                    var response = History(new[] { _continuousContract.Symbol }, 60 * 24 * 90);
+                    if (!response.Any())
+                    {
+                        throw new Exception("Unexpected empty history response");
+                    }
+                }
+            }
+        }
+
+        public override void OnOrderEvent(OrderEvent orderEvent)
+        {
+            if (orderEvent.Status == OrderStatus.Filled)
+            {
+                Log($"{orderEvent}");
+            }
+        }
+
+        public override void OnEndOfAlgorithm()
+        {
+            var expectedMappingCounts = 2;
+            if (_mappings.Count != expectedMappingCounts)
+            {
+                throw new Exception($"Unexpected symbol changed events: {_mappings.Count}, was expecting {expectedMappingCounts}");
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public Language[] Languages { get; } = { Language.CSharp };
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Trades", "2"},
+            {"Average Win", "1.11%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "2.199%"},
+            {"Drawdown", "1.700%"},
+            {"Expectancy", "0"},
+            {"Net Profit", "1.109%"},
+            {"Sharpe Ratio", "0.717"},
+            {"Probabilistic Sharpe Ratio", "38.157%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "100%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "-0.007"},
+            {"Beta", "0.099"},
+            {"Annual Standard Deviation", "0.022"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "-2.732"},
+            {"Tracking Error", "0.076"},
+            {"Treynor Ratio", "0.156"},
+            {"Total Fees", "$3.70"},
+            {"Estimated Strategy Capacity", "$3900000.00"},
+            {"Lowest Capacity Asset", "ES 1S1"},
+            {"Fitness Score", "0.007"},
+            {"Kelly Criterion Estimate", "0"},
+            {"Kelly Criterion Probability Value", "0"},
+            {"Sortino Ratio", "0.484"},
+            {"Return Over Maximum Drawdown", "1.736"},
+            {"Portfolio Turnover", "0.011"},
+            {"Total Insights Generated", "0"},
+            {"Total Insights Closed", "0"},
+            {"Total Insights Analysis Completed", "0"},
+            {"Long Insight Count", "0"},
+            {"Short Insight Count", "0"},
+            {"Long/Short Ratio", "100%"},
+            {"Estimated Monthly Alpha Value", "$0"},
+            {"Total Accumulated Estimated Alpha Value", "$0"},
+            {"Mean Population Estimated Insight Value", "$0"},
+            {"Mean Population Direction", "0%"},
+            {"Mean Population Magnitude", "0%"},
+            {"Rolling Averaged Population Direction", "0%"},
+            {"Rolling Averaged Population Magnitude", "0%"},
+            {"OrderListHash", "7d6fb409115f2f8d403c7eb261b9b3b6"}
+        };
+    }
+}

Algorithm.CSharp/ContinuousFutureBackMonthRegressionAlgorithm.cs

@@ -0,0 +1,197 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Orders;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+using QuantConnect.Securities.Future;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Continuous Futures Back Month #1 Regression algorithm. Asserting and showcasing the behavior of adding a continuous future
+    /// </summary>
+    public class ContinuousFutureBackMonthRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        private List<SymbolChangedEvent> _mappings = new();
+        private Future _continuousContract;
+        private DateTime _lastDateLog;
+
+        /// <summary>
+        /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
+        /// </summary>
+        public override void Initialize()
+        {
+            SetStartDate(2013, 7, 1);
+            SetEndDate(2014, 1, 1);
+
+            try
+            {
+                AddFuture(Futures.Indices.SP500EMini,
+                    dataNormalizationMode: DataNormalizationMode.BackwardsPanamaCanal,
+                    dataMappingMode: DataMappingMode.OpenInterest,
+                    contractDepthOffset: 5
+                );
+                throw new Exception("Expected out of rage exception. We don't support that many back months");
+            }
+            catch (ArgumentOutOfRangeException)
+            {
+                // expected
+            }
+
+            _continuousContract = AddFuture(Futures.Indices.SP500EMini,
+                dataNormalizationMode: DataNormalizationMode.BackwardsPanamaCanal,
+                dataMappingMode: DataMappingMode.OpenInterest,
+                contractDepthOffset: 1
+            );
+        }
+
+        /// <summary>
+        /// OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
+        /// </summary>
+        /// <param name="data">Slice object keyed by symbol containing the stock data</param>
+        public override void OnData(Slice data)
+        {
+            if (data.Keys.Count != 1)
+            {
+                throw new Exception($"We are getting data for more than one symbols! {string.Join(",", data.Keys.Select(symbol => symbol))}");
+            }
+
+            foreach (var changedEvent in data.SymbolChangedEvents.Values)
+            {
+                if (changedEvent.Symbol == _continuousContract.Symbol)
+                {
+                    _mappings.Add(changedEvent);
+                    Log($"SymbolChanged event: {changedEvent}");
+
+                    var backMonthExpiration = changedEvent.Symbol.Underlying.ID.Date;
+                    var frontMonthExpiration = FuturesExpiryFunctions.FuturesExpiryFunction(_continuousContract.Symbol)(Time.AddMonths(1));
+
+                    if (backMonthExpiration <= frontMonthExpiration.Date)
+                    {
+                        throw new Exception($"Unexpected current mapped contract expiration {backMonthExpiration}" +
+                            $" @ {Time} it should be AFTER front month expiration {frontMonthExpiration}");
+                    }
+                }
+            }
+
+            if (_lastDateLog.Month != Time.Month)
+            {
+                _lastDateLog = Time;
+
+                Log($"{Time}- {Securities[_continuousContract.Symbol].GetLastData()}");
+                if (Portfolio.Invested)
+                {
+                    Liquidate();
+                }
+                else if(_continuousContract.HasData)
+                {
+                    // This works because we set this contract as tradable, even if it's a canonical security
+                    Buy(_continuousContract.Symbol, 1);
+                }
+
+                if(Time.Month == 1 && Time.Year == 2013)
+                {
+                    var response = History(new[] { _continuousContract.Symbol }, 60 * 24 * 90);
+                    if (!response.Any())
+                    {
+                        throw new Exception("Unexpected empty history response");
+                    }
+                }
+            }
+        }
+
+        public override void OnOrderEvent(OrderEvent orderEvent)
+        {
+            if (orderEvent.Status == OrderStatus.Filled)
+            {
+                Log($"{orderEvent}");
+            }
+        }
+
+        public override void OnEndOfAlgorithm()
+        {
+            var expectedMappingCounts = 2;
+            if (_mappings.Count != expectedMappingCounts)
+            {
+                throw new Exception($"Unexpected symbol changed events: {_mappings.Count}, was expecting {expectedMappingCounts}");
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public Language[] Languages { get; } = { Language.CSharp };
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Trades", "3"},
+            {"Average Win", "1.11%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "2.092%"},
+            {"Drawdown", "1.700%"},
+            {"Expectancy", "0"},
+            {"Net Profit", "1.055%"},
+            {"Sharpe Ratio", "0.682"},
+            {"Probabilistic Sharpe Ratio", "36.937%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "100%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "-0.007"},
+            {"Beta", "0.099"},
+            {"Annual Standard Deviation", "0.022"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "-2.742"},
+            {"Tracking Error", "0.076"},
+            {"Treynor Ratio", "0.149"},
+            {"Total Fees", "$5.55"},
+            {"Estimated Strategy Capacity", "$190000.00"},
+            {"Lowest Capacity Asset", "ES 1S1"},
+            {"Fitness Score", "0.01"},
+            {"Kelly Criterion Estimate", "0"},
+            {"Kelly Criterion Probability Value", "0"},
+            {"Sortino Ratio", "0.479"},
+            {"Return Over Maximum Drawdown", "1.652"},
+            {"Portfolio Turnover", "0.015"},
+            {"Total Insights Generated", "0"},
+            {"Total Insights Closed", "0"},
+            {"Total Insights Analysis Completed", "0"},
+            {"Long Insight Count", "0"},
+            {"Short Insight Count", "0"},
+            {"Long/Short Ratio", "100%"},
+            {"Estimated Monthly Alpha Value", "$0"},
+            {"Total Accumulated Estimated Alpha Value", "$0"},
+            {"Mean Population Estimated Insight Value", "$0"},
+            {"Mean Population Direction", "0%"},
+            {"Mean Population Magnitude", "0%"},
+            {"Rolling Averaged Population Direction", "0%"},
+            {"Rolling Averaged Population Magnitude", "0%"},
+            {"OrderListHash", "9f7574803b8ebfac8f912019c943d27e"}
+        };
+    }
+}

Algorithm.CSharp/ContinuousFutureRegressionAlgorithm.cs

@@ -0,0 +1,183 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Orders;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+using QuantConnect.Securities.Future;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Continuous Futures Regression algorithm. Asserting and showcasing the behavior of adding a continuous future
+    /// </summary>
+    public class ContinuousFutureRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        private List<SymbolChangedEvent> _mappings = new();
+        private Future _continuousContract;
+        private DateTime _lastDateLog;
+
+        /// <summary>
+        /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
+        /// </summary>
+        public override void Initialize()
+        {
+            SetStartDate(2013, 7, 1);
+            SetEndDate(2014, 1, 1);
+
+            _continuousContract = AddFuture(Futures.Indices.SP500EMini,
+                dataNormalizationMode: DataNormalizationMode.BackwardsRatio,
+                dataMappingMode: DataMappingMode.LastTradingDay,
+                contractDepthOffset: 0
+            );
+        }
+
+        /// <summary>
+        /// OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
+        /// </summary>
+        /// <param name="data">Slice object keyed by symbol containing the stock data</param>
+        public override void OnData(Slice data)
+        {
+            if (data.Keys.Count != 1)
+            {
+                throw new Exception($"We are getting data for more than one symbols! {string.Join(",", data.Keys.Select(symbol => symbol))}");
+            }
+
+            foreach (var changedEvent in data.SymbolChangedEvents.Values)
+            {
+                if (changedEvent.Symbol == _continuousContract.Symbol)
+                {
+                    _mappings.Add(changedEvent);
+                    Log($"SymbolChanged event: {changedEvent}");
+
+                    var currentExpiration = changedEvent.Symbol.Underlying.ID.Date;
+                    var frontMonthExpiration = FuturesExpiryFunctions.FuturesExpiryFunction(_continuousContract.Symbol)(Time.AddMonths(1));
+
+                    if (currentExpiration != frontMonthExpiration.Date)
+                    {
+                        throw new Exception($"Unexpected current mapped contract expiration {currentExpiration}" +
+                            $" @ {Time} it should be AT front month expiration {frontMonthExpiration}");
+                    }
+                }
+            }
+
+            if (_lastDateLog.Month != Time.Month)
+            {
+                _lastDateLog = Time;
+
+                Log($"{Time}- {Securities[_continuousContract.Symbol].GetLastData()}");
+                if (Portfolio.Invested)
+                {
+                    Liquidate();
+                }
+                else if(_continuousContract.HasData)
+                {
+                    // This works because we set this contract as tradable, even if it's a canonical security
+                    Buy(_continuousContract.Symbol, 1);
+                }
+
+                if(Time.Month == 1 && Time.Year == 2013)
+                {
+                    var response = History(new[] { _continuousContract.Symbol }, 60 * 24 * 90);
+                    if (!response.Any())
+                    {
+                        throw new Exception("Unexpected empty history response");
+                    }
+                }
+            }
+        }
+
+        public override void OnOrderEvent(OrderEvent orderEvent)
+        {
+            if (orderEvent.Status == OrderStatus.Filled)
+            {
+                Log($"{orderEvent}");
+            }
+        }
+
+        public override void OnEndOfAlgorithm()
+        {
+            var expectedMappingCounts = 2;
+            if (_mappings.Count != expectedMappingCounts)
+            {
+                throw new Exception($"Unexpected symbol changed events: {_mappings.Count}, was expecting {expectedMappingCounts}");
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public Language[] Languages { get; } = { Language.CSharp, Language.Python };
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Trades", "3"},
+            {"Average Win", "1.03%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "1.970%"},
+            {"Drawdown", "1.400%"},
+            {"Expectancy", "0"},
+            {"Net Profit", "0.994%"},
+            {"Sharpe Ratio", "0.7"},
+            {"Probabilistic Sharpe Ratio", "37.553%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "100%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "-0.006"},
+            {"Beta", "0.091"},
+            {"Annual Standard Deviation", "0.02"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "-2.745"},
+            {"Tracking Error", "0.076"},
+            {"Treynor Ratio", "0.153"},
+            {"Total Fees", "$5.55"},
+            {"Estimated Strategy Capacity", "$48000000.00"},
+            {"Lowest Capacity Asset", "ES 1S1"},
+            {"Fitness Score", "0.01"},
+            {"Kelly Criterion Estimate", "0"},
+            {"Kelly Criterion Probability Value", "0"},
+            {"Sortino Ratio", "0.492"},
+            {"Return Over Maximum Drawdown", "1.708"},
+            {"Portfolio Turnover", "0.016"},
+            {"Total Insights Generated", "0"},
+            {"Total Insights Closed", "0"},
+            {"Total Insights Analysis Completed", "0"},
+            {"Long Insight Count", "0"},
+            {"Short Insight Count", "0"},
+            {"Long/Short Ratio", "100%"},
+            {"Estimated Monthly Alpha Value", "$0"},
+            {"Total Accumulated Estimated Alpha Value", "$0"},
+            {"Mean Population Estimated Insight Value", "$0"},
+            {"Mean Population Direction", "0%"},
+            {"Mean Population Magnitude", "0%"},
+            {"Rolling Averaged Population Direction", "0%"},
+            {"Rolling Averaged Population Magnitude", "0%"},
+            {"OrderListHash", "fb3bb82d84fc6c390a40f36d0d1faf59"}
+        };
+    }
+}

Algorithm.CSharp/DelistedFutureLiquidateRegressionAlgorithm.cs

@@ -97,7 +97,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Average Win", "1.63%"},
             {"Average Loss", "0%"},
             {"Compounding Annual Return", "7.292%"},
-            {"Drawdown", "1.300%"},
+            {"Drawdown", "1.500%"},
             {"Expectancy", "0"},
             {"Net Profit", "1.634%"},
             {"Sharpe Ratio", "2.351"},

Algorithm.CSharp/RawDataRegressionAlgorithm.cs

@@ -32,7 +32,7 @@ namespace QuantConnect.Algorithm.CSharp
     public class RawDataRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
         private const string Ticker = "GOOGL";
-        private FactorFile _factorFile;
+        private CorporateFactorProvider _factorFile;
         private readonly IEnumerator<decimal> _expectedRawPrices = new List<decimal> { 1157.93m, 1158.72m,
             1131.97m, 1114.28m, 1120.15m, 1114.51m, 1134.89m, 567.55m, 571.50m, 545.25m, 540.63m }.GetEnumerator();
         private Symbol _googl;
@@ -56,7 +56,7 @@ namespace QuantConnect.Algorithm.CSharp
             mapFileProvider.Initialize(dataProvider);
             var factorFileProvider = new LocalDiskFactorFileProvider();
             factorFileProvider.Initialize(mapFileProvider, dataProvider);
-            _factorFile = factorFileProvider.Get(_googl);
+            _factorFile = factorFileProvider.Get(_googl) as CorporateFactorProvider;
 
             // Prime our expected values
             _expectedRawPrices.MoveNext();
@@ -81,7 +81,7 @@ namespace QuantConnect.Algorithm.CSharp
                 if (_expectedRawPrices.Current != googlData.Close)
                 {
                     // Our values don't match lets try and give a reason why
-                    var dayFactor = _factorFile.GetPriceScaleFactor(googlData.Time);
+                    var dayFactor = _factorFile.GetPriceScaleFactor(googlData.Time, DataNormalizationMode.Adjusted);
                     var probableRawPrice = googlData.Close / dayFactor; // Undo adjustment
 
                     if (_expectedRawPrices.Current == probableRawPrice)

Algorithm.Python/ContinuousFutureRegressionAlgorithm.py

@@ -0,0 +1,71 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+### <summary>
+### Continuous Futures Regression algorithm. Asserting and showcasing the behavior of adding a continuous future
+### </summary>
+class ContinuousFutureRegressionAlgorithm(QCAlgorithm):
+    '''Basic template algorithm simply initializes the date range and cash'''
+
+    def Initialize(self):
+        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
+
+        self.SetStartDate(2013, 7, 1)
+        self.SetEndDate(2014, 1, 1)
+
+        self._mappings = []
+        self._lastDateLog = -1
+        self._continuousContract = self.AddFuture(Futures.Indices.SP500EMini,
+                                                  dataNormalizationMode = DataNormalizationMode.BackwardsRatio,
+                                                  dataMappingMode = DataMappingMode.LastTradingDay,
+                                                  contractDepthOffset= 0)
+
+    def OnData(self, data):
+        '''OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
+
+        Arguments:
+            data: Slice object keyed by symbol containing the stock data
+        '''
+        if len(data.Keys) != 1:
+            raise ValueError(f"We are getting data for more than one symbols! {','.join(data.Keys)}")
+
+        for changedEvent in data.SymbolChangedEvents.Values:
+            if changedEvent.Symbol == self._continuousContract.Symbol:
+                self._mappings.append(changedEvent)
+                self.Log(f"SymbolChanged event: {changedEvent}")
+
+        if self._lastDateLog != self.Time.month:
+            self._lastDateLog = self.Time.month
+
+            self.Log(f"{self.Time}- {self._continuousContract.GetLastData()}")
+            if self.Portfolio.Invested:
+                self.Liquidate()
+            elif self._continuousContract.HasData:
+                # This works because we set this contract as tradable, even if it's a canonical security
+                self.Buy(self._continuousContract.Symbol, 1)
+
+            if self.Time.month == 1 and self.Time.year == 2013:
+                response = self.History( [ self._continuousContract.Symbol ], 60 * 24 * 90)
+                if response.empty:
+                    raise ValueError("Unexpected empty history response")
+
+    def OnOrderEvent(self, orderEvent):
+        if orderEvent.Status == OrderStatus.Filled:
+            self.Debug("Purchased Stock: {0}".format(orderEvent.Symbol))
+
+    def OnEndOfAlgorithm(self):
+        expectedMappingCounts = 2
+        if len(self._mappings) != expectedMappingCounts:
+            raise ValueError(f"Unexpected symbol changed events: {self._mappings.count()}, was expecting {expectedMappingCounts}")

Algorithm/QCAlgorithm.History.cs

@@ -475,7 +475,7 @@ namespace QuantConnect.Algorithm
         /// <returns>Securities historical data</returns>
         public IEnumerable<BaseData> GetLastKnownPrices(Symbol symbol)
         {
-            if (symbol.IsCanonical() || HistoryProvider == null)
+            if (!HistoryRequestValid(symbol) || HistoryProvider == null)
             {
                 return Enumerable.Empty<BaseData>();
             }
@@ -568,7 +568,7 @@ namespace QuantConnect.Algorithm
         {
             var sentMessage = false;
             // filter out any universe securities that may have made it this far
-            var filteredRequests = requests.Where(hr => !UniverseManager.ContainsKey(hr.Symbol)).ToList();
+            var filteredRequests = requests.Where(hr => HistoryRequestValid(hr.Symbol)).ToList();
             for (var i = 0; i < filteredRequests.Count; i++)
             {
                 var request  = filteredRequests[i];
@@ -604,7 +604,7 @@ namespace QuantConnect.Algorithm
         /// </summary>
         private IEnumerable<HistoryRequest> CreateDateRangeHistoryRequests(IEnumerable<Symbol> symbols, DateTime startAlgoTz, DateTime endAlgoTz, Resolution? resolution = null, bool? fillForward = null, bool? extendedMarket = null)
         {
-            return symbols.Where(x => !x.IsCanonical()).SelectMany(x =>
+            return symbols.Where(HistoryRequestValid).SelectMany(x =>
             {
                 var requests = new List<HistoryRequest>();
 
@@ -629,7 +629,7 @@ namespace QuantConnect.Algorithm
         /// </summary>
         private IEnumerable<HistoryRequest> CreateBarCountHistoryRequests(IEnumerable<Symbol> symbols, int periods, Resolution? resolution = null)
         {
-            return symbols.Where(x => !x.IsCanonical()).SelectMany(x =>
+            return symbols.Where(HistoryRequestValid).SelectMany(x =>
             {
                 var res = GetResolution(x, resolution);
                 var exchange = GetExchangeHours(x);
@@ -778,5 +778,14 @@ namespace QuantConnect.Algorithm
                 return resolution ?? UniverseSettings.Resolution;
             }
         }
+
+        /// <summary>
+        /// Validate a symbol for a history request.
+        /// Universe and canonical symbols are only valid for future security types
+        /// </summary>
+        private bool HistoryRequestValid(Symbol symbol)
+        {
+            return symbol.SecurityType == SecurityType.Future || !UniverseManager.ContainsKey(symbol) && !symbol.IsCanonical();
+        }
     }
 }

Algorithm/QCAlgorithm.cs

@@ -45,6 +45,7 @@ using QuantConnect.Algorithm.Framework.Portfolio;
 using QuantConnect.Algorithm.Framework.Risk;
 using QuantConnect.Algorithm.Framework.Selection;
 using QuantConnect.Algorithm.Selection;
+using QuantConnect.Data.Market;
 using QuantConnect.Storage;
 using Index = QuantConnect.Securities.Index.Index;
 
@@ -1403,9 +1404,12 @@ namespace QuantConnect.Algorithm
         /// <param name="resolution">Resolution of the Data Required</param>
         /// <param name="fillDataForward">When no data available on a tradebar, return the last data that was generated</param>
         /// <param name="extendedMarketHours">Show the after market data as well</param>
-        public Security AddSecurity(SecurityType securityType, string ticker, Resolution? resolution = null, bool fillDataForward = true, bool extendedMarketHours = false)
+        /// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
+        /// <param name="dataNormalizationMode">The price scaling mode to use for the security</param>
+        public Security AddSecurity(SecurityType securityType, string ticker, Resolution? resolution = null, bool fillDataForward = true, bool extendedMarketHours = false,
+            DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null)
         {
-            return AddSecurity(securityType, ticker, resolution, fillDataForward, Security.NullLeverage, extendedMarketHours);
+            return AddSecurity(securityType, ticker, resolution, fillDataForward, Security.NullLeverage, extendedMarketHours, dataMappingMode, dataNormalizationMode);
         }
 
         /// <summary>
@@ -1417,10 +1421,13 @@ namespace QuantConnect.Algorithm
         /// <param name="fillDataForward">When no data available on a tradebar, return the last data that was generated</param>
         /// <param name="leverage">Custom leverage per security</param>
         /// <param name="extendedMarketHours">Extended market hours</param>
+        /// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
+        /// <param name="dataNormalizationMode">The price scaling mode to use for the security</param>
         /// <remarks> AddSecurity(SecurityType securityType, Symbol symbol, Resolution resolution, bool fillDataForward, decimal leverage, bool extendedMarketHours)</remarks>
-        public Security AddSecurity(SecurityType securityType, string ticker, Resolution? resolution, bool fillDataForward, decimal leverage, bool extendedMarketHours)
+        public Security AddSecurity(SecurityType securityType, string ticker, Resolution? resolution, bool fillDataForward, decimal leverage, bool extendedMarketHours,
+            DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null)
         {
-            return AddSecurity(securityType, ticker, resolution, null, fillDataForward, leverage, extendedMarketHours);
+            return AddSecurity(securityType, ticker, resolution, null, fillDataForward, leverage, extendedMarketHours, dataMappingMode, dataNormalizationMode);
         }
 
         /// <summary>
@@ -1433,7 +1440,10 @@ namespace QuantConnect.Algorithm
         /// <param name="fillDataForward">If true, returns the last available data even if none in that timeslice.</param>
         /// <param name="leverage">leverage for this security</param>
         /// <param name="extendedMarketHours">ExtendedMarketHours send in data from 4am - 8pm, not used for FOREX</param>
-        public Security AddSecurity(SecurityType securityType, string ticker, Resolution? resolution, string market, bool fillDataForward, decimal leverage, bool extendedMarketHours)
+        /// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
+        /// <param name="dataNormalizationMode">The price scaling mode to use for the security</param>
+        public Security AddSecurity(SecurityType securityType, string ticker, Resolution? resolution, string market, bool fillDataForward, decimal leverage, bool extendedMarketHours,
+            DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null)
         {
             // if AddSecurity method is called to add an option or a future, we delegate a call to respective methods
             if (securityType == SecurityType.Option)
@@ -1464,7 +1474,7 @@ namespace QuantConnect.Algorithm
                     symbol = QuantConnect.Symbol.Create(ticker, securityType, market);
                 }
 
-                return AddSecurity(symbol, resolution, fillDataForward, leverage, extendedMarketHours);
+                return AddSecurity(symbol, resolution, fillDataForward, leverage, extendedMarketHours, dataMappingMode, dataNormalizationMode);
             }
             catch (Exception err)
             {
@@ -1481,9 +1491,22 @@ namespace QuantConnect.Algorithm
         /// <param name="fillDataForward">If true, returns the last available data even if none in that timeslice.</param>
         /// <param name="leverage">leverage for this security</param>
         /// <param name="extendedMarketHours">ExtendedMarketHours send in data from 4am - 8pm, not used for FOREX</param>
+        /// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
+        /// <param name="dataNormalizationMode">The price scaling mode to use for the security</param>
+        /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
+        /// For example, 0 (default) will use the front month, 1 will use the back month contract</param>
         /// <returns>The new Security that was added to the algorithm</returns>
-        public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool fillDataForward = true, decimal leverage = Security.NullLeverage, bool extendedMarketHours = false)
+        public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool fillDataForward = true, decimal leverage = Security.NullLeverage, bool extendedMarketHours = false,
+            DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null, int contractDepthOffset = 0)
         {
+            // allow users to specify negative numbers, we get the abs of it
+            var contractOffset = (uint)Math.Abs(contractDepthOffset);
+            if (contractOffset > 2)
+            {
+                throw new ArgumentOutOfRangeException(nameof(contractDepthOffset), "'contractDepthOffset' current maximum value is 2." +
+                    " Front month (0) and only 2 back month contracts are currently supported.");
+            }
+
             var isCanonical = symbol.IsCanonical();
 
             // Short-circuit to AddOptionContract because it will add the underlying if required
@@ -1516,6 +1539,21 @@ namespace QuantConnect.Algorithm
                     }
                     else
                     {
+                        security.IsTradable = true;
+
+                        var continuousConfigs = SubscriptionManager.SubscriptionDataConfigService.Add(symbol,
+                            resolution,
+                            fillDataForward,
+                            extendedMarketHours,
+                            isFilteredSubscription: false,
+                            subscriptionDataTypes: SubscriptionManager.LookupSubscriptionConfigDataTypes(SecurityType.Future,
+                                GetResolution(symbol, resolution), isCanonical:false),
+                            dataNormalizationMode: dataNormalizationMode ?? UniverseSettings.GetDefaultNormalizationMode(symbol.SecurityType),
+                            dataMappingMode: dataMappingMode ?? UniverseSettings.DataMappingMode,
+                            contractDepthOffset: contractOffset
+                            );
+                        AddToUserDefinedUniverse(security, continuousConfigs);
+
                         universe = new FuturesChainUniverse((Future)security, settings);
                     }
 
@@ -1616,8 +1654,14 @@ namespace QuantConnect.Algorithm
         /// <param name="market">The futures market, <seealso cref="Market"/>. Default is value null and looked up using BrokerageModel.DefaultMarkets in <see cref="AddSecurity{T}"/></param>
         /// <param name="fillDataForward">If true, returns the last available data even if none in that timeslice. Default is <value>true</value></param>
         /// <param name="leverage">The requested leverage for this equity. Default is set by <see cref="SecurityInitializer"/></param>
+        /// <param name="dataMappingMode">The contract mapping mode to use for the continuous future contract</param>
+        /// <param name="dataNormalizationMode">The price scaling mode to use for the continuous future contract</param>
+        /// <param name="contractDepthOffset">The continuous future contract desired offset from the current front month.
+        /// For example, 0 (default) will use the front month, 1 will use the back month contract</param>
         /// <returns>The new <see cref="Future"/> security</returns>
-        public Future AddFuture(string ticker, Resolution? resolution = null, string market = null, bool fillDataForward = true, decimal leverage = Security.NullLeverage)
+        public Future AddFuture(string ticker, Resolution? resolution = null, string market = null,
+            bool fillDataForward = true, decimal leverage = Security.NullLeverage,
+            DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null, int contractDepthOffset = 0)
         {
             if (market == null)
             {
@@ -1637,7 +1681,8 @@ namespace QuantConnect.Algorithm
                 canonicalSymbol = QuantConnect.Symbol.Create(ticker, SecurityType.Future, market, alias);
             }
 
-            return (Future)AddSecurity(canonicalSymbol, resolution, fillDataForward, leverage);
+            return (Future)AddSecurity(canonicalSymbol, resolution, fillDataForward, leverage, dataMappingMode: dataMappingMode,
+                dataNormalizationMode: dataNormalizationMode, contractDepthOffset: contractDepthOffset);
         }
 
         /// <summary>
@@ -2261,7 +2306,8 @@ namespace QuantConnect.Algorithm
         /// <summary>
         /// Creates and adds a new <see cref="Security"/> to the algorithm
         /// </summary>
-        private T AddSecurity<T>(SecurityType securityType, string ticker, Resolution? resolution, string market, bool fillDataForward, decimal leverage, bool extendedMarketHours)
+        private T AddSecurity<T>(SecurityType securityType, string ticker, Resolution? resolution, string market, bool fillDataForward, decimal leverage, bool extendedMarketHours,
+            DataMappingMode? mappingMode = null, DataNormalizationMode? normalizationMode = null)
             where T : Security
         {
             if (market == null)
@@ -2280,7 +2326,9 @@ namespace QuantConnect.Algorithm
                 symbol = QuantConnect.Symbol.Create(ticker, securityType, market);
             }
 
-            var configs = SubscriptionManager.SubscriptionDataConfigService.Add(symbol, resolution, fillDataForward, extendedMarketHours, dataNormalizationMode: UniverseSettings.DataNormalizationMode);
+            var configs = SubscriptionManager.SubscriptionDataConfigService.Add(symbol, resolution, fillDataForward, extendedMarketHours,
+                dataNormalizationMode: normalizationMode ?? UniverseSettings.DataNormalizationMode,
+                dataMappingMode: mappingMode ?? UniverseSettings.DataMappingMode);
             var security = Securities.CreateSecurity(symbol, configs, leverage);
 
             return (T) AddToUserDefinedUniverse(security, configs);

AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs

@@ -440,8 +440,11 @@ namespace QuantConnect.AlgorithmFactory.Python.Wrappers
         /// <param name="fillDataForward">If true, returns the last available data even if none in that timeslice.</param>
         /// <param name="leverage">leverage for this security</param>
         /// <param name="extendedMarketHours">ExtendedMarketHours send in data from 4am - 8pm, not used for FOREX</param>
-        public Security AddSecurity(SecurityType securityType, string symbol, Resolution? resolution, string market, bool fillDataForward, decimal leverage, bool extendedMarketHours)
-            => _baseAlgorithm.AddSecurity(securityType, symbol, resolution, market, fillDataForward, leverage, extendedMarketHours);
+        /// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
+        /// <param name="dataNormalizationMode">The price scaling mode to use for the security</param>
+        public Security AddSecurity(SecurityType securityType, string symbol, Resolution? resolution, string market, bool fillDataForward, decimal leverage, bool extendedMarketHours,
+            DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null)
+            => _baseAlgorithm.AddSecurity(securityType, symbol, resolution, market, fillDataForward, leverage, extendedMarketHours, dataMappingMode, dataNormalizationMode);
 
         /// <summary>
         /// Creates and adds a new single <see cref="Future"/> contract to the algorithm

Brokerages/InteractiveBrokers/InteractiveBrokersSymbolMapper.cs

@@ -25,6 +25,7 @@ using System.IO;
 using System.Linq;
 using System.Text.RegularExpressions;
 using IBApi;
+using QuantConnect.Data.Auxiliary;
 using QuantConnect.Logging;
 using QuantConnect.Securities;
 
@@ -283,7 +284,7 @@ namespace QuantConnect.Brokerages.InteractiveBrokers
             var ticker = symbol.ID.Symbol;
             if (symbol.ID.SecurityType == SecurityType.Equity)
             {
-                var mapFile = _mapFileProvider.Get(symbol.ID.Market).ResolveMapFile(symbol.ID.Symbol, symbol.ID.Date);
+                var mapFile = _mapFileProvider.Get(CorporateActionsKey.Create(symbol)).ResolveMapFile(symbol);
                 ticker = mapFile.GetMappedSymbol(DateTime.UtcNow, symbol.Value);
             }
 

Common/Data/Auxiliary/CorporateActionsKey.cs

@@ -0,0 +1,97 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+namespace QuantConnect.Data.Auxiliary
+{
+    /// <summary>
+    /// Unique definition key for a collection of corporate actions for a Market and SecurityType
+    /// </summary>
+    public class CorporateActionsKey
+    {
+        /// <summary>
+        /// USA equities market corporate actions key definition
+        /// </summary>
+        public static CorporateActionsKey EquityUsa { get; } = new (QuantConnect.Market.USA, SecurityType.Equity);
+
+        /// <summary>
+        /// The market associated with these corporate actions
+        /// </summary>
+        public string Market { get; }
+
+        /// <summary>
+        /// The associated security type
+        /// </summary>
+        public SecurityType SecurityType { get; }
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        public CorporateActionsKey(string market, SecurityType securityType)
+        {
+            Market = market;
+            SecurityType = securityType;
+        }
+
+        /// <summary>
+        /// Serves as a hash function for a particular type.
+        /// </summary>
+        public override int GetHashCode()
+        {
+            unchecked
+            {
+                var hashCode = Market.GetHashCode();
+                return (hashCode*397) ^ SecurityType.GetHashCode();
+            }
+        }
+
+        /// <summary>
+        /// Determines whether the specified <see cref="T:System.Object"/> is equal to the current <see cref="T:System.Object"/>.
+        /// </summary>
+        /// <returns>
+        /// true if the specified object  is equal to the current object; otherwise, false.
+        /// </returns>
+        /// <param name="obj">The object to compare with the current object. </param><filterpriority>2</filterpriority>
+        public override bool Equals(object? obj)
+        {
+            if (ReferenceEquals(null, obj)) return false;
+            if (obj.GetType() != GetType()) return false;
+
+            var other = (CorporateActionsKey)obj;
+
+            return other.Market == Market
+                && other.SecurityType == SecurityType;
+        }
+
+        public override string ToString()
+        {
+            return $"{Market}:{SecurityType}";
+        }
+
+        /// <summary>
+        /// Helper method to create a new instance from a Symbol
+        /// </summary>
+        public static CorporateActionsKey Create(Symbol symbol) => Create(symbol.HasUnderlying ? symbol.Underlying.ID : symbol.ID);
+
+        /// <summary>
+        /// Helper method to create a new instance from a SecurityIdentifier
+        /// </summary>
+        public static CorporateActionsKey Create(SecurityIdentifier securityIdentifier)
+        {
+            securityIdentifier = securityIdentifier.HasUnderlying ? securityIdentifier.Underlying : securityIdentifier;
+            return new CorporateActionsKey(securityIdentifier.Market, securityIdentifier.SecurityType);
+        }
+    }
+}

Common/Data/Auxiliary/CorporateFactorProvider.cs

@@ -0,0 +1,273 @@
+/*
+* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+*/
+
+using System;
+using System.Linq;
+using QuantConnect.Util;
+using QuantConnect.Logging;
+using QuantConnect.Securities;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+
+namespace QuantConnect.Data.Auxiliary
+{
+    /// <summary>
+    /// Corporate related factor provider. Factors based on splits and dividends
+    /// </summary>
+    public class CorporateFactorProvider : FactorFile<FactorFileRow>
+    {
+        /// <summary>
+        ///Creates a new instance
+        /// </summary>
+        public CorporateFactorProvider(string permtick, IEnumerable<FactorFileRow> data, DateTime? factorFileMinimumDate = null) : base(permtick, data, factorFileMinimumDate)
+        {
+        }
+
+        /// <summary>
+        /// Gets the price scale factor that includes dividend and split adjustments for the specified search date
+        /// </summary>
+        public override decimal GetPriceScaleFactor(DateTime searchDate, DataNormalizationMode dataNormalizationMode, DataMappingMode? dataMappingMode = null, uint contractOffset = 0)
+        {
+            if (dataNormalizationMode == DataNormalizationMode.Raw)
+            {
+                return 0;
+            }
+
+            var factor = 1m;
+
+            for (var i = 0; i < _reversedFactorFileDates.Count; i++)
+            {
+                var factorDate = _reversedFactorFileDates[i];
+                if (factorDate.Date < searchDate.Date)
+                {
+                    break;
+                }
+
+                var factorFileRow = SortedFactorFileData[factorDate];
+                switch (dataNormalizationMode)
+                {
+                    case DataNormalizationMode.TotalReturn:
+                    case DataNormalizationMode.SplitAdjusted:
+                        factor = factorFileRow.First().SplitFactor;
+                        break;
+                    case DataNormalizationMode.Adjusted:
+                        factor = factorFileRow.First().PriceScaleFactor;
+                        break;
+                    default:
+                        throw new ArgumentOutOfRangeException();
+                }
+            }
+
+            return factor;
+        }
+
+        /// <summary>
+        /// Gets price and split factors to be applied at the specified date
+        /// </summary>
+        public FactorFileRow GetScalingFactors(DateTime searchDate)
+        {
+            var factors = new FactorFileRow(searchDate, 1m, 1m, 0m);
+
+            // Iterate backwards to find the most recent factors
+            foreach (var splitDate in _reversedFactorFileDates)
+            {
+                if (splitDate.Date < searchDate.Date) break;
+                factors = SortedFactorFileData[splitDate][0];
+            }
+
+            return factors;
+        }
+
+        /// <summary>
+        /// Returns true if the specified date is the last trading day before a dividend event
+        /// is to be fired
+        /// </summary>
+        /// <remarks>
+        /// NOTE: The dividend event in the algorithm should be fired at the end or AFTER
+        /// this date. This is the date in the file that a factor is applied, so for example,
+        /// MSFT has a 31 cent dividend on 2015.02.17, but in the factor file the factor is applied
+        /// to 2015.02.13, which is the first trading day BEFORE the actual effective date.
+        /// </remarks>
+        /// <param name="date">The date to check the factor file for a dividend event</param>
+        /// <param name="priceFactorRatio">When this function returns true, this value will be populated
+        /// with the price factor ratio required to scale the closing value (pf_i/pf_i+1)</param>
+        /// <param name="referencePrice">When this function returns true, this value will be populated
+        /// with the reference raw price, which is the close of the provided date</param>
+        public bool HasDividendEventOnNextTradingDay(DateTime date, out decimal priceFactorRatio, out decimal referencePrice)
+        {
+            priceFactorRatio = 0;
+            referencePrice = 0;
+            var index = SortedFactorFileData.IndexOfKey(date);
+            if (index > -1 && index < SortedFactorFileData.Count - 1)
+            {
+                // grab the next key to ensure it's a dividend event
+                var thisRow = SortedFactorFileData.Values[index].First();
+                var nextRow = SortedFactorFileData.Values[index + 1].First();
+
+                // if the price factors have changed then it's a dividend event
+                if (thisRow.PriceFactor != nextRow.PriceFactor)
+                {
+                    priceFactorRatio = thisRow.PriceFactor / nextRow.PriceFactor;
+                    referencePrice = thisRow.ReferencePrice;
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        /// <summary>
+        /// Returns true if the specified date is the last trading day before a split event
+        /// is to be fired
+        /// </summary>
+        /// <remarks>
+        /// NOTE: The split event in the algorithm should be fired at the end or AFTER this
+        /// date. This is the date in the file that a factor is applied, so for example MSFT
+        /// has a split on 1999.03.29, but in the factor file the split factor is applied on
+        /// 1999.03.26, which is the first trading day BEFORE the actual split date.
+        /// </remarks>
+        /// <param name="date">The date to check the factor file for a split event</param>
+        /// <param name="splitFactor">When this function returns true, this value will be populated
+        /// with the split factor ratio required to scale the closing value</param>
+        /// <param name="referencePrice">When this function returns true, this value will be populated
+        /// with the reference raw price, which is the close of the provided date</param>
+        public bool HasSplitEventOnNextTradingDay(DateTime date, out decimal splitFactor, out decimal referencePrice)
+        {
+            splitFactor = 1;
+            referencePrice = 0;
+            var index = SortedFactorFileData.IndexOfKey(date);
+            if (index > -1 && index < SortedFactorFileData.Count - 1)
+            {
+                // grab the next key to ensure it's a split event
+                var thisRow = SortedFactorFileData.Values[index].First();
+                var nextRow = SortedFactorFileData.Values[index + 1].First();
+
+                // if the split factors have changed then it's a split event
+                if (thisRow.SplitFactor != nextRow.SplitFactor)
+                {
+                    splitFactor = thisRow.SplitFactor / nextRow.SplitFactor;
+                    referencePrice = thisRow.ReferencePrice;
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        /// <summary>
+        /// Gets all of the splits and dividends represented by this factor file
+        /// </summary>
+        /// <param name="symbol">The symbol to ues for the dividend and split objects</param>
+        /// <param name="exchangeHours">Exchange hours used for resolving the previous trading day</param>
+        /// <param name="decimalPlaces">The number of decimal places to round the dividend's distribution to, defaulting to 2</param>
+        /// <returns>All splits and dividends represented by this factor file in chronological order</returns>
+        public List<BaseData> GetSplitsAndDividends(Symbol symbol, SecurityExchangeHours exchangeHours, int decimalPlaces = 2)
+        {
+            var dividendsAndSplits = new List<BaseData>();
+            if (SortedFactorFileData.Count == 0)
+            {
+                Log.Trace($"{symbol} has no factors!");
+                return dividendsAndSplits;
+            }
+
+            var futureFactorFileRow = SortedFactorFileData.Last().Value.First();
+            for (var i = SortedFactorFileData.Count - 2; i >= 0; i--)
+            {
+                var row = SortedFactorFileData.Values[i].First();
+                var dividend = row.GetDividend(futureFactorFileRow, symbol, exchangeHours, decimalPlaces);
+                if (dividend.Distribution != 0m)
+                {
+                    dividendsAndSplits.Add(dividend);
+                }
+
+                var split = row.GetSplit(futureFactorFileRow, symbol, exchangeHours);
+                if (split.SplitFactor != 1m)
+                {
+                    dividendsAndSplits.Add(split);
+                }
+
+                futureFactorFileRow = row;
+            }
+
+            return dividendsAndSplits.OrderBy(d => d.Time.Date).ToList();
+        }
+
+        /// <summary>
+        /// Creates a new factor file with the specified data applied.
+        /// Only <see cref="Dividend"/> and <see cref="Split"/> data types
+        /// will be used.
+        /// </summary>
+        /// <param name="data">The data to apply</param>
+        /// <param name="exchangeHours">Exchange hours used for resolving the previous trading day</param>
+        /// <returns>A new factor file that incorporates the specified dividend</returns>
+        public CorporateFactorProvider Apply(List<BaseData> data, SecurityExchangeHours exchangeHours)
+        {
+            if (data.Count == 0)
+            {
+                return this;
+            }
+
+            var factorFileRows = new List<FactorFileRow>();
+            var firstEntry = SortedFactorFileData.First().Value.First();
+            var lastEntry = SortedFactorFileData.Last().Value.First();
+            factorFileRows.Add(lastEntry);
+
+            var splitsAndDividends = GetSplitsAndDividends(data[0].Symbol, exchangeHours);
+
+            var combinedData = splitsAndDividends.Concat(data)
+                .DistinctBy(e => $"{e.GetType().Name}{e.Time.ToStringInvariant(DateFormat.EightCharacter)}")
+                .OrderByDescending(d => d.Time.Date);
+
+            foreach (var datum in combinedData)
+            {
+                FactorFileRow nextEntry = null;
+                var split = datum as Split;
+                var dividend = datum as Dividend;
+                if (dividend != null)
+                {
+                    nextEntry = lastEntry.Apply(dividend, exchangeHours);
+                    lastEntry = nextEntry;
+                }
+                else if (split != null)
+                {
+                    nextEntry = lastEntry.Apply(split, exchangeHours);
+                    lastEntry = nextEntry;
+                }
+
+                if (nextEntry != null)
+                {
+                    // overwrite the latest entry -- this handles splits/dividends on the same date
+                    if (nextEntry.Date == factorFileRows.Last().Date)
+                    {
+                        factorFileRows[factorFileRows.Count - 1] = nextEntry;
+                    }
+                    else
+                    {
+                        factorFileRows.Add(nextEntry);
+                    }
+                }
+            }
+
+            var firstFactorFileRow = new FactorFileRow(firstEntry.Date, factorFileRows.Last().PriceFactor, factorFileRows.Last().SplitFactor, firstEntry.ReferencePrice == 0 ? 0 : firstEntry.ReferencePrice);
+            var existing = factorFileRows.FindIndex(row => row.Date == firstFactorFileRow.Date);
+            if (existing == -1)
+            {
+                // only add it if not present
+                factorFileRows.Add(firstFactorFileRow);
+            }
+
+            return new CorporateFactorProvider(Permtick, factorFileRows, FactorFileMinimumDate);
+        }
+    }
+}

Common/Data/Auxiliary/FactorFile.cs

@@ -15,32 +15,29 @@
 */
 
 using System;
-using System.Collections;
-using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using QuantConnect.Data.Market;
-using QuantConnect.Logging;
-using QuantConnect.Securities;
 using QuantConnect.Util;
-using static QuantConnect.StringExtensions;
+using System.Collections;
+using System.Collections.Generic;
 
 namespace QuantConnect.Data.Auxiliary
 {
     /// <summary>
     /// Represents an entire factor file for a specified symbol
     /// </summary>
-    public class FactorFile : IEnumerable<FactorFileRow>
+    public abstract class FactorFile<T> : IFactorProvider
+        where T : IFactorRow
     {
         /// <summary>
         /// Keeping a reversed version is more performant that reversing it each time we need it
         /// </summary>
-        private readonly List<DateTime> _reversedFactorFileDates;
+        protected readonly List<DateTime> _reversedFactorFileDates;
 
         /// <summary>
         /// The factor file data rows sorted by date
         /// </summary>
-        public SortedList<DateTime, FactorFileRow> SortedFactorFileData { get; set; }
+        public SortedList<DateTime, List<T>> SortedFactorFileData { get; set; }
 
         /// <summary>
         /// The minimum tradeable date for the symbol
@@ -68,24 +65,22 @@ namespace QuantConnect.Data.Auxiliary
         /// <summary>
         /// Initializes a new instance of the <see cref="FactorFile"/> class.
         /// </summary>
-        public FactorFile(string permtick, IEnumerable<FactorFileRow> data, DateTime? factorFileMinimumDate = null)
+        protected FactorFile(string permtick, IEnumerable<T> data, DateTime? factorFileMinimumDate = null)
         {
             Permtick = permtick.LazyToUpper();
 
-            var dictionary = new Dictionary<DateTime, FactorFileRow>();
+            SortedFactorFileData = new SortedList<DateTime, List<T>>();
             foreach (var row in data)
             {
-                if (dictionary.ContainsKey(row.Date))
+                if (!SortedFactorFileData.TryGetValue(row.Date, out var factorFileRows))
                 {
-                    Log.Trace(Invariant($"Skipping duplicate factor file row for symbol: {permtick}, date: {row.Date:yyyyMMdd}"));
-                    continue;
+                    SortedFactorFileData[row.Date] = factorFileRows = new List<T>();
                 }
 
-                dictionary.Add(row.Date, row);
+                factorFileRows.Add(row);
             }
-            SortedFactorFileData = new SortedList<DateTime, FactorFileRow>(dictionary);
 
-            _reversedFactorFileDates = new List<DateTime>();
+            _reversedFactorFileDates = new List<DateTime>(SortedFactorFileData.Count);
             foreach (var time in SortedFactorFileData.Keys.Reverse())
             {
                 _reversedFactorFileDates.Add(time);
@@ -95,288 +90,45 @@ namespace QuantConnect.Data.Auxiliary
         }
 
         /// <summary>
-        /// Reads a FactorFile in from the <see cref="Globals.DataFolder"/>.
+        /// Gets the price scale factor for the specified search date
         /// </summary>
-        public static FactorFile Read(string permtick, Stream file)
-        {
-            return new FactorFile(permtick, FactorFileRow.Read(file, out DateTime? factorFileMinimumDate), factorFileMinimumDate);
-        }
-
-        /// <summary>
-        /// Parses the specified lines as a factor file
-        /// </summary>
-        public static FactorFile Parse(string permtick, IEnumerable<string> lines)
-        {
-            DateTime? factorFileMinimumDate;
-            return new FactorFile(permtick, FactorFileRow.Parse(lines, out factorFileMinimumDate), factorFileMinimumDate);
-        }
-
-        /// <summary>
-        /// Gets the price scale factor that includes dividend and split adjustments for the specified search date
-        /// </summary>
-        public decimal GetPriceScaleFactor(DateTime searchDate)
-        {
-            decimal factor = 1;
-            //Iterate backwards to find the most recent factor:
-            foreach (var splitDate in _reversedFactorFileDates)
-            {
-                if (splitDate.Date < searchDate.Date) break;
-                factor = SortedFactorFileData[splitDate].PriceScaleFactor;
-            }
-            return factor;
-        }
-
-        /// <summary>
-        /// Gets the split factor to be applied at the specified date
-        /// </summary>
-        public decimal GetSplitFactor(DateTime searchDate)
-        {
-            decimal factor = 1;
-            //Iterate backwards to find the most recent factor:
-            foreach (var splitDate in _reversedFactorFileDates)
-            {
-                if (splitDate.Date < searchDate.Date) break;
-                factor = SortedFactorFileData[splitDate].SplitFactor;
-            }
-            return factor;
-        }
-
-        /// <summary>
-        /// Gets price and split factors to be applied at the specified date
-        /// </summary>
-        public FactorFileRow GetScalingFactors(DateTime searchDate)
-        {
-            var factors = new FactorFileRow(searchDate, 1m, 1m, 0m);
-
-            // Iterate backwards to find the most recent factors
-            foreach (var splitDate in _reversedFactorFileDates)
-            {
-                if (splitDate.Date < searchDate.Date) break;
-                factors = SortedFactorFileData[splitDate];
-            }
-
-            return factors;
-        }
-
-        /// <summary>
-        /// Checks whether or not a symbol has scaling factors
-        /// </summary>
-        public static bool HasScalingFactors(string permtick, string market)
-        {
-            // check for factor files
-            var path = Path.Combine(Globals.CacheDataFolder, "equity", market, "factor_files", permtick.ToLowerInvariant() + ".csv");
-            if (File.Exists(path))
-            {
-                return true;
-            }
-            Log.Trace($"FactorFile.HasScalingFactors(): Factor file not found: {permtick}");
-            return false;
-        }
-
-        /// <summary>
-        /// Returns true if the specified date is the last trading day before a dividend event
-        /// is to be fired
-        /// </summary>
-        /// <remarks>
-        /// NOTE: The dividend event in the algorithm should be fired at the end or AFTER
-        /// this date. This is the date in the file that a factor is applied, so for example,
-        /// MSFT has a 31 cent dividend on 2015.02.17, but in the factor file the factor is applied
-        /// to 2015.02.13, which is the first trading day BEFORE the actual effective date.
-        /// </remarks>
-        /// <param name="date">The date to check the factor file for a dividend event</param>
-        /// <param name="priceFactorRatio">When this function returns true, this value will be populated
-        /// with the price factor ratio required to scale the closing value (pf_i/pf_i+1)</param>
-        /// <param name="referencePrice">When this function returns true, this value will be populated
-        /// with the reference raw price, which is the close of the provided date</param>
-        public bool HasDividendEventOnNextTradingDay(DateTime date, out decimal priceFactorRatio, out decimal referencePrice)
-        {
-            priceFactorRatio = 0;
-            referencePrice = 0;
-            var index = SortedFactorFileData.IndexOfKey(date);
-            if (index > -1 && index < SortedFactorFileData.Count - 1)
-            {
-                // grab the next key to ensure it's a dividend event
-                var thisRow = SortedFactorFileData.Values[index];
-                var nextRow = SortedFactorFileData.Values[index + 1];
-
-                // if the price factors have changed then it's a dividend event
-                if (thisRow.PriceFactor != nextRow.PriceFactor)
-                {
-                    priceFactorRatio = thisRow.PriceFactor / nextRow.PriceFactor;
-                    referencePrice = thisRow.ReferencePrice;
-                    return true;
-                }
-            }
-            return false;
-        }
-
-        /// <summary>
-        /// Returns true if the specified date is the last trading day before a split event
-        /// is to be fired
-        /// </summary>
-        /// <remarks>
-        /// NOTE: The split event in the algorithm should be fired at the end or AFTER this
-        /// date. This is the date in the file that a factor is applied, so for example MSFT
-        /// has a split on 1999.03.29, but in the factor file the split factor is applied on
-        /// 1999.03.26, which is the first trading day BEFORE the actual split date.
-        /// </remarks>
-        /// <param name="date">The date to check the factor file for a split event</param>
-        /// <param name="splitFactor">When this function returns true, this value will be populated
-        /// with the split factor ratio required to scale the closing value</param>
-        /// <param name="referencePrice">When this function returns true, this value will be populated
-        /// with the reference raw price, which is the close of the provided date</param>
-        public bool HasSplitEventOnNextTradingDay(DateTime date, out decimal splitFactor, out decimal referencePrice)
-        {
-            splitFactor = 1;
-            referencePrice = 0;
-            var index = SortedFactorFileData.IndexOfKey(date);
-            if (index > -1 && index < SortedFactorFileData.Count - 1)
-            {
-                // grab the next key to ensure it's a split event
-                var thisRow = SortedFactorFileData.Values[index];
-                var nextRow = SortedFactorFileData.Values[index + 1];
-
-                // if the split factors have changed then it's a split event
-                if (thisRow.SplitFactor != nextRow.SplitFactor)
-                {
-                    splitFactor = thisRow.SplitFactor / nextRow.SplitFactor;
-                    referencePrice = thisRow.ReferencePrice;
-                    return true;
-                }
-            }
-            return false;
-        }
+        public abstract decimal GetPriceScaleFactor(
+            DateTime searchDate,
+            DataNormalizationMode dataNormalizationMode,
+            DataMappingMode? dataMappingMode = null,
+            uint contractOffset = 0
+            );
 
         /// <summary>
         /// Writes this factor file data to an enumerable of csv lines
         /// </summary>
         /// <returns>An enumerable of lines representing this factor file</returns>
-        public IEnumerable<string> ToCsvLines()
+        public IEnumerable<string> GetFileFormat()
         {
-            foreach (var kvp in SortedFactorFileData)
-            {
-                yield return kvp.Value.ToCsv();
-            }
+            return SortedFactorFileData.SelectMany(kvp => kvp.Value.Select(row => row.GetFileFormat()));
         }
 
         /// <summary>
         /// Write the factor file to the correct place in the default Data folder
         /// </summary>
         /// <param name="symbol">The symbol this factor file represents</param>
-        public void WriteToCsv(Symbol symbol)
+        public void WriteToFile(Symbol symbol)
         {
             var filePath = LeanData.GenerateRelativeFactorFilePath(symbol);
-            File.WriteAllLines(filePath, ToCsvLines());
-        }
-
-        /// <summary>
-        /// Gets all of the splits and dividends represented by this factor file
-        /// </summary>
-        /// <param name="symbol">The symbol to ues for the dividend and split objects</param>
-        /// <param name="exchangeHours">Exchange hours used for resolving the previous trading day</param>
-        /// <param name="decimalPlaces">The number of decimal places to round the dividend's distribution to, defaulting to 2</param>
-        /// <returns>All splits and diviends represented by this factor file in chronological order</returns>
-        public List<BaseData> GetSplitsAndDividends(Symbol symbol, SecurityExchangeHours exchangeHours, int decimalPlaces = 2)
-        {
-            var dividendsAndSplits = new List<BaseData>();
-            if (SortedFactorFileData.Count == 0)
-            {
-                Log.Trace($"{symbol} has no factors!");
-                return dividendsAndSplits;
-            }
-
-            var futureFactorFileRow = SortedFactorFileData.Last().Value;
-            for (var i = SortedFactorFileData.Count - 2; i >= 0; i--)
-            {
-                var row = SortedFactorFileData.Values[i];
-                var dividend = row.GetDividend(futureFactorFileRow, symbol, exchangeHours, decimalPlaces);
-                if (dividend.Distribution != 0m)
-                {
-                    dividendsAndSplits.Add(dividend);
-                }
-
-                var split = row.GetSplit(futureFactorFileRow, symbol, exchangeHours);
-                if (split.SplitFactor != 1m)
-                {
-                    dividendsAndSplits.Add(split);
-                }
-
-                futureFactorFileRow = row;
-            }
-
-            return dividendsAndSplits.OrderBy(d => d.Time.Date).ToList();
-        }
-
-        /// <summary>
-        /// Creates a new factor file with the specified data applied.
-        /// Only <see cref="Dividend"/> and <see cref="Split"/> data types
-        /// will be used.
-        /// </summary>
-        /// <param name="data">The data to apply</param>
-        /// <param name="exchangeHours">Exchange hours used for resolving the previous trading day</param>
-        /// <returns>A new factor file that incorporates the specified dividend</returns>
-        public FactorFile Apply(List<BaseData> data, SecurityExchangeHours exchangeHours)
-        {
-            if (data.Count == 0)
-            {
-                return this;
-            }
-
-            var factorFileRows = new List<FactorFileRow>();
-            var firstEntry = SortedFactorFileData.First().Value;
-            var lastEntry = SortedFactorFileData.Last().Value;
-            factorFileRows.Add(lastEntry);
-
-            var splitsAndDividends = GetSplitsAndDividends(data[0].Symbol, exchangeHours);
-
-            var combinedData = splitsAndDividends.Concat(data)
-                .DistinctBy(e => $"{e.GetType().Name}{e.Time.ToStringInvariant(DateFormat.EightCharacter)}")
-                .OrderByDescending(d => d.Time.Date);
-
-            foreach (var datum in combinedData)
-            {
-                FactorFileRow nextEntry = null;
-                var split = datum as Split;
-                var dividend = datum as Dividend;
-                if (dividend != null)
-                {
-                    nextEntry = lastEntry.Apply(dividend, exchangeHours);
-                    lastEntry = nextEntry;
-                }
-                else if (split != null)
-                {
-                    nextEntry = lastEntry.Apply(split, exchangeHours);
-                    lastEntry = nextEntry;
-                }
-
-                if (nextEntry != null)
-                {
-                    // overwrite the latest entry -- this handles splits/dividends on the same date
-                    if (nextEntry.Date == factorFileRows.Last().Date)
-                    {
-                        factorFileRows[factorFileRows.Count - 1] = nextEntry;
-                    }
-                    else
-                    {
-                        factorFileRows.Add(nextEntry);
-                    }
-                }
-            }
-
-            var firstFactorFileRow = new FactorFileRow(firstEntry.Date, factorFileRows.Last().PriceFactor, factorFileRows.Last().SplitFactor, firstEntry.ReferencePrice == 0 ? 0 : firstEntry.ReferencePrice);
-            factorFileRows.Add(firstFactorFileRow);
-
-            return new FactorFile(Permtick, factorFileRows, FactorFileMinimumDate);
+            File.WriteAllLines(filePath, GetFileFormat());
         }
 
         /// <summary>Returns an enumerator that iterates through the collection.</summary>
         /// <returns>A <see cref="T:System.Collections.Generic.IEnumerator`1" /> that can be used to iterate through the collection.</returns>
         /// <filterpriority>1</filterpriority>
-        public IEnumerator<FactorFileRow> GetEnumerator()
+        public IEnumerator<IFactorRow> GetEnumerator()
         {
             foreach (var kvp in SortedFactorFileData)
             {
-                yield return kvp.Value;
+                foreach (var factorRow in kvp.Value)
+                {
+                    yield return factorRow;
+                }
             }
         }
 

Common/Data/Auxiliary/FactorFileRow.cs

@@ -15,13 +15,11 @@
 */
 
 using System;
-using System.Collections.Generic;
-using System.Globalization;
-using System.IO;
 using System.Linq;
-using System.Text;
-using QuantConnect.Data.Market;
+using System.Globalization;
 using QuantConnect.Securities;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
 using static QuantConnect.StringExtensions;
 
 namespace QuantConnect.Data.Auxiliary
@@ -29,7 +27,7 @@ namespace QuantConnect.Data.Auxiliary
     /// <summary>
     /// Defines a single row in a factor_factor file. This is a csv file ordered as {date, price factor, split factor, reference price}
     /// </summary>
-    public class FactorFileRow
+    public class FactorFileRow : IFactorRow
     {
         private decimal _splitFactor;
         private decimal _priceFactor;
@@ -93,31 +91,6 @@ namespace QuantConnect.Data.Auxiliary
             SplitFactor = splitFactor;
         }
 
-        /// <summary>
-        /// Reads in the factor file for the specified equity symbol
-        /// </summary>
-        public static IEnumerable<FactorFileRow> Read(Stream file, out DateTime? factorFileMinimumDate)
-        {
-            factorFileMinimumDate = null;
-
-            var streamReader = new StreamReader(file, Encoding.UTF8);
-
-            string line;
-            var lines = new List<string>();
-            while (!streamReader.EndOfStream)
-            {
-                line = streamReader.ReadLine();
-                if (!string.IsNullOrWhiteSpace(line))
-                {
-                    lines.Add(line);
-                }
-            }
-
-            streamReader.Dispose();
-
-            return Parse(lines, out factorFileMinimumDate);
-        }
-
         /// <summary>
         /// Parses the lines as factor files rows while properly handling inf entries
         /// </summary>
@@ -150,7 +123,7 @@ namespace QuantConnect.Data.Auxiliary
                 }
             }
 
-            if (factorFileMinimumDate == null && rows.Count > 0)
+            if (rows.Count > 0)
             {
                 factorFileMinimumDate = rows.Min(ffr => ffr.Date).AddDays(-1);
             }
@@ -309,9 +282,10 @@ namespace QuantConnect.Data.Auxiliary
         }
 
         /// <summary>
-        /// Writes this row to csv format
+        /// Writes factor file row into it's file format
         /// </summary>
-        public string ToCsv(string source = null)
+        /// <remarks>CSV formatted</remarks>
+        public string GetFileFormat(string source = null)
         {
             source = source == null ? "" : $",{source}";
             return $"{Date.ToStringInvariant(DateFormat.EightCharacter)}," +

Common/Data/Auxiliary/FactorFileZipHelper.cs

@@ -25,48 +25,52 @@ namespace QuantConnect.Data.Auxiliary
     /// </summary>
     public static class FactorFileZipHelper
     {
+        /// <summary>
+        /// Gets the factor file zip filename for the specified date
+        /// </summary>
+        public static string GetFactorFileZipFileName(string market, DateTime date, SecurityType securityType)
+        {
+            return Path.Combine(Globals.DataFolder, $"{securityType.SecurityTypeToLower()}/{market}/factor_files/factor_files_{date:yyyyMMdd}.zip");
+        }
+
         /// <summary>
         /// Reads the zip bytes as text and parses as FactorFileRows to create FactorFiles
         /// </summary>
-        public static IEnumerable<KeyValuePair<Symbol, FactorFile>> ReadFactorFileZip(Stream file, MapFileResolver mapFileResolver, string market)
+        public static IEnumerable<KeyValuePair<Symbol, IFactorProvider>> ReadFactorFileZip(Stream file, MapFileResolver mapFileResolver, string market, SecurityType securityType)
         {
             if (file == null || file.Length == 0)
             {
-                return new Dictionary<Symbol, FactorFile>();
+                return new Dictionary<Symbol, IFactorProvider>();
             }
 
             var keyValuePairs = (
                     from kvp in Compression.Unzip(file)
                     let filename = kvp.Key
                     let lines = kvp.Value
-                    let factorFile = SafeRead(filename, lines)
+                    let factorFile = PriceScalingExtensions.SafeRead(Path.GetFileNameWithoutExtension(filename), lines, securityType)
                     let mapFile = mapFileResolver.GetByPermtick(factorFile.Permtick)
                     where mapFile != null
-                    let sid = SecurityIdentifier.GenerateEquity(mapFile.FirstDate, mapFile.FirstTicker, market)
-                    let symbol = new Symbol(sid, mapFile.Permtick)
-                    select new KeyValuePair<Symbol, FactorFile>(symbol, factorFile)
+                    select new KeyValuePair<Symbol, IFactorProvider>(GetSymbol(mapFile, market, securityType), factorFile)
                 );
 
             return keyValuePairs;
         }
 
-        /// <summary>
-        /// Parses the contents as a FactorFile, if error returns a new empty factor file
-        /// </summary>
-        public static FactorFile SafeRead(string filename, IEnumerable<string> contents)
+        private static Symbol GetSymbol(MapFile mapFile, string market, SecurityType securityType)
         {
-            var permtick = Path.GetFileNameWithoutExtension(filename);
-            try
+            SecurityIdentifier sid;
+            switch (securityType)
             {
-                DateTime? minimumDate;
-                // FactorFileRow.Parse handles entries with 'inf' and exponential notation and provides the associated minimum tradeable date for these cases
-                // previously these cases were not handled causing an exception and returning an empty factor file
-                return new FactorFile(permtick, FactorFileRow.Parse(contents, out minimumDate), minimumDate);
-            }
-            catch
-            {
-                return new FactorFile(permtick, Enumerable.Empty<FactorFileRow>());
+                case SecurityType.Equity:
+                    sid = SecurityIdentifier.GenerateEquity(mapFile.FirstDate, mapFile.FirstTicker, market);
+                    break;
+                case SecurityType.Future:
+                    sid = SecurityIdentifier.GenerateFuture(SecurityIdentifier.DefaultDate, mapFile.Permtick, market);
+                    break;
+                default:
+                    throw new ArgumentOutOfRangeException(nameof(securityType), securityType, null);
             }
+            return new Symbol(sid, mapFile.Permtick);
         }
     }
 }

Common/Data/Auxiliary/IFactorProvider.cs

@@ -0,0 +1,49 @@
+/*
+* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+*/
+
+using System;
+using System.Collections.Generic;
+
+namespace QuantConnect.Data.Auxiliary
+{
+    /// <summary>
+    /// Providers price scaling factors for a permanent tick
+    /// </summary>
+    public interface IFactorProvider : IEnumerable<IFactorRow>
+    {
+        /// <summary>
+        /// Gets the symbol this factor file represents
+        /// </summary>
+        public string Permtick { get; }
+
+        /// <summary>
+        /// The minimum tradeable date for the symbol
+        /// </summary>
+        /// <remarks>
+        /// Some factor files have INF split values, indicating that the stock has so many splits
+        /// that prices can't be calculated with correct numerical precision.
+        /// To allow backtesting these symbols, we need to move the starting date
+        /// forward when reading the data.
+        /// Known symbols: GBSN, JUNI, NEWL
+        /// </remarks>
+        public DateTime? FactorFileMinimumDate { get; set; }
+
+        /// <summary>
+        /// Gets the price scale factor for the specified search date
+        /// </summary>
+        decimal GetPriceScaleFactor(DateTime searchDate, DataNormalizationMode dataNormalizationMode, DataMappingMode? dataMappingMode = null, uint contractOffset = 0);
+    }
+}

Common/Data/Auxiliary/IFactorRow.cs

@@ -0,0 +1,36 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+
+namespace QuantConnect.Data.Auxiliary
+{
+    /// <summary>
+    /// Factor row abstraction. <see cref="IFactorProvider"/>
+    /// </summary>
+    public interface IFactorRow
+    {
+        /// <summary>
+        /// Gets the date associated with this data
+        /// </summary>
+        DateTime Date { get; }
+
+        /// <summary>
+        /// Writes factor file row into it's file format
+        /// </summary>
+        string GetFileFormat(string source = null);
+    }
+}

Common/Data/Auxiliary/LocalDiskFactorFileProvider.cs

@@ -14,10 +14,10 @@
  *
 */
 
-using System.Collections.Concurrent;
 using System.IO;
-using QuantConnect.Interfaces;
 using QuantConnect.Util;
+using QuantConnect.Interfaces;
+using System.Collections.Concurrent;
 
 namespace QuantConnect.Data.Auxiliary
 {
@@ -28,14 +28,14 @@ namespace QuantConnect.Data.Auxiliary
     {
         private IMapFileProvider _mapFileProvider;
         private IDataProvider _dataProvider;
-        private readonly ConcurrentDictionary<Symbol, FactorFile> _cache;
+        private readonly ConcurrentDictionary<Symbol, IFactorProvider> _cache;
 
         /// <summary>
         /// Creates a new instance of the <see cref="LocalDiskFactorFileProvider"/>
         /// </summary>
         public LocalDiskFactorFileProvider()
         {
-            _cache = new ConcurrentDictionary<Symbol, FactorFile>();
+            _cache = new ConcurrentDictionary<Symbol, IFactorProvider>();
         }
 
         /// <summary>
@@ -55,54 +55,40 @@ namespace QuantConnect.Data.Auxiliary
         /// </summary>
         /// <param name="symbol">The security's symbol whose factor file we seek</param>
         /// <returns>The resolved factor file, or null if not found</returns>
-        public FactorFile Get(Symbol symbol)
+        public IFactorProvider Get(Symbol symbol)
         {
-            FactorFile factorFile;
+            symbol = symbol.GetFactorFileSymbol();
+            IFactorProvider factorFile;
             if (_cache.TryGetValue(symbol, out factorFile))
             {
                 return factorFile;
             }
 
-            var market = symbol.ID.Market;
-
             // we first need to resolve the map file to get a permtick, that's how the factor files are stored
-            var mapFileResolver = _mapFileProvider.Get(market);
+            var mapFileResolver = _mapFileProvider.Get(CorporateActionsKey.Create(symbol));
             if (mapFileResolver == null)
             {
-                return GetFactorFile(symbol, symbol.Value, market);
+                return GetFactorFile(symbol, symbol.Value);
             }
 
-            var mapFile = mapFileResolver.ResolveMapFile(symbol.ID.Symbol, symbol.ID.Date);
+            var mapFile = mapFileResolver.ResolveMapFile(symbol);
             if (mapFile.IsNullOrEmpty())
             {
-                return GetFactorFile(symbol, symbol.Value, market);
+                return GetFactorFile(symbol, symbol.Value);
             }
 
-            return GetFactorFile(symbol, mapFile.Permtick, market);
+            return GetFactorFile(symbol, mapFile.Permtick);
         }
 
         /// <summary>
         /// Checks that the factor file exists on disk, and if it does, loads it into memory
         /// </summary>
-        private FactorFile GetFactorFile(Symbol symbol, string permtick, string market)
+        private IFactorProvider GetFactorFile(Symbol symbol, string permtick)
         {
-            FactorFile factorFile = null;
+            var path = Path.Combine(Globals.CacheDataFolder, symbol.SecurityType.SecurityTypeToLower(), symbol.ID.Market, "factor_files", permtick.ToLowerInvariant() + ".csv");
 
-            var path = Path.Combine(Globals.CacheDataFolder, "equity", market, "factor_files", permtick.ToLowerInvariant() + ".csv");
-
-            var factorFileStream = _dataProvider.Fetch(path);
-            if (factorFileStream != null)
-            {
-                factorFile = FactorFile.Read(permtick, factorFileStream);
-                factorFileStream.DisposeSafely();
-                _cache.AddOrUpdate(symbol, factorFile, (s, c) => factorFile);
-            }
-            else
-            {
-                // add null value to the cache, we don't want to check the disk multiple times
-                // but keep existing value if it exists, just in case
-                _cache.AddOrUpdate(symbol, factorFile, (s, oldValue) => oldValue);
-            }
+            var factorFile = PriceScalingExtensions.SafeRead(permtick, _dataProvider.ReadLines(path), symbol.SecurityType);
+            _cache.AddOrUpdate(symbol, factorFile, (s, c) => factorFile);
             return factorFile;
         }
     }

Common/Data/Auxiliary/LocalDiskMapFileProvider.cs

@@ -14,11 +14,11 @@
  *
 */
 
-using System.Collections.Concurrent;
 using System.IO;
 using System.Threading;
-using QuantConnect.Interfaces;
 using QuantConnect.Logging;
+using QuantConnect.Interfaces;
+using System.Collections.Concurrent;
 
 namespace QuantConnect.Data.Auxiliary
 {
@@ -29,7 +29,7 @@ namespace QuantConnect.Data.Auxiliary
     public class LocalDiskMapFileProvider : IMapFileProvider
     {
         private static int _wroteTraceStatement;
-        private readonly ConcurrentDictionary<string, MapFileResolver> _cache;
+        private readonly ConcurrentDictionary<CorporateActionsKey, MapFileResolver> _cache;
         private IDataProvider _dataProvider;
 
         /// <summary>
@@ -37,7 +37,7 @@ namespace QuantConnect.Data.Auxiliary
         /// </summary>
         public LocalDiskMapFileProvider()
         {
-            _cache = new ConcurrentDictionary<string, MapFileResolver>();
+            _cache = new ConcurrentDictionary<CorporateActionsKey, MapFileResolver>();
         }
 
         /// <summary>
@@ -53,20 +53,19 @@ namespace QuantConnect.Data.Auxiliary
         /// Gets a <see cref="MapFileResolver"/> representing all the map
         /// files for the specified market
         /// </summary>
-        /// <param name="market">The equity market, for example, 'usa'</param>
+        /// <param name="corporateActionsKey">Key used to fetch a map file resolver. Specifying market and security type</param>
         /// <returns>A <see cref="MapFileRow"/> containing all map files for the specified market</returns>
-        public MapFileResolver Get(string market)
+        public MapFileResolver Get(CorporateActionsKey corporateActionsKey)
         {
-            // TODO: Consider using DataProvider to load in the files from disk to unify data fetching behavior
-            // Reference LocalDiskFactorFile, LocalZipFactorFile, and LocalZipMapFile providers for examples.
-
-            market = market.ToLowerInvariant();
-            return _cache.GetOrAdd(market, GetMapFileResolver);
+            return _cache.GetOrAdd(corporateActionsKey, GetMapFileResolver);
         }
 
-        private static MapFileResolver GetMapFileResolver(string market)
+        private MapFileResolver GetMapFileResolver(CorporateActionsKey key)
         {
-            var mapFileDirectory = Path.Combine(Globals.CacheDataFolder, "equity", market, "map_files");
+            var securityType = key.SecurityType;
+            var market = key.Market;
+
+            var mapFileDirectory =  MapFile.GetMapFilePath(market, securityType);
             if (!Directory.Exists(mapFileDirectory))
             {
                 // only write this message once per application instance
@@ -78,7 +77,7 @@ namespace QuantConnect.Data.Auxiliary
                 }
                 return MapFileResolver.Empty;
             }
-            return new MapFileResolver(MapFile.GetMapFiles(mapFileDirectory, market));
+            return new MapFileResolver(MapFile.GetMapFiles(mapFileDirectory, market, securityType, _dataProvider));
         }
     }
 }

Common/Data/Auxiliary/LocalZipFactorFileProvider.cs

@@ -14,7 +14,6 @@
 */
 
 using System;
-using System.IO;
 using QuantConnect.Util;
 using QuantConnect.Logging;
 using System.Threading.Tasks;
@@ -31,8 +30,8 @@ namespace QuantConnect.Data.Auxiliary
         private readonly object _lock;
         private IDataProvider _dataProvider;
         private IMapFileProvider _mapFileProvider;
-        private Dictionary<string, bool> _seededMarket;
-        private readonly Dictionary<Symbol, FactorFile> _factorFiles;
+        private Dictionary<CorporateActionsKey, bool> _seededMarket;
+        private readonly Dictionary<Symbol, IFactorProvider> _factorFiles;
 
         /// <summary>
         /// The cached refresh period for the factor files
@@ -45,8 +44,8 @@ namespace QuantConnect.Data.Auxiliary
         /// </summary>
         public LocalZipFactorFileProvider()
         {
-            _factorFiles = new Dictionary<Symbol, FactorFile>();
-            _seededMarket = new Dictionary<string, bool>();
+            _factorFiles = new Dictionary<Symbol, IFactorProvider>();
+            _seededMarket = new Dictionary<CorporateActionsKey, bool>();
             _lock = new object();
         }
 
@@ -73,27 +72,27 @@ namespace QuantConnect.Data.Auxiliary
         /// </summary>
         /// <param name="symbol">The security's symbol whose factor file we seek</param>
         /// <returns>The resolved factor file, or null if not found</returns>
-        public FactorFile Get(Symbol symbol)
+        public IFactorProvider Get(Symbol symbol)
         {
-            var market = symbol.ID.Market.ToLowerInvariant();
+            symbol = symbol.GetFactorFileSymbol();
+            var key = CorporateActionsKey.Create(symbol);
             lock (_lock)
             {
-                if (!_seededMarket.ContainsKey(market))
+                if (!_seededMarket.ContainsKey(key))
                 {
-                    HydrateFactorFileFromLatestZip(market);
-                    _seededMarket[market] = true;
+                    HydrateFactorFileFromLatestZip(key);
+                    _seededMarket[key] = true;
                 }
 
-                FactorFile factorFile;
-                if (_factorFiles.TryGetValue(symbol, out factorFile))
+                IFactorProvider factorFile;
+                if (!_factorFiles.TryGetValue(symbol, out factorFile))
                 {
-                    return factorFile;
+                    // Could not find factor file for symbol
+                    Log.Error($"LocalZipFactorFileProvider.Get({symbol}): No factor file found.");
+                    _factorFiles[symbol] = factorFile = symbol.GetEmptyFactorFile();
                 }
+                return factorFile;
             }
-
-            // Could not find factor file for symbol
-            Log.Error($"LocalZipFactorFileProvider.Get({symbol}): No factor file found.");
-            return null;
         }
 
         /// <summary>
@@ -104,19 +103,15 @@ namespace QuantConnect.Data.Auxiliary
             lock (_lock)
             {
                 // we clear the seeded markets so they are reloaded
-                _seededMarket = new Dictionary<string, bool>();
+                _seededMarket = new Dictionary<CorporateActionsKey, bool>();
             }
             _ = Task.Delay(CacheRefreshPeriod).ContinueWith(_ => StartExpirationTask());
         }
 
         /// Hydrate the <see cref="_factorFiles"/> from the latest zipped factor file on disk
-        private void HydrateFactorFileFromLatestZip(string market)
+        private void HydrateFactorFileFromLatestZip(CorporateActionsKey key)
         {
-            if (market != QuantConnect.Market.USA.ToLowerInvariant())
-            {
-                // don't explode for other markets which request factor files and we don't have
-                return;
-            }
+            var market = key.Market;
             // start the search with yesterday, today's file will be available tomorrow
             var todayNewYork = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork).Date;
             var date = todayNewYork.AddDays(-1);
@@ -125,8 +120,7 @@ namespace QuantConnect.Data.Auxiliary
 
             do
             {
-                var zipFileName = $"equity/{market}/factor_files/factor_files_{date:yyyyMMdd}.zip";
-                var factorFilePath = Path.Combine(Globals.DataFolder, zipFileName);
+                var factorFilePath = FactorFileZipHelper.GetFactorFileZipFileName(market, date, key.SecurityType);
 
                 // Fetch a stream for our zip from our data provider
                 var stream = _dataProvider.Fetch(factorFilePath);
@@ -134,8 +128,8 @@ namespace QuantConnect.Data.Auxiliary
                 // If the file was found we can read the file
                 if (stream != null)
                 {
-                    var mapFileResolver = _mapFileProvider.Get(market);
-                    foreach (var keyValuePair in FactorFileZipHelper.ReadFactorFileZip(stream, mapFileResolver, market))
+                    var mapFileResolver = _mapFileProvider.Get(key);
+                    foreach (var keyValuePair in FactorFileZipHelper.ReadFactorFileZip(stream, mapFileResolver, market, key.SecurityType))
                     {
                         // we merge with existing, this will allow to hold multiple markets
                         _factorFiles[keyValuePair.Key] = keyValuePair.Value;

Common/Data/Auxiliary/LocalZipMapFileProvider.cs

@@ -14,7 +14,6 @@
 */
 
 using System;
-using System.IO;
 using QuantConnect.Util;
 using QuantConnect.Logging;
 using System.Threading.Tasks;
@@ -28,7 +27,7 @@ namespace QuantConnect.Data.Auxiliary
     /// </summary>
     public class LocalZipMapFileProvider : IMapFileProvider
     {
-        private Dictionary<string, MapFileResolver> _cache;
+        private Dictionary<CorporateActionsKey, MapFileResolver> _cache;
         private IDataProvider _dataProvider;
         private object _lock;
 
@@ -44,7 +43,7 @@ namespace QuantConnect.Data.Auxiliary
         public LocalZipMapFileProvider()
         {
             _lock = new object();
-            _cache = new Dictionary<string, MapFileResolver>();
+            _cache = new Dictionary<CorporateActionsKey, MapFileResolver>();
         }
 
         /// <summary>
@@ -65,19 +64,18 @@ namespace QuantConnect.Data.Auxiliary
         /// <summary>
         /// Gets a <see cref="MapFileResolver"/> representing all the map files for the specified market
         /// </summary>
-        /// <param name="market">The equity market, for example, 'usa'</param>
+        /// <param name="corporateActionsKey">Key used to fetch a map file resolver. Specifying market and security type</param>
         /// <returns>A <see cref="MapFileResolver"/> containing all map files for the specified market</returns>
-        public MapFileResolver Get(string market)
+        public MapFileResolver Get(CorporateActionsKey corporateActionsKey)
         {
-            market = market.ToLowerInvariant();
             MapFileResolver result;
             // we use a lock so that only 1 thread loads the map file resolver while the rest wait
             // else we could have multiple threads loading the map file resolver at the same time!
             lock (_lock)
             {
-                if (!_cache.TryGetValue(market, out result))
+                if (!_cache.TryGetValue(corporateActionsKey, out result))
                 {
-                    _cache[market] = result = GetMapFileResolver(market);
+                    _cache[corporateActionsKey] = result = GetMapFileResolver(corporateActionsKey);
                 }
             }
             return result;
@@ -91,18 +89,14 @@ namespace QuantConnect.Data.Auxiliary
             lock (_lock)
             {
                 // we clear the seeded markets so they are reloaded
-                _cache = new Dictionary<string, MapFileResolver>();
+                _cache = new Dictionary<CorporateActionsKey, MapFileResolver>();
             }
             _ = Task.Delay(CacheRefreshPeriod).ContinueWith(_ => StartExpirationTask());
         }
 
-        private MapFileResolver GetMapFileResolver(string market)
+        private MapFileResolver GetMapFileResolver(CorporateActionsKey corporateActionsKey)
         {
-            if (market != QuantConnect.Market.USA.ToLowerInvariant())
-            {
-                // don't explode for other markets which request map files and we don't have
-                return MapFileResolver.Empty;
-            }
+            var market = corporateActionsKey.Market;
             var timestamp = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
             var todayNewYork = timestamp.Date;
             var yesterdayNewYork = todayNewYork.AddDays(-1);
@@ -112,7 +106,7 @@ namespace QuantConnect.Data.Auxiliary
             var date = yesterdayNewYork;
             do
             {
-                var zipFileName = Path.Combine(Globals.DataFolder, MapFileZipHelper.GetMapFileZipFileName(market, date));
+                var zipFileName = MapFileZipHelper.GetMapFileZipFileName(market, date, corporateActionsKey.SecurityType);
 
                 // Fetch a stream for our zip from our data provider
                 var stream = _dataProvider.Fetch(zipFileName);
@@ -121,7 +115,7 @@ namespace QuantConnect.Data.Auxiliary
                 if (stream != null)
                 {
                     Log.Trace("LocalZipMapFileProvider.Get({0}): Fetched map files for: {1} NY", market, date.ToShortDateString());
-                    var result =  new MapFileResolver(MapFileZipHelper.ReadMapFileZip(stream, market));
+                    var result =  new MapFileResolver(MapFileZipHelper.ReadMapFileZip(stream, market, corporateActionsKey.SecurityType));
                     stream.DisposeSafely();
                     return result;
                 }

Common/Data/Auxiliary/MapFile.cs

@@ -21,6 +21,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
+using QuantConnect.Interfaces;
 using QuantConnect.Logging;
 
 namespace QuantConnect.Data.Auxiliary
@@ -30,7 +31,7 @@ namespace QuantConnect.Data.Auxiliary
     /// </summary>
     public class MapFile : IEnumerable<MapFileRow>
     {
-        private readonly SortedDictionary<DateTime, MapFileRow> _data;
+        private readonly List<MapFileRow> _data;
 
         /// <summary>
         /// Gets the entity's unique symbol, i.e OIH.1
@@ -52,24 +53,29 @@ namespace QuantConnect.Data.Auxiliary
         /// </summary>
         public string FirstTicker { get; }
 
+        /// <summary>
+        /// Allows the consumer to specify a desired mapping mode
+        /// </summary>
+        public DataMappingMode? DataMappingMode { get; set; }
+
         /// <summary>
         /// Initializes a new instance of the <see cref="MapFile"/> class.
         /// </summary>
         public MapFile(string permtick, IEnumerable<MapFileRow> data)
         {
             Permtick = permtick.LazyToUpper();
-            _data = new SortedDictionary<DateTime, MapFileRow>(data.Distinct().ToDictionary(x => x.Date));
+            _data = data.Distinct().OrderBy(row => row.Date).ToList();
 
             // for performance we set first and last date on ctr
-            if (_data.Keys.Count == 0)
+            if (_data.Count == 0)
             {
                 FirstDate = Time.BeginningOfTime;
                 DelistingDate = Time.EndOfTime;
             }
             else
             {
-                FirstDate = _data.Keys.First();
-                DelistingDate = _data.Keys.Last();
+                FirstDate = _data[0].Date;
+                DelistingDate = _data[_data.Count - 1].Date;
             }
 
             var firstTicker = GetMappedSymbol(FirstDate, Permtick);
@@ -79,7 +85,7 @@ namespace QuantConnect.Data.Auxiliary
                 if (dotIndex > 0)
                 {
                     int value;
-                    var number = firstTicker.Substring(dotIndex + 1);
+                    var number = firstTicker.AsSpan(dotIndex + 1);
                     if (int.TryParse(number, out value))
                     {
                         firstTicker = firstTicker.Substring(0, dotIndex);
@@ -100,10 +106,14 @@ namespace QuantConnect.Data.Auxiliary
         {
             var mappedSymbol = defaultReturnValue;
             //Iterate backwards to find the most recent factor:
-            foreach (var splitDate in _data.Keys)
+            for (var i = 0; i < _data.Count; i++)
             {
-                if (splitDate < searchDate) continue;
-                mappedSymbol = _data[splitDate].MappedSymbol;
+                var row = _data[i];
+                if (row.Date < searchDate || row.DataMappingMode.HasValue && row.DataMappingMode != DataMappingMode)
+                {
+                    continue;
+                }
+                mappedSymbol = row.MappedSymbol;
                 break;
             }
             return mappedSymbol;
@@ -134,27 +144,17 @@ namespace QuantConnect.Data.Auxiliary
         /// <returns>Enumerable of csv lines</returns>
         public IEnumerable<string> ToCsvLines()
         {
-            foreach (var mapRow in _data.Values)
-            {
-                yield return mapRow.ToCsv();
-            }
-        }
-
-        /// <summary>
-        /// Reads in an entire map file for the requested symbol from the DataFolder
-        /// </summary>
-        public static MapFile Read(string permtick, string market)
-        {
-            return new MapFile(permtick, MapFileRow.Read(GetMapFilePath(permtick, market), market));
+            return _data.Select(mapRow => mapRow.ToCsv());
         }
 
         /// <summary>
         /// Writes the map file to a CSV file
         /// </summary>
         /// <param name="market">The market to save the MapFile to</param>
-        public void WriteToCsv(string market)
+        /// <param name="securityType">The map file security type</param>
+        public void WriteToCsv(string market, SecurityType securityType)
         {
-            var filePath = GetMapFilePath(Permtick, market);
+            var filePath = Path.Combine(GetMapFilePath(market, securityType), Permtick.ToLowerInvariant() + ".csv");
             var fileDir = Path.GetDirectoryName(filePath);
 
             if (!Directory.Exists(fileDir))
@@ -169,12 +169,12 @@ namespace QuantConnect.Data.Auxiliary
         /// <summary>
         /// Constructs the map file path for the specified market and symbol
         /// </summary>
-        /// <param name="permtick">The symbol as on disk, OIH or OIH.1</param>
         /// <param name="market">The market this symbol belongs to</param>
+        /// <param name="securityType">The map file security type</param>
         /// <returns>The file path to the requested map file</returns>
-        public static string GetMapFilePath(string permtick, string market)
+        public static string GetMapFilePath(string market, SecurityType securityType)
         {
-            return Path.Combine(Globals.CacheDataFolder, "equity", market, "map_files", permtick.ToLowerInvariant() + ".csv");
+            return Path.Combine(Globals.CacheDataFolder, securityType.SecurityTypeToLower(), market, "map_files");
         }
 
         #region Implementation of IEnumerable
@@ -188,7 +188,7 @@ namespace QuantConnect.Data.Auxiliary
         /// <filterpriority>1</filterpriority>
         public IEnumerator<MapFileRow> GetEnumerator()
         {
-            return _data.Values.GetEnumerator();
+            return _data.GetEnumerator();
         }
 
         /// <summary>
@@ -210,8 +210,10 @@ namespace QuantConnect.Data.Auxiliary
         /// </summary>
         /// <param name="mapFileDirectory">The map file directory path</param>
         /// <param name="market">The map file market</param>
+        /// <param name="securityType">The map file security type</param>
+        /// <param name="dataProvider">The data provider instance to use</param>
         /// <returns>An enumerable of all map files</returns>
-        public static IEnumerable<MapFile> GetMapFiles(string mapFileDirectory, string market)
+        public static IEnumerable<MapFile> GetMapFiles(string mapFileDirectory, string market, SecurityType securityType, IDataProvider dataProvider)
         {
             var mapFiles = new ConcurrentBag<MapFile>();
             Parallel.ForEach(Directory.EnumerateFiles(mapFileDirectory), file =>
@@ -219,7 +221,7 @@ namespace QuantConnect.Data.Auxiliary
                 if (file.EndsWith(".csv"))
                 {
                     var permtick = Path.GetFileNameWithoutExtension(file);
-                    var fileRead = SafeMapFileRowRead(file, market);
+                    var fileRead = SafeMapFileRowRead(file, market, securityType, dataProvider);
                     mapFiles.Add(new MapFile(permtick, fileRead));
                 }
             });
@@ -229,11 +231,11 @@ namespace QuantConnect.Data.Auxiliary
         /// <summary>
         /// Reads in the map file at the specified path, returning null if any exceptions are encountered
         /// </summary>
-        private static List<MapFileRow> SafeMapFileRowRead(string file, string market)
+        private static List<MapFileRow> SafeMapFileRowRead(string file, string market, SecurityType securityType, IDataProvider dataProvider)
         {
             try
             {
-                return MapFileRow.Read(file, market).ToList();
+                return MapFileRow.Read(file, market, securityType, dataProvider).ToList();
             }
             catch (Exception err)
             {

Common/Data/Auxiliary/MapFilePrimaryExchangeProvider.cs

@@ -48,7 +48,8 @@ namespace QuantConnect.Data.Auxiliary
             Exchange primaryExchange;
             if (!_primaryExchangeBySid.TryGetValue(securityIdentifier, out primaryExchange))
             {
-                var mapFile = _mapFileProvider.Get(securityIdentifier.Market).ResolveMapFile(securityIdentifier.Symbol, securityIdentifier.Date);
+                var mapFile = _mapFileProvider.Get(CorporateActionsKey.Create(securityIdentifier))
+                    .ResolveMapFile(securityIdentifier.Symbol, securityIdentifier.Date);
                 if (mapFile != null && mapFile.Any())
                 {
                     primaryExchange = mapFile.Last().PrimaryExchange;

Common/Data/Auxiliary/MapFileResolver.cs

@@ -20,6 +20,7 @@ using System.Collections.Generic;
 using System.Data;
 using System.IO;
 using System.Linq;
+using QuantConnect.Interfaces;
 using QuantConnect.Util;
 
 namespace QuantConnect.Data.Auxiliary
@@ -81,20 +82,6 @@ namespace QuantConnect.Data.Auxiliary
             }
         }
 
-        /// <summary>
-        /// Creates a new instance of the <see cref="MapFileResolver"/> class by reading all map files
-        /// for the specified market into memory
-        /// </summary>
-        /// <param name="dataDirectory">The root data directory</param>
-        /// <param name="market">The equity market to produce a map file collection for</param>
-        /// <returns>The collection of map files capable of mapping equity symbols within the specified market</returns>
-        public static MapFileResolver Create(string dataDirectory, string market)
-        {
-            market = market.ToLowerInvariant();
-            var path = Path.Combine(dataDirectory, "equity", market, "map_files");
-            return new MapFileResolver(MapFile.GetMapFiles(path, market));
-        }
-
         /// <summary>
         /// Gets the map file matching the specified permtick
         /// </summary>
@@ -121,7 +108,7 @@ namespace QuantConnect.Data.Auxiliary
             {
                 if (entries.Count == 0)
                 {
-                    return new MapFile(symbol, new List<MapFileRow>());
+                    return new MapFile(symbol, Enumerable.Empty<MapFileRow>());
                 }
 
                 // Return value of BinarySearch (from MSDN):
@@ -152,9 +139,9 @@ namespace QuantConnect.Data.Auxiliary
             // secondary search for exact mapping, find path than ends with symbol.csv
             MapFile mapFile;
             if (!_mapFilesByPermtick.TryGetValue(symbol, out mapFile)
-                || mapFile.FirstDate > date)
+                || mapFile.FirstDate > date && date != Time.BeginningOfTime)
             {
-                return new MapFile(symbol, new List<MapFileRow>());
+                return new MapFile(symbol, Enumerable.Empty<MapFileRow>());
             }
             return mapFile;
         }

Common/Data/Auxiliary/MapFileRow.cs

@@ -15,8 +15,8 @@
 */
 
 using System;
-using System.IO;
 using System.Linq;
+using QuantConnect.Interfaces;
 using System.Collections.Generic;
 
 namespace QuantConnect.Data.Auxiliary
@@ -41,46 +41,59 @@ namespace QuantConnect.Data.Auxiliary
         /// </summary>
         public Exchange PrimaryExchange { get; }
 
+        /// <summary>
+        /// Gets the securities mapping mode associated to this mapping row
+        /// </summary>
+        public DataMappingMode? DataMappingMode { get; }
+
         /// <summary>
         /// Initializes a new instance of the <see cref="MapFileRow"/> class.
         /// </summary>
-        public MapFileRow(DateTime date, string mappedSymbol, string primaryExchange, string market = QuantConnect.Market.USA)
-            : this(date, mappedSymbol, primaryExchange.GetPrimaryExchange(SecurityType.Equity, market))
+        public MapFileRow(DateTime date, string mappedSymbol, string primaryExchange,
+            string market = QuantConnect.Market.USA, SecurityType securityType = SecurityType.Equity, DataMappingMode? dataMappingMode = null)
+            : this(date, mappedSymbol, primaryExchange.GetPrimaryExchange(securityType, market), dataMappingMode)
         { }
         
         /// <summary>
         /// Initializes a new instance of the <see cref="MapFileRow"/> class.
         /// </summary>
-        public MapFileRow(DateTime date, string mappedSymbol, Exchange primaryExchange = null)
+        public MapFileRow(DateTime date, string mappedSymbol, Exchange primaryExchange = null, DataMappingMode? dataMappingMode = null)
         {
             Date = date;
             MappedSymbol = mappedSymbol.LazyToUpper();
             PrimaryExchange = primaryExchange ?? Exchange.UNKNOWN;
+            DataMappingMode = dataMappingMode;
         }
 
         /// <summary>
         /// Reads in the map_file for the specified equity symbol
         /// </summary>
-        public static IEnumerable<MapFileRow> Read(string file, string market)
+        public static IEnumerable<MapFileRow> Read(string file, string market, SecurityType securityType, IDataProvider dataProvider)
         {
-            return File.Exists(file)
-                ? File.ReadAllLines(file).Where(l => !string.IsNullOrWhiteSpace(l)).Select(s => Parse(s, market))
-                : Enumerable.Empty<MapFileRow>();
+            return dataProvider.ReadLines(file)
+                .Where(l => !string.IsNullOrWhiteSpace(l))
+                .Select(s => Parse(s, market, securityType));
         }
 
         /// <summary>
         /// Parses the specified line into a MapFileRow
         /// </summary>
-        public static MapFileRow Parse(string line, string market)
+        public static MapFileRow Parse(string line, string market, SecurityType securityType)
         {
             var csv = line.Split(',');
             var primaryExchange = Exchange.UNKNOWN;
-            if (csv.Length == 3)
+            DataMappingMode? mappingMode = null;
+
+            if (csv.Length >= 3)
             {
-                primaryExchange = csv[2].GetPrimaryExchange(SecurityType.Equity, market);
+                primaryExchange = csv[2].GetPrimaryExchange(securityType, market);
+            }
+            if (csv.Length >= 4)
+            {
+                mappingMode = csv[3].ParseDataMappingMode();
             }
 
-            return new MapFileRow(DateTime.ParseExact(csv[0], DateFormat.EightCharacter, null), csv[1], primaryExchange);
+            return new MapFileRow(DateTime.ParseExact(csv[0], DateFormat.EightCharacter, null), csv[1], primaryExchange, mappingMode);
         }
 
         #region Equality members
@@ -98,7 +111,8 @@ namespace QuantConnect.Data.Auxiliary
             if (ReferenceEquals(this, other)) return true;
             return Date.Equals(other.Date) &&
                    string.Equals(MappedSymbol, other.MappedSymbol) &&
-                   string.Equals(PrimaryExchange, other.PrimaryExchange);
+                   string.Equals(PrimaryExchange, other.PrimaryExchange) &&
+                   DataMappingMode == other.DataMappingMode;
         }
 
         /// <summary>
@@ -129,6 +143,7 @@ namespace QuantConnect.Data.Auxiliary
             {
                 return (Date.GetHashCode() * 397) ^ 
                        (MappedSymbol != null ? MappedSymbol.GetHashCode() : 0) ^
+                       (DataMappingMode != null ? DataMappingMode.GetHashCode() : 0) ^
                        (PrimaryExchange.GetHashCode());
             }
         }
@@ -156,8 +171,21 @@ namespace QuantConnect.Data.Auxiliary
         /// </summary>
         public string ToCsv()
         {
-            var encodedExchange = PrimaryExchange == Exchange.UNKNOWN? string.Empty : $",{PrimaryExchange.Code}";
-            return $"{Date.ToStringInvariant(DateFormat.EightCharacter)},{MappedSymbol.ToLowerInvariant()}{encodedExchange}";
+            var encodedExchange = string.Empty;
+            if (PrimaryExchange == Exchange.UNKNOWN)
+            {
+                if (DataMappingMode != null)
+                {
+                    // be lazy, only add a comma if we have a mapping mode after
+                    encodedExchange = ",";
+                }
+            }
+            else
+            {
+                encodedExchange = $",{PrimaryExchange.Code}";
+            }
+            var mappingMode = DataMappingMode != null ? $",{(int)DataMappingMode}" : string.Empty;
+            return $"{Date.ToStringInvariant(DateFormat.EightCharacter)},{MappedSymbol.ToLowerInvariant()}{encodedExchange}{mappingMode}";
         }
 
         /// <summary>
@@ -167,7 +195,8 @@ namespace QuantConnect.Data.Auxiliary
         public override string ToString()
         {
             var mainExchange = PrimaryExchange == Exchange.UNKNOWN ? string.Empty : $" - {PrimaryExchange}";
-            return Date.ToShortDateString() + ": " + MappedSymbol + mainExchange;
+            var mappingMode = DataMappingMode != null ? $" - {DataMappingMode}" : string.Empty;
+            return Date.ToShortDateString() + ": " + MappedSymbol + mainExchange + mappingMode;
         }
     }
 }

Common/Data/Auxiliary/MapFileZipHelper.cs

@@ -28,15 +28,15 @@ namespace QuantConnect.Data.Auxiliary
         /// <summary>
         /// Gets the mapfile zip filename for the specified date
         /// </summary>
-        public static string GetMapFileZipFileName(string market, DateTime date)
+        public static string GetMapFileZipFileName(string market, DateTime date, SecurityType securityType)
         {
-            return $"equity/{market}/map_files/map_files_{date:yyyyMMdd}.zip";
+            return Path.Combine(Globals.DataFolder, $"{securityType.SecurityTypeToLower()}/{market}/map_files/map_files_{date:yyyyMMdd}.zip");
         }
 
         /// <summary>
         /// Reads the zip bytes as text and parses as MapFileRows to create MapFiles
         /// </summary>
-        public static IEnumerable<MapFile> ReadMapFileZip(Stream file, string market)
+        public static IEnumerable<MapFile> ReadMapFileZip(Stream file, string market, SecurityType securityType)
         {
             if (file == null || file.Length == 0)
             {
@@ -47,7 +47,7 @@ namespace QuantConnect.Data.Auxiliary
                    let filename = kvp.Key
                    where filename.EndsWith(".csv", StringComparison.InvariantCultureIgnoreCase)
                    let lines = kvp.Value.Where(line => !string.IsNullOrEmpty(line))
-                   let mapFile = SafeRead(filename, lines, market)
+                   let mapFile = SafeRead(filename, lines, market, securityType)
                    select mapFile;
             return result;
         }
@@ -55,12 +55,12 @@ namespace QuantConnect.Data.Auxiliary
         /// <summary>
         /// Parses the contents as a MapFile, if error returns a new empty map file
         /// </summary>
-        private static MapFile SafeRead(string filename, IEnumerable<string> contents, string market)
+        private static MapFile SafeRead(string filename, IEnumerable<string> contents, string market, SecurityType securityType)
         {
             var permtick = Path.GetFileNameWithoutExtension(filename);
             try
             {
-                return new MapFile(permtick, contents.Select(s => MapFileRow.Parse(s, market)));
+                return new MapFile(permtick, contents.Select(s => MapFileRow.Parse(s, market, securityType)));
             }
             catch
             {

Common/Data/Auxiliary/MappingContractFactorProvider.cs

@@ -0,0 +1,83 @@
+using System;
+using System.Linq;
+using System.Collections.Generic;
+
+namespace QuantConnect.Data.Auxiliary
+{
+    /// <summary>
+    /// Mapping related factor provider. Factors based on price differences on mapping dates
+    /// </summary>
+    public class MappingContractFactorProvider : FactorFile<MappingContractFactorRow>
+    {
+        /// <summary>
+        ///Creates a new instance
+        /// </summary>
+        public MappingContractFactorProvider(string permtick, IEnumerable<MappingContractFactorRow> data, DateTime? factorFileMinimumDate = null)
+            : base(permtick, data, factorFileMinimumDate)
+        {
+        }
+
+        /// <summary>
+        /// Gets the price scale factor for the specified search date
+        /// </summary>
+        public override decimal GetPriceScaleFactor(DateTime searchDate, DataNormalizationMode dataNormalizationMode, DataMappingMode? dataMappingMode = null, uint contractOffset = 0)
+        {
+            if (dataNormalizationMode == DataNormalizationMode.Raw)
+            {
+                return 0;
+            }
+
+            var factor = 1m;
+            if (dataNormalizationMode is DataNormalizationMode.BackwardsPanamaCanal or DataNormalizationMode.ForwardPanamaCanal)
+            {
+                // default value depends on the data mode
+                factor = 0;
+            }
+
+            for (var i = 0; i < _reversedFactorFileDates.Count; i++)
+            {
+                var factorDate = _reversedFactorFileDates[i];
+                if (factorDate.Date < searchDate.Date)
+                {
+                    break;
+                }
+
+                var factorFileRow = SortedFactorFileData[factorDate];
+                switch (dataNormalizationMode)
+                {
+                    case DataNormalizationMode.BackwardsRatio:
+                    {
+                        var row = factorFileRow.FirstOrDefault(row => row.DataMappingMode == dataMappingMode);
+                        if (row != null && row.BackwardsRatioScale.Count > contractOffset)
+                        {
+                            factor = row.BackwardsRatioScale[(int)contractOffset];
+                        }
+                        break;
+                    }
+                    case DataNormalizationMode.BackwardsPanamaCanal:
+                    {
+                        var row = factorFileRow.FirstOrDefault(row => row.DataMappingMode == dataMappingMode);
+                        if (row != null && row.BackwardsPanamaCanalScale.Count > contractOffset)
+                        {
+                            factor = row.BackwardsPanamaCanalScale[(int)contractOffset];
+                        }
+                        break;
+                    }
+                    case DataNormalizationMode.ForwardPanamaCanal:
+                    {
+                        var row = factorFileRow.FirstOrDefault(row => row.DataMappingMode == dataMappingMode);
+                        if (row != null && row.ForwardPanamaCanalScale.Count > contractOffset)
+                        {
+                            factor = row.ForwardPanamaCanalScale[(int)contractOffset];
+                        }
+                        break;
+                    }
+                    default:
+                        throw new ArgumentOutOfRangeException();
+                }
+            }
+
+            return factor;
+        }
+    }
+}

Common/Data/Auxiliary/MappingContractFactorRow.cs

@@ -0,0 +1,99 @@
+/*
+* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+*/
+
+using System;
+using System.Linq;
+using Newtonsoft.Json;
+using System.Collections.Generic;
+
+namespace QuantConnect.Data.Auxiliary
+{
+    /// <summary>
+    /// Collection of factors for continuous contracts and their back months contracts for a specific mapping mode <see cref="DataMappingMode"/> and date
+    /// </summary>
+    public class MappingContractFactorRow : IFactorRow
+    {
+        /// <summary>
+        /// Gets the date associated with this data
+        /// </summary>
+        public DateTime Date { get; set; }
+
+        /// <summary>
+        /// Backwards ratio price scaling factors for the front month [index 0] and it's 'i' back months [index 0 + i]
+        /// <see cref="DataNormalizationMode.BackwardsRatio"/>
+        /// </summary>
+        public IReadOnlyList<decimal> BackwardsRatioScale { get; set;  } = new List<decimal>();
+
+        /// <summary>
+        /// Backwards Panama Canal price scaling factors for the front month [index 0] and it's 'i' back months [index 0 + i]
+        /// <see cref="DataNormalizationMode.BackwardsPanamaCanal"/>
+        /// </summary>
+        public IReadOnlyList<decimal> BackwardsPanamaCanalScale { get; set;  } = new List<decimal>();
+
+        /// <summary>
+        /// Forward Panama Canal price scaling factors for the front month [index 0] and it's 'i' back months [index 0 + i]
+        /// <see cref="DataNormalizationMode.ForwardPanamaCanal"/>
+        /// </summary>
+        public IReadOnlyList<decimal> ForwardPanamaCanalScale { get; set;  } = new List<decimal>();
+
+        /// <summary>
+        /// Allows the consumer to specify a desired mapping mode
+        /// </summary>
+        public DataMappingMode? DataMappingMode { get; set; }
+
+        /// <summary>
+        /// Empty constructor for json converter
+        /// </summary>
+        public MappingContractFactorRow()
+        {
+        }
+
+        /// <summary>
+        /// Writes factor file row into it's file format
+        /// </summary>
+        /// <remarks>Json formatted</remarks>
+        public string GetFileFormat(string source = null)
+        {
+            return JsonConvert.SerializeObject(this);
+        }
+
+        /// <summary>
+        /// Parses the lines as factor files rows while properly handling inf entries
+        /// </summary>
+        /// <param name="lines">The lines from the factor file to be parsed</param>
+        /// <param name="factorFileMinimumDate">The minimum date from the factor file</param>
+        /// <returns>An enumerable of factor file rows</returns>
+        public static List<MappingContractFactorRow> Parse(IEnumerable<string> lines, out DateTime? factorFileMinimumDate)
+        {
+            factorFileMinimumDate = null;
+
+            var rows = new List<MappingContractFactorRow>();
+
+            // parse factor file lines
+            foreach (var line in lines)
+            {
+                rows.Add(JsonConvert.DeserializeObject<MappingContractFactorRow>(line));
+            }
+
+            if (rows.Count > 0)
+            {
+                factorFileMinimumDate = rows.Min(ffr => ffr.Date).AddDays(-1);
+            }
+
+            return rows;
+        }
+    }
+}

Common/Data/Auxiliary/MappingExtensions.cs

@@ -14,7 +14,7 @@
  *
 */
 
-using System;
+using QuantConnect.Interfaces;
 
 namespace QuantConnect.Data.Auxiliary
 {
@@ -28,15 +28,17 @@ namespace QuantConnect.Data.Auxiliary
         /// </summary>
         /// <remarks>This method is aware of the data type being added for <see cref="SecurityType.Base"/>
         /// to the <see cref="SecurityIdentifier.Symbol"/> value</remarks>
-        /// <param name="mapFileResolver">The map file resolver</param>
-        /// <param name="symbol">The symbol that we want to map</param>
-        /// <param name="dataType">The configuration data type <see cref="SubscriptionDataConfig.Type"/></param>
+        /// <param name="mapFileProvider">The map file provider</param>
+        /// <param name="dataConfig">The configuration to fetch the map file for</param>
         /// <returns>The mapping file to use</returns>
-        public static MapFile ResolveMapFile(this MapFileResolver mapFileResolver,
-            Symbol symbol,
-            Type dataType)
+        public static MapFile ResolveMapFile(this IMapFileProvider mapFileProvider, SubscriptionDataConfig dataConfig)
         {
-            return mapFileResolver.ResolveMapFile(symbol , dataType.Name);
+            var resolver = MapFileResolver.Empty;
+            if(dataConfig.TickerShouldBeMapped())
+            {
+                resolver = mapFileProvider.Get(CorporateActionsKey.Create(dataConfig.Symbol));
+            }
+            return resolver.ResolveMapFile(dataConfig.Symbol , dataConfig.Type.Name, dataConfig.DataMappingMode);
         }
 
         /// <summary>
@@ -47,22 +49,34 @@ namespace QuantConnect.Data.Auxiliary
         /// <param name="mapFileResolver">The map file resolver</param>
         /// <param name="symbol">The symbol that we want to map</param>
         /// <param name="dataType">The string data type name if any</param>
+        /// <param name="mappingMode">The mapping mode to use if any</param>
         /// <returns>The mapping file to use</returns>
         public static MapFile ResolveMapFile(this MapFileResolver mapFileResolver,
             Symbol symbol,
-            string dataType = null)
+            string dataType = null,
+            DataMappingMode? mappingMode = null)
         {
             // Load the symbol and date to complete the mapFile checks in one statement
             var symbolID = symbol.HasUnderlying ? symbol.Underlying.ID.Symbol : symbol.ID.Symbol;
-            var date = symbol.HasUnderlying ? symbol.Underlying.ID.Date : symbol.ID.Date;
-
             if (dataType == null && symbol.SecurityType == SecurityType.Base)
             {
                 symbol.ID.Symbol.TryGetCustomDataType(out dataType);
             }
-            return mapFileResolver.ResolveMapFile(
-                symbol.SecurityType == SecurityType.Base && dataType != null ? symbolID.RemoveFromEnd($".{dataType}") : symbolID,
-                date);
+            symbolID = symbol.SecurityType == SecurityType.Base && dataType != null ? symbolID.RemoveFromEnd($".{dataType}") : symbolID;
+
+            MapFile result;
+            if (ReferenceEquals(mapFileResolver, MapFileResolver.Empty))
+            {
+                result = mapFileResolver.ResolveMapFile(symbol.Value, Time.BeginningOfTime);
+            }
+            else
+            {
+                var date = symbol.HasUnderlying ? symbol.Underlying.ID.Date : symbol.ID.Date;
+                result = mapFileResolver.ResolveMapFile(symbolID, date);
+            }
+            result.DataMappingMode = mappingMode;
+
+            return result;
         }
     }
 }

Common/Data/Auxiliary/PriceScalingExtensions.cs

@@ -0,0 +1,107 @@
+/*
+* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+*/
+
+using System;
+using System.Linq;
+using System.Collections.Generic;
+
+namespace QuantConnect.Data.Auxiliary
+{
+    /// <summary>
+    /// Set of helper methods for factor files and price scaling operations
+    /// </summary>
+    public static class PriceScalingExtensions
+    {
+        /// <summary>
+        /// Resolves the price scale for a date given a factor file and required settings
+        /// </summary>
+        /// <param name="factorFile">The factor file to use</param>
+        /// <param name="dateTime">The date for the price scale lookup</param>
+        /// <param name="normalizationMode">The price normalization mode requested</param>
+        /// <param name="contractOffset">The contract offset, useful for continuous contracts</param>
+        /// <param name="dataMappingMode">The data mapping mode used, useful for continuous contracts</param>
+        /// <returns>The price scale to use</returns>
+        public static decimal GetPriceScale(
+            this IFactorProvider factorFile,
+            DateTime dateTime,
+            DataNormalizationMode normalizationMode,
+            uint contractOffset = 0,
+            DataMappingMode? dataMappingMode = null
+            )
+        {
+            if (factorFile == null)
+            {
+                if (normalizationMode is DataNormalizationMode.BackwardsPanamaCanal or DataNormalizationMode.ForwardPanamaCanal)
+                {
+                    return 0;
+                }
+                return 1;
+            }
+
+            return factorFile.GetPriceScaleFactor(dateTime, normalizationMode, dataMappingMode, contractOffset);
+        }
+
+        /// <summary>
+        /// Determines the symbol to use to fetch it's factor file
+        /// </summary>
+        /// <remarks>This is useful for futures where the symbol to use is the canonical</remarks>
+        public static Symbol GetFactorFileSymbol(this Symbol symbol)
+        {
+            return symbol.SecurityType == SecurityType.Future ? symbol.Canonical : symbol;
+        }
+
+        /// <summary>
+        /// Helper method to return an empty factor file
+        /// </summary>
+        public static IFactorProvider GetEmptyFactorFile(this Symbol symbol)
+        {
+            if (symbol.SecurityType == SecurityType.Future)
+            {
+                return new MappingContractFactorProvider(symbol.ID.Symbol, Enumerable.Empty<MappingContractFactorRow>());
+            }
+            return new CorporateFactorProvider(symbol.ID.Symbol, Enumerable.Empty<FactorFileRow>());
+        }
+
+        /// <summary>
+        /// Parses the contents as a FactorFile, if error returns a new empty factor file
+        /// </summary>
+        public static IFactorProvider SafeRead(string permtick, IEnumerable<string> contents, SecurityType securityType)
+        {
+            try
+            {
+                DateTime? minimumDate;
+
+                contents = contents.Distinct();
+
+                if (securityType == SecurityType.Future)
+                {
+                    return new MappingContractFactorProvider(permtick, MappingContractFactorRow.Parse(contents, out minimumDate), minimumDate);
+                }
+                // FactorFileRow.Parse handles entries with 'inf' and exponential notation and provides the associated minimum tradeable date for these cases
+                // previously these cases were not handled causing an exception and returning an empty factor file
+                return new CorporateFactorProvider(permtick, FactorFileRow.Parse(contents, out minimumDate), minimumDate);
+            }
+            catch (Exception e)
+            {
+                if (securityType == SecurityType.Future)
+                {
+                    return new MappingContractFactorProvider(permtick, Enumerable.Empty<MappingContractFactorRow>());
+                }
+                return new CorporateFactorProvider(permtick, Enumerable.Empty<FactorFileRow>());
+            }
+        }
+    }
+}

Common/Data/BaseData.cs

@@ -198,7 +198,7 @@ namespace QuantConnect.Data
         /// <returns>True indicates mapping should be used</returns>
         public virtual bool RequiresMapping()
         {
-            return Symbol.SecurityType.RequiresMapping();
+            return Symbol.RequiresMapping();
         }
 
         /// <summary>

Common/Data/Market/SymbolChangedEvent.cs

@@ -69,5 +69,13 @@ namespace QuantConnect.Data.Market
         {
             return new SymbolChangedEvent(Symbol, Time, OldSymbol, NewSymbol);
         }
+
+        /// <summary>
+        /// Friendly string representation of this symbol changed event
+        /// </summary>
+        public override string ToString()
+        {
+            return $"{Time} {OldSymbol}->{NewSymbol}";
+        }
     }
 }

Common/Data/Shortable/LocalDiskShortableProvider.cs

@@ -59,28 +59,22 @@ namespace QuantConnect.Data.Shortable
             while (i <= 7)
             {
                 var shortableListFile = Path.Combine(_shortableDataDirectory.FullName, "dates", $"{localTime.AddDays(-i):yyyyMMdd}.csv");
-                var stream = _dataProvider.Fetch(shortableListFile);
 
-                if (stream != null)
+                foreach (var line in _dataProvider.ReadLines(shortableListFile))
                 {
-                    using (var streamReader = new StreamReader(stream))
-                    {
-                        foreach (var line in streamReader.ReadAllLines())
-                        {
-                            var csv = line.Split(',');
-                            var ticker = csv[0];
+                    var csv = line.Split(',');
+                    var ticker = csv[0];
 
-                            var symbol =
-                                new Symbol(
-                                    SecurityIdentifier.GenerateEquity(ticker, QuantConnect.Market.USA,
-                                        mappingResolveDate: localTime), ticker);
-                            var quantity = Parse.Long(csv[1]);
+                    var symbol = new Symbol(
+                            SecurityIdentifier.GenerateEquity(ticker, QuantConnect.Market.USA,
+                                mappingResolveDate: localTime), ticker);
+                    var quantity = Parse.Long(csv[1]);
 
-                            allSymbols[symbol] = quantity;
-                        }
-                    }
+                    allSymbols[symbol] = quantity;
+                }
 
-                    stream.Dispose();
+                if (allSymbols.Count > 0)
+                {
                     return allSymbols;
                 }
 
@@ -107,28 +101,16 @@ namespace QuantConnect.Data.Shortable
             // Implicitly trusts that Symbol.Value has been mapped and updated to the latest ticker
             var shortableSymbolFile = Path.Combine(_shortableDataDirectory.FullName, "symbols", $"{symbol.Value.ToLowerInvariant()}.csv");
 
-            using (var stream = _dataProvider.Fetch(shortableSymbolFile))
+            var localDate = localTime.Date;
+            foreach (var line in _dataProvider.ReadLines(shortableSymbolFile))
             {
-                if (stream == null)
-                {
-                    // Don't allow shorting if data is missing for the provided Symbol.
-                    return 0;
-                }
+                var csv = line.Split(',');
+                var date = Parse.DateTimeExact(csv[0], "yyyyMMdd");
 
-                var localDate = localTime.Date;
-                using (var streamReader = new StreamReader(stream))
+                if (localDate == date)
                 {
-                    foreach (var line in streamReader.ReadAllLines())
-                    {
-                        var csv = line.Split(',');
-                        var date = Parse.DateTimeExact(csv[0], "yyyyMMdd");
-                        
-                        if (localDate == date)
-                        {
-                            var quantity = Parse.Long(csv[1]);
-                            return quantity;
-                        }
-                    }
+                    var quantity = Parse.Long(csv[1]);
+                    return quantity;
                 }
             }
 

Common/Data/SubscriptionDataConfig.cs

@@ -14,13 +14,13 @@
 */
 
 using System;
-using System.Collections.Generic;
-using System.ComponentModel;
-using System.Runtime.CompilerServices;
 using NodaTime;
-using QuantConnect.Data.Consolidators;
-using QuantConnect.Securities;
 using QuantConnect.Util;
+using System.ComponentModel;
+using QuantConnect.Securities;
+using System.Collections.Generic;
+using QuantConnect.Data.Consolidators;
+using QuantConnect.Data.Market;
 using static QuantConnect.StringExtensions;
 
 namespace QuantConnect.Data
@@ -32,15 +32,20 @@ namespace QuantConnect.Data
     {
         private readonly SecurityIdentifier _sid;
 
+        /// <summary>
+        /// Event fired when there is a new symbol due to mapping
+        /// </summary>
+        public event EventHandler<NewSymbolEventArgs> NewSymbol;
+
         /// <summary>
         /// Type of data
         /// </summary>
-        public readonly Type Type;
+        public Type Type { get; }
 
         /// <summary>
         /// Security type of this data subscription
         /// </summary>
-        public readonly SecurityType SecurityType;
+        public SecurityType SecurityType => Symbol.SecurityType;
 
         /// <summary>
         /// Symbol of the asset we're requesting: this is really a perm tick!!
@@ -50,52 +55,64 @@ namespace QuantConnect.Data
         /// <summary>
         /// Trade, quote or open interest data
         /// </summary>
-        public readonly TickType TickType;
+        public TickType TickType { get; }
 
         /// <summary>
         /// Resolution of the asset we're requesting, second minute or tick
         /// </summary>
-        public readonly Resolution Resolution;
+        public Resolution Resolution { get; }
 
         /// <summary>
         /// Timespan increment between triggers of this data:
         /// </summary>
-        public readonly TimeSpan Increment;
+        public TimeSpan Increment { get; }
 
         /// <summary>
         /// True if wish to send old data when time gaps in data feed.
         /// </summary>
-        public readonly bool FillDataForward;
+        public bool FillDataForward { get; }
 
         /// <summary>
         /// Boolean Send Data from between 4am - 8am (Equities Setting Only)
         /// </summary>
-        public readonly bool ExtendedMarketHours;
+        public bool ExtendedMarketHours { get; }
 
         /// <summary>
         /// True if this subscription was added for the sole purpose of providing currency conversion rates via <see cref="CashBook.EnsureCurrencyDataFeeds"/>
         /// </summary>
-        public readonly bool IsInternalFeed;
+        public bool IsInternalFeed { get; }
 
         /// <summary>
         /// True if this subscription is for custom user data, false for QC data
         /// </summary>
-        public readonly bool IsCustomData;
+        public bool IsCustomData { get; }
 
         /// <summary>
         /// The sum of dividends accrued in this subscription, used for scaling total return prices
         /// </summary>
-        public decimal SumOfDividends;
+        public decimal SumOfDividends{ get; set; }
 
         /// <summary>
         /// Gets the normalization mode used for this subscription
         /// </summary>
-        public DataNormalizationMode DataNormalizationMode = DataNormalizationMode.Adjusted;
+        public DataNormalizationMode DataNormalizationMode { get; set; }
+
+        /// <summary>
+        /// Gets the securities mapping mode used for this subscription
+        /// </summary>
+        /// <remarks>This is particular useful when generating continuous futures</remarks>
+        public DataMappingMode DataMappingMode { get; }
+
+        /// <summary>
+        /// The continuous contract desired offset from the current front month.
+        /// For example, 0 (default) will use the front month, 1 will use the back month contract
+        /// </summary>
+        public uint ContractDepthOffset { get; }
 
         /// <summary>
         /// Price Scaling Factor:
         /// </summary>
-        public decimal PriceScaleFactor;
+        public decimal PriceScaleFactor { get; set; }
 
         /// <summary>
         /// Symbol Mapping: When symbols change over time (e.g. CHASE-> JPM) need to update the symbol requested.
@@ -104,40 +121,56 @@ namespace QuantConnect.Data
         {
             get
             {
-                return Symbol.ID.SecurityType.IsOption() && Symbol.HasUnderlying
-                    ? Symbol.Underlying.Value
-                    : Symbol.Value;
+                if (Symbol.HasUnderlying)
+                {
+                    if (SecurityType == SecurityType.Future)
+                    {
+                        return Symbol.Underlying.ID.ToString();
+                    }
+                    if (SecurityType.IsOption())
+                    {
+                        return Symbol.Underlying.Value;
+                    }
+                }
+                return Symbol.Value;
             }
             set
             {
-                Symbol = Symbol.UpdateMappedSymbol(value);
+                var oldMappedValue = MappedSymbol;
+                var oldSymbol = Symbol;
+                Symbol = Symbol.UpdateMappedSymbol(value, ContractDepthOffset);
+
+                if (MappedSymbol != oldMappedValue)
+                {
+                    NewSymbol?.Invoke(this, new NewSymbolEventArgs(Symbol, oldSymbol));
+                }
             }
         }
 
         /// <summary>
         /// Gets the market / scope of the symbol
         /// </summary>
-        public readonly string Market;
+        public string Market => Symbol.ID.Market;
 
         /// <summary>
         /// Gets the data time zone for this subscription
         /// </summary>
-        public readonly DateTimeZone DataTimeZone;
+        public DateTimeZone DataTimeZone { get; }
 
         /// <summary>
         /// Gets the exchange time zone for this subscription
         /// </summary>
-        public readonly DateTimeZone ExchangeTimeZone;
+        public DateTimeZone ExchangeTimeZone { get; }
 
         /// <summary>
         /// Consolidators that are registred with this subscription
         /// </summary>
-        public readonly ISet<IDataConsolidator> Consolidators;
+        public ISet<IDataConsolidator> Consolidators { get; }
 
         /// <summary>
         /// Gets whether or not this subscription should have filters applied to it (market hours/user filters from security)
         /// </summary>
-        public readonly bool IsFilteredSubscription;
+        public bool IsFilteredSubscription { get; }
 
         /// <summary>
         /// Constructor for Data Subscriptions
@@ -156,6 +189,9 @@ namespace QuantConnect.Data
         /// <param name="tickType">Specifies if trade or quote data is subscribed</param>
         /// <param name="isFilteredSubscription">True if this subscription should have filters applied to it (market hours/user filters from security), false otherwise</param>
         /// <param name="dataNormalizationMode">Specifies normalization mode used for this subscription</param>
+        /// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
+        /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
+        /// For example, 0 (default) will use the front month, 1 will use the back month contract</param>
         public SubscriptionDataConfig(Type objectType,
             Symbol symbol,
             Resolution resolution,
@@ -167,7 +203,9 @@ namespace QuantConnect.Data
             bool isCustom = false,
             TickType? tickType = null,
             bool isFilteredSubscription = true,
-            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted)
+            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted,
+            DataMappingMode dataMappingMode = DataMappingMode.OpenInterest,
+            uint contractDepthOffset = 0)
         {
             if (objectType == null) throw new ArgumentNullException(nameof(objectType));
             if (symbol == null) throw new ArgumentNullException(nameof(symbol));
@@ -175,7 +213,6 @@ namespace QuantConnect.Data
             if (exchangeTimeZone == null) throw new ArgumentNullException(nameof(exchangeTimeZone));
 
             Type = objectType;
-            SecurityType = symbol.ID.SecurityType;
             Resolution = resolution;
             _sid = symbol.ID;
             Symbol = symbol;
@@ -184,9 +221,10 @@ namespace QuantConnect.Data
             PriceScaleFactor = 1;
             IsInternalFeed = isInternalFeed;
             IsCustomData = isCustom;
-            Market = symbol.ID.Market;
             DataTimeZone = dataTimeZone;
+            DataMappingMode = dataMappingMode;
             ExchangeTimeZone = exchangeTimeZone;
+            ContractDepthOffset = contractDepthOffset;
             IsFilteredSubscription = isFilteredSubscription;
             Consolidators = new ConcurrentSet<IDataConsolidator>();
             DataNormalizationMode = dataNormalizationMode;
@@ -235,6 +273,9 @@ namespace QuantConnect.Data
         /// <param name="tickType">Specifies if trade or quote data is subscribed</param>
         /// <param name="isFilteredSubscription">True if this subscription should have filters applied to it (market hours/user filters from security), false otherwise</param>
         /// <param name="dataNormalizationMode">Specifies normalization mode used for this subscription</param>
+        /// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
+        /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
+        /// For example, 0 (default) will use the front month, 1 will use the back month contract</param>
         public SubscriptionDataConfig(SubscriptionDataConfig config,
             Type objectType = null,
             Symbol symbol = null,
@@ -247,7 +288,9 @@ namespace QuantConnect.Data
             bool? isCustom = null,
             TickType? tickType = null,
             bool? isFilteredSubscription = null,
-            DataNormalizationMode? dataNormalizationMode = null)
+            DataNormalizationMode? dataNormalizationMode = null,
+            DataMappingMode? dataMappingMode = null,
+            uint? contractDepthOffset = null)
             : this(
             objectType ?? config.Type,
             symbol ?? config.Symbol,
@@ -260,7 +303,9 @@ namespace QuantConnect.Data
             isCustom ?? config.IsCustomData,
             tickType ?? config.TickType,
             isFilteredSubscription ?? config.IsFilteredSubscription,
-            dataNormalizationMode ?? config.DataNormalizationMode
+            dataNormalizationMode ?? config.DataNormalizationMode,
+            dataMappingMode ?? config.DataMappingMode,
+            contractDepthOffset ?? config.ContractDepthOffset
             )
         {
             PriceScaleFactor = config.PriceScaleFactor;
@@ -287,7 +332,9 @@ namespace QuantConnect.Data
                 && IsInternalFeed == other.IsInternalFeed
                 && IsCustomData == other.IsCustomData
                 && DataTimeZone.Equals(other.DataTimeZone)
+                && DataMappingMode == other.DataMappingMode
                 && ExchangeTimeZone.Equals(other.ExchangeTimeZone)
+                && ContractDepthOffset == other.ContractDepthOffset
                 && IsFilteredSubscription == other.IsFilteredSubscription;
         }
 
@@ -324,8 +371,10 @@ namespace QuantConnect.Data
                 hashCode = (hashCode*397) ^ ExtendedMarketHours.GetHashCode();
                 hashCode = (hashCode*397) ^ IsInternalFeed.GetHashCode();
                 hashCode = (hashCode*397) ^ IsCustomData.GetHashCode();
+                hashCode = (hashCode*397) ^ DataMappingMode.GetHashCode();
                 hashCode = (hashCode*397) ^ DataTimeZone.Id.GetHashCode();// timezone hash is expensive, use id instead
                 hashCode = (hashCode*397) ^ ExchangeTimeZone.Id.GetHashCode();// timezone hash is expensive, use id instead
+                hashCode = (hashCode*397) ^ ContractDepthOffset.GetHashCode();
                 hashCode = (hashCode*397) ^ IsFilteredSubscription.GetHashCode();
                 return hashCode;
             }
@@ -356,7 +405,26 @@ namespace QuantConnect.Data
         /// <filterpriority>2</filterpriority>
         public override string ToString()
         {
-            return Invariant($"{Symbol.Value},{MappedSymbol},{Resolution},{Type.Name},{TickType},{DataNormalizationMode}{(IsInternalFeed ? ",Internal" : string.Empty)}");
+            return Invariant($"{Symbol.Value},#{ContractDepthOffset},{MappedSymbol},{Resolution},{Type.Name},{TickType},{DataNormalizationMode},{DataMappingMode}{(IsInternalFeed ? ",Internal" : string.Empty)}");
+        }
+
+        public class NewSymbolEventArgs : EventArgs
+        {
+            /// <summary>
+            /// The old symbol instance
+            /// </summary>
+            public Symbol Old { get; }
+
+            /// <summary>
+            /// The new symbol instance
+            /// </summary>
+            public Symbol New { get; }
+
+            public NewSymbolEventArgs(Symbol @new, Symbol old)
+            {
+                New = @new;
+                Old = old;
+            }
         }
     }
 }

Common/Data/SubscriptionDataConfigExtensions.cs

@@ -15,6 +15,7 @@
 
 using System.Collections.Generic;
 using System.Linq;
+using QuantConnect.Util;
 
 namespace QuantConnect.Data
 {
@@ -120,6 +121,44 @@ namespace QuantConnect.Data
             return config.GetBaseDataInstance().RequiresMapping();
         }
 
+        /// <summary>
+        /// Will determine if price scaling should be used for this subscription configuration
+        /// </summary>
+        /// <param name="config">The subscription data configuration we are processing</param>
+        /// <remarks>One of the objectives of this method is to normalize the 'use price scale'
+        /// check and void code duplication and related issues</remarks>
+        /// <returns>True if ticker prices should be scaled</returns>
+        public static bool PricesShouldBeScaled(this SubscriptionDataConfig config)
+        {
+            if (config.IsCustomData || config.Symbol.Value.Contains("UNIVERSE"))
+            {
+                return false;
+            }
+
+            if(config.SecurityType == SecurityType.Equity)
+            {
+                return true;
+            }
+            if (config.SecurityType == SecurityType.Future && config.Symbol.IsCanonical())
+            {
+                return LeanData.IsCommonLeanDataType(config.Type);
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// Will determine if splits and dividends should be used for this subscription configuration
+        /// </summary>
+        /// <param name="config">The subscription data configuration we are processing</param>
+        /// <remarks>Different than <see cref="PricesShouldBeScaled"/> because prices could be scale and no split and dividends
+        /// really exist, like in the continuous futures case</remarks>
+        /// <returns>True if this configuration requires split and divided handling</returns>
+        public static bool EmitSplitsAndDividends(this SubscriptionDataConfig config)
+        {
+            return !config.IsCustomData && !config.Symbol.Value.Contains("UNIVERSE") && config.SecurityType == SecurityType.Equity;
+        }
+
         /// <summary>
         /// Initializes a new instance of the <see cref="BaseData"/> type defined in <paramref name="config"/> with the symbol properly set
         /// </summary>

Common/Data/UniverseSelection/Universe.cs

@@ -251,7 +251,9 @@ namespace QuantConnect.Data.UniverseSelection
                 UniverseSettings.FillForward,
                 UniverseSettings.ExtendedMarketHours,
                 dataNormalizationMode: UniverseSettings.DataNormalizationMode,
-                subscriptionDataTypes: UniverseSettings.SubscriptionDataTypes);
+                subscriptionDataTypes: UniverseSettings.SubscriptionDataTypes,
+                dataMappingMode: UniverseSettings.DataMappingMode,
+                contractDepthOffset: (uint)Math.Abs(UniverseSettings.ContractDepthOffset));
             return result.Select(config => new SubscriptionRequest(isUniverseSubscription: false,
                 universe: this,
                 security: security,

Common/Data/UniverseSelection/UniverseSettings.cs

@@ -57,6 +57,18 @@ namespace QuantConnect.Data.UniverseSelection
         /// </summary>
         public DataNormalizationMode DataNormalizationMode;
 
+        /// <summary>
+        /// Defines how universe data is mapped together
+        /// </summary>
+        /// <remarks>This is particular useful when generating continuous futures</remarks>
+        public DataMappingMode DataMappingMode;
+
+        /// <summary>
+        /// The continuous contract desired offset from the current front month.
+        /// For example, 0 (default) will use the front month, 1 will use the back month contra
+        /// </summary>
+        public int ContractDepthOffset;
+
         /// <summary>
         /// Allows a universe to specify which data types to add for a selected symbol
         /// </summary>
@@ -71,11 +83,17 @@ namespace QuantConnect.Data.UniverseSelection
         /// <param name="extendedMarketHours">True to allow extended market hours data, false otherwise</param>
         /// <param name="minimumTimeInUniverse">Defines the minimum amount of time a security must remain in the universe before being removed</param>
         /// <param name="dataNormalizationMode">Defines how universe data is normalized before being send into the algorithm</param>
-        public UniverseSettings(Resolution resolution, decimal leverage, bool fillForward, bool extendedMarketHours, TimeSpan minimumTimeInUniverse, DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted)
+        /// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
+        /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
+        /// For example, 0 (default) will use the front month, 1 will use the back month contract</param>
+        public UniverseSettings(Resolution resolution, decimal leverage, bool fillForward, bool extendedMarketHours, TimeSpan minimumTimeInUniverse, DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted,
+            DataMappingMode dataMappingMode = DataMappingMode.OpenInterest, int contractDepthOffset = 0)
         {
             Resolution = resolution;
             Leverage = leverage;
             FillForward = fillForward;
+            DataMappingMode = dataMappingMode;
+            ContractDepthOffset = contractDepthOffset;
             ExtendedMarketHours = extendedMarketHours;
             MinimumTimeInUniverse = minimumTimeInUniverse;
             DataNormalizationMode = dataNormalizationMode;
@@ -89,6 +107,8 @@ namespace QuantConnect.Data.UniverseSelection
             Resolution = universeSettings.Resolution;
             Leverage = universeSettings.Leverage;
             FillForward = universeSettings.FillForward;
+            DataMappingMode = universeSettings.DataMappingMode;
+            ContractDepthOffset = universeSettings.ContractDepthOffset;
             ExtendedMarketHours = universeSettings.ExtendedMarketHours;
             MinimumTimeInUniverse = universeSettings.MinimumTimeInUniverse;
             DataNormalizationMode = universeSettings.DataNormalizationMode;

Common/Exchange.cs

@@ -183,6 +183,42 @@ namespace QuantConnect
         public static Exchange ISE_MERCURY { get; }
             = new("ISE_MERCURY", "J", "International Securities Options Exchange MERCURY", QuantConnect.Market.USA, SecurityType.Option);
 
+        /// <summary>
+        /// The Chicago Mercantile Exchange (CME), is an organized exchange for the trading of futures and options.
+        /// </summary>
+        public static Exchange CME { get; }
+            = new("CME", "CME", "Futures and Options Chicago Mercantile Exchange", QuantConnect.Market.CME, SecurityType.Future, SecurityType.FutureOption);
+
+        /// <summary>
+        /// The Chicago Board of Trade (CBOT) is a commodity exchange
+        /// </summary>
+        public static Exchange CBOT { get; }
+            = new("CBOT", "CBOT", " Chicago Board of Trade Commodity Exchange", QuantConnect.Market.CBOT, SecurityType.Future, SecurityType.FutureOption);
+
+        /// <summary>
+        /// Cboe Futures Exchange
+        /// </summary>
+        public static Exchange CFE { get; }
+            = new("CFE", "CFE", "CFE Futures Exchange", QuantConnect.Market.CFE, SecurityType.Future);
+
+        /// <summary>
+        /// COMEX Commodity Exchange
+        /// </summary>
+        public static Exchange COMEX { get; }
+            = new("COMEX", "COMEX", "COMEX Futures Exchange", QuantConnect.Market.COMEX, SecurityType.Future);
+
+        /// <summary>
+        /// The Intercontinental Exchange
+        /// </summary>
+        public static Exchange ICE { get; }
+            = new("ICE", "ICE", "The Intercontinental Exchange", QuantConnect.Market.ICE, SecurityType.Future);
+
+        /// <summary>
+        /// New York Mercantile Exchange
+        /// </summary>
+        public static Exchange NYMEX { get; }
+            = new("NYMEX", "NYMEX", "New York Mercantile Exchange", QuantConnect.Market.NYMEX, SecurityType.Future, SecurityType.FutureOption);
+
         /// <summary>
         /// Exchange description
         /// </summary>

Common/Extensions.cs

@@ -55,6 +55,7 @@ using NodaTime.TimeZones;
 using QuantConnect.Configuration;
 using QuantConnect.Data.Auxiliary;
 using QuantConnect.Exceptions;
+using QuantConnect.Securities.Future;
 using QuantConnect.Securities.FutureOption;
 using QuantConnect.Securities.Option;
 
@@ -171,7 +172,26 @@ namespace QuantConnect
                 catch (WebException ex)
                 {
                     Log.Error(ex, $"DownloadData(): failed for: '{url}'");
-                    // If server returned an error most likely on this day there is no data we are going to the next cycle
+                    return null;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Helper method to download a provided url as a byte array
+        /// </summary>
+        /// <param name="url">The url to download data from</param>
+        public static byte[] DownloadByteArray(this string url)
+        {
+            using (var wc = new HttpClient())
+            {
+                try
+                {
+                    return wc.GetByteArrayAsync(url).Result;
+                }
+                catch (Exception ex)
+                {
+                    Log.Error(ex, $"DownloadByteArray(): failed for: '{url}'");
                     return null;
                 }
             }
@@ -2210,6 +2230,34 @@ namespace QuantConnect
             }
         }
 
+        /// <summary>
+        /// Converts the specified string to its corresponding DataMappingMode
+        /// </summary>
+        /// <remarks>This method provides faster performance than enum parse</remarks>
+        /// <param name="dataMappingMode">The dataMappingMode string value</param>
+        /// <returns>The DataMappingMode value</returns>
+        public static DataMappingMode? ParseDataMappingMode(this string dataMappingMode)
+        {
+            if (string.IsNullOrEmpty(dataMappingMode))
+            {
+                return null;
+            }
+            switch (dataMappingMode.LazyToLower())
+            {
+                case "0":
+                case "lasttradingday":
+                    return DataMappingMode.LastTradingDay;
+                case "1":
+                case "firstdaymonth":
+                    return DataMappingMode.FirstDayMonth;
+                case "2":
+                case "openinterest":
+                    return DataMappingMode.OpenInterest;
+                default:
+                    throw new ArgumentException($"Unexpected DataMappingMode: {dataMappingMode}");
+            }
+        }
+
         /// <summary>
         /// Converts the specified <paramref name="securityType"/> value to its corresponding lower-case string representation
         /// </summary>
@@ -2892,30 +2940,6 @@ namespace QuantConnect
             }
         }
 
-        /// <summary>
-        /// Normalizes the specified price based on the DataNormalizationMode
-        /// </summary>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static decimal GetNormalizedPrice(this SubscriptionDataConfig config, decimal price)
-        {
-            switch (config.DataNormalizationMode)
-            {
-                case DataNormalizationMode.Raw:
-                    return price;
-
-                // the price scale factor will be set accordingly based on the mode in update scale factors
-                case DataNormalizationMode.Adjusted:
-                case DataNormalizationMode.SplitAdjusted:
-                    return price * config.PriceScaleFactor;
-
-                case DataNormalizationMode.TotalReturn:
-                    return (price * config.PriceScaleFactor) + config.SumOfDividends;
-
-                default:
-                    throw new ArgumentOutOfRangeException();
-            }
-        }
-
         /// <summary>
         /// Gets the delisting date for the provided Symbol
         /// </summary>
@@ -2927,7 +2951,7 @@ namespace QuantConnect
             switch (symbol.ID.SecurityType)
             {
                 case SecurityType.Future:
-                    return symbol.ID.Date;
+                    return symbol.ID.Date == SecurityIdentifier.DefaultDate ? Time.EndOfTime : symbol.ID.Date;
                 case SecurityType.Option:
                     return OptionSymbol.GetLastDayOfTrading(symbol);
                 case SecurityType.FutureOption:
@@ -2949,6 +2973,62 @@ namespace QuantConnect
                 && type.Equals(typeof(T).Name, StringComparison.InvariantCultureIgnoreCase);
         }
 
+        /// <summary>
+        /// Helper method that will return a back month, with future expiration, future contract based on the given offset
+        /// </summary>
+        /// <param name="symbol">The none canonical future symbol</param>
+        /// <param name="offset">The quantity of contracts to move into the future expiration chain</param>
+        /// <returns>A new future expiration symbol instance</returns>
+        public static Symbol AdjustSymbolByOffset(this Symbol symbol, uint offset)
+        {
+            if (symbol.SecurityType != SecurityType.Future || symbol.IsCanonical())
+            {
+                throw new InvalidOperationException("Adjusting a symbol by an offset is currently only supported for non canonical futures");
+            }
+
+            var expiration = symbol.ID.Date;
+            for (var i = 0; i < offset; i++)
+            {
+                var expiryFunction = FuturesExpiryFunctions.FuturesExpiryFunction(symbol);
+                DateTime newExpiration;
+                // for the current expiration we add a month to get the next one
+                var monthOffset = 0;
+                do
+                {
+                    monthOffset++;
+                    newExpiration = expiryFunction(expiration.AddMonths(monthOffset)).Date;
+                } while (newExpiration <= expiration);
+
+                expiration = newExpiration;
+                symbol = Symbol.CreateFuture(symbol.ID.Symbol, symbol.ID.Market, newExpiration);
+            }
+
+            return symbol;
+        }
+
+        /// <summary>
+        /// Helper method to stream read lines from a file
+        /// </summary>
+        /// <param name="dataProvider">The data provider to use</param>
+        /// <param name="file">The file path to read from</param>
+        /// <returns>Enumeration of lines in file</returns>
+        public static IEnumerable<string> ReadLines(this IDataProvider dataProvider, string file)
+        {
+            var stream = dataProvider.Fetch(file);
+            if (stream == null)
+            {
+                yield break;
+            }
+
+            using (var streamReader = new StreamReader(stream))
+            {
+                while (!streamReader.EndOfStream)
+                {
+                    yield return streamReader.ReadLine();
+                }
+            }
+        }
+
         /// <summary>
         /// Returns the delisted liquidation time for a given delisting warning and exchange hours
         /// </summary>
@@ -3086,22 +3166,50 @@ namespace QuantConnect
         /// Normalize prices based on configuration
         /// </summary>
         /// <param name="data">Data to be normalized</param>
-        /// <param name="config">Price scale</param>
-        /// <returns></returns>
-        public static BaseData Normalize(this BaseData data, SubscriptionDataConfig config)
+        /// <param name="factor">Price scale</param>
+        /// <param name="normalizationMode">The price scaling normalization mode</param>
+        /// <param name="sumOfDividends">The current dividend sum</param>
+        /// <returns>The provided data point adjusted</returns>
+        public static BaseData Normalize(this BaseData data, decimal factor, DataNormalizationMode normalizationMode, decimal sumOfDividends)
         {
-            return data?.Scale(p => config.GetNormalizedPrice(p), 1/config.PriceScaleFactor);
+            switch (normalizationMode)
+            {
+                case DataNormalizationMode.Adjusted:
+                case DataNormalizationMode.SplitAdjusted:
+                    return data?.Scale(p => p * factor, 1/factor);
+                case DataNormalizationMode.TotalReturn:
+                    return data.Scale(p => p * factor + sumOfDividends, 1/factor);
+
+                case DataNormalizationMode.BackwardsRatio:
+                    return data.Scale(p => p * factor, 1);
+                case DataNormalizationMode.BackwardsPanamaCanal:
+                    return data.Scale(p => p + factor, 1);
+                case DataNormalizationMode.ForwardPanamaCanal:
+                    return data.Scale(p => p + factor, 1);
+
+                case DataNormalizationMode.Raw:
+                default:
+                    return data;
+            }
         }
 
         /// <summary>
-        /// Adjust prices based on price scale
+        /// Helper method to determine the right data normalization mode to use by default
         /// </summary>
-        /// <param name="data">Data to be adjusted</param>
-        /// <param name="scale">Price scale</param>
-        /// <returns></returns>
-        public static BaseData Adjust(this BaseData data, decimal scale)
+        public static DataNormalizationMode GetDefaultNormalizationMode(this UniverseSettings universeSettings, SecurityType securityType)
         {
-            return data?.Scale(p => p * scale, 1/scale);
+            switch (securityType)
+            {
+                case SecurityType.Future:
+                    if (universeSettings.DataNormalizationMode is DataNormalizationMode.BackwardsRatio
+                        or DataNormalizationMode.BackwardsPanamaCanal or DataNormalizationMode.ForwardPanamaCanal)
+                    {
+                        return universeSettings.DataNormalizationMode;
+                    }
+                    return DataNormalizationMode.BackwardsRatio;
+                default:
+                    return universeSettings.DataNormalizationMode;
+            }
         }
 
         /// <summary>
@@ -3453,29 +3561,19 @@ namespace QuantConnect
             }
         }
 
-        /// <summary>
-        /// Read all lines from a stream reader
-        /// </summary>
-        /// <param name="reader">Stream reader to read from</param>
-        /// <returns>Enumerable of lines in stream</returns>
-        public static IEnumerable<string> ReadAllLines(this StreamReader reader)
-        {
-            string line;
-            while ((line = reader.ReadLine()) != null)
-            {
-                yield return line;
-            }
-        }
-
         /// <summary>
         /// Determine if this SecurityType requires mapping
         /// </summary>
-        /// <param name="securityType">Type to check</param>
+        /// <param name="symbol">Type to check</param>
         /// <returns>True if it needs to be mapped</returns>
-        public static bool RequiresMapping(this SecurityType securityType)
+        public static bool RequiresMapping(this Symbol symbol)
         {
-            switch (securityType)
+            switch (symbol.SecurityType)
             {
+                case SecurityType.Base:
+                    return symbol.HasUnderlying && symbol.Underlying.RequiresMapping();
+                case SecurityType.Future:
+                    return symbol.IsCanonical();
                 case SecurityType.Equity:
                 case SecurityType.Option:
                     return true;

Common/Global.cs

@@ -692,7 +692,44 @@ namespace QuantConnect
         /// <summary>
         /// The split adjusted price plus dividends
         /// </summary>
-        TotalReturn
+        TotalReturn,
+        /// <summary>
+        /// Eliminates price jumps between two consecutive contracts, adding a factor based on the difference of their prices.
+        /// </summary>
+        /// <remarks>First contract is the true one, factor 0</remarks>
+        ForwardPanamaCanal,
+        /// <summary>
+        /// Eliminates price jumps between two consecutive contracts, adding a factor based on the difference of their prices.
+        /// </summary>
+        /// <remarks>Last contract is the true one, factor 0</remarks>
+        BackwardsPanamaCanal,
+        /// <summary>
+        /// Eliminates price jumps between two consecutive contracts, multiplying the prices by their ratio.
+        /// </summary>
+        /// <remarks>Last contract is the true one, factor 1</remarks>
+        BackwardsRatio
+    }
+
+    /// <summary>
+    /// Continuous contracts mapping modes
+    /// </summary>
+    public enum DataMappingMode
+    {
+        /// <summary>
+        /// The contract maps on the previous day of expiration of the front month.
+        /// </summary>
+        LastTradingDay,
+        /// <summary>
+        /// The contract maps on the first date of the delivery month of the front month. If the contract expires prior to this date,
+        /// then it rolls on the contract's last trading date instead.
+        /// </summary>
+        /// <remarks>For example Crude Oil WTI (CL) 'DEC 2021 CLZ1' contract expires on Nov 19 2021, so mapping date will be it's expiration date</remarks>
+        /// <remarks>Another example Corn 'DEC 2021 ZCZ1' contract expires on Dec 14 2021, so mapping date will be Dec 1st</remarks>
+        FirstDayMonth,
+        /// <summary>
+        /// The contract maps when the back month contract has a higher volume that the current front month.
+        /// </summary>
+        OpenInterest
     }
 
     /// <summary>
@@ -852,6 +889,26 @@ namespace QuantConnect
                         return Exchange.UNKNOWN;
                 }
             }
+            else if (securityType == SecurityType.Future || securityType == SecurityType.FutureOption)
+            {
+                switch (exchange.LazyToUpper())
+                {
+                    case "CME":
+                        return Exchange.CME;
+                    case "CBOT":
+                        return Exchange.CBOT;
+                    case "NYMEX":
+                        return Exchange.NYMEX;
+                    case "ICE":
+                        return Exchange.ICE;
+                    case "CFE":
+                        return Exchange.CFE;
+                    case "COMEX":
+                        return Exchange.COMEX;
+                    default:
+                        return Exchange.UNKNOWN;
+                }
+            }
             return Exchange.UNKNOWN;
         }
     }

Common/Interfaces/IAlgorithm.cs

@@ -560,7 +560,10 @@ namespace QuantConnect.Interfaces
         /// <param name="fillDataForward">If true, returns the last available data even if none in that timeslice.</param>
         /// <param name="leverage">leverage for this security</param>
         /// <param name="extendedMarketHours">ExtendedMarketHours send in data from 4am - 8pm, not used for FOREX</param>
-        Security AddSecurity(SecurityType securityType, string symbol, Resolution? resolution, string market, bool fillDataForward, decimal leverage, bool extendedMarketHours);
+        /// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
+        /// <param name="dataNormalizationMode">The price scaling mode to use for the security</param>
+        Security AddSecurity(SecurityType securityType, string symbol, Resolution? resolution, string market, bool fillDataForward, decimal leverage, bool extendedMarketHours,
+            DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null);
 
         /// <summary>
         /// Creates and adds a new single <see cref="Future"/> contract to the algorithm

Common/Interfaces/IFactorFileProvider.cs

@@ -38,6 +38,6 @@ namespace QuantConnect.Interfaces
         /// </summary>
         /// <param name="symbol">The security's symbol whose factor file we seek</param>
         /// <returns>The resolved factor file, or null if not found</returns>
-        FactorFile Get(Symbol symbol);
+        IFactorProvider Get(Symbol symbol);
     }
 }

Common/Interfaces/IMapFileProvider.cs

@@ -35,8 +35,8 @@ namespace QuantConnect.Interfaces
         /// Gets a <see cref="MapFileResolver"/> representing all the map
         /// files for the specified market
         /// </summary>
-        /// <param name="market">The equity market, for example, 'usa'</param>
+        /// <param name="corporateActionsKey">Key used to fetch a map file resolver. Specifying market and security type</param>
         /// <returns>A <see cref="MapFileResolver"/> containing all map files for the specified market</returns>
-        MapFileResolver Get(string market);
+        MapFileResolver Get(CorporateActionsKey corporateActionsKey);
     }
 }

Common/Interfaces/ISubscriptionDataConfigService.cs

@@ -40,7 +40,9 @@ namespace QuantConnect.Interfaces
             bool isFilteredSubscription = true,
             bool isInternalFeed = false,
             bool isCustomData = false,
-            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted
+            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted,
+            DataMappingMode dataMappingMode = DataMappingMode.OpenInterest,
+            uint contractDepthOffset = 0
             );
 
         /// <summary>
@@ -57,7 +59,9 @@ namespace QuantConnect.Interfaces
             bool isInternalFeed = false,
             bool isCustomData = false,
             List<Tuple<Type, TickType>> subscriptionDataTypes = null,
-            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted
+            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted,
+            DataMappingMode dataMappingMode = DataMappingMode.OpenInterest,
+            uint contractDepthOffset = 0
             );
 
         /// <summary>

Common/Securities/Future/Future.cs

@@ -79,7 +79,7 @@ namespace QuantConnect.Securities.Future
             SettlementType = SettlementType.Cash;
             Holdings = new FutureHolding(this, currencyConverter);
             _symbolProperties = symbolProperties;
-            SetFilter(TimeSpan.Zero, TimeSpan.FromDays(35));
+            SetFilter(TimeSpan.Zero, TimeSpan.Zero);
         }
 
         /// <summary>
@@ -126,7 +126,7 @@ namespace QuantConnect.Securities.Future
             SettlementType = SettlementType.Cash;
             Holdings = new FutureHolding(this, currencyConverter);
             _symbolProperties = symbolProperties;
-            SetFilter(TimeSpan.Zero, TimeSpan.FromDays(35));
+            SetFilter(TimeSpan.Zero, TimeSpan.Zero);
             Underlying = underlying;
         }
 

Common/Securities/Future/FutureExpirationCycles.cs

@@ -40,6 +40,16 @@ namespace QuantConnect.Securities
         /// </summary>
         public static readonly int[] AllYear = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 };
 
+        /// <summary>
+        /// GJMQVZ Cycle
+        /// </summary>
+        public static readonly int[] GJMQVZ = { 2, 4, 6, 8, 10, 12 };
+
+        /// <summary>
+        /// GJKMNQVZ Cycle
+        /// </summary>
+        public static readonly int[] GJKMNQVZ = { 2, 4, 5, 6, 7, 8, 10, 12 };
+
         /// <summary>
         /// HMUZ Cycle
         /// </summary>
@@ -50,6 +60,26 @@ namespace QuantConnect.Securities
         /// </summary>
         public static readonly int[] HKNUZ = { 3, 5, 7, 9, 12 };
 
+        /// <summary>
+        /// HKNV Cycle
+        /// </summary>
+        public static readonly int[] HKNV = { 3, 5, 7, 10 };
+
+        /// <summary>
+        /// HKNVZ Cycle
+        /// </summary>
+        public static readonly int[] HKNVZ = { 3, 5, 7, 10, 12 };
+
+        /// <summary>
+        /// FHKNUX Cycle
+        /// </summary>
+        public static readonly int[] FHKNUX = { 1, 3, 5, 7, 9, 11 };
+
+        /// <summary>
+        /// FHJKQUVX Cycle
+        /// </summary>
+        public static readonly int[] FHJKQUVX = { 1, 3, 4, 5, 8, 9, 10, 11 };
+
         /// <summary>
         /// HKNUVZ Cycle
         /// </summary>
@@ -58,7 +88,12 @@ namespace QuantConnect.Securities
         /// <summary>
         /// FHKNQUVZ Cycle
         /// </summary>
-        public static readonly int[] FHKNQUVZ = { 1, 3, 5, 7, 9, 10, 12 };
+        public static readonly int[] FHKNUVZ = { 1, 3, 5, 7, 9, 10, 12 };
+
+        /// <summary>
+        /// FHKMQUVZ Cycle
+        /// </summary>
+        public static readonly int[] FHKNQUVZ = { 1, 3, 5, 7, 8, 9, 10, 12 };
 
         /// <summary>
         /// FHKNQUX Cycle

Common/Securities/Future/FutureMarginModel.cs

@@ -239,32 +239,25 @@ namespace QuantConnect.Securities.Future
         {
             lock (DataFolderSymbolLock)
             {
-                var stream = _dataProvider.Fetch(file);
-                if (stream == null)
+                // skip the first header line, also skip #'s as these are comment lines
+                var marginRequirementsEntries = _dataProvider.ReadLines(file)
+                    .Where(x => !x.StartsWith("#") && !string.IsNullOrWhiteSpace(x))
+                    .Skip(1)
+                    .Select(FromCsvLine)
+                    .OrderBy(x => x.Date)
+                    .ToArray();
+
+                if(marginRequirementsEntries.Length == 0)
                 {
                     Log.Trace($"Unable to locate future margin requirements file. Defaulting to zero margin for this symbol. File: {file}");
 
                     return new[] {
-                                new MarginRequirementsEntry
-                                {
-                                  Date = DateTime.MinValue
-                                }
-                            };
+                        new MarginRequirementsEntry
+                        {
+                            Date = DateTime.MinValue
+                        }
+                    };
                 }
-
-                MarginRequirementsEntry[] marginRequirementsEntries;
-                using (var streamReader = new StreamReader(stream))
-                {
-                    // skip the first header line, also skip #'s as these are comment lines
-                    marginRequirementsEntries = streamReader.ReadAllLines()
-                        .Where(x => !x.StartsWith("#") && !string.IsNullOrWhiteSpace(x))
-                        .Skip(1)
-                        .Select(FromCsvLine)
-                        .OrderBy(x => x.Date)
-                        .ToArray();
-                }
-                
-                stream.Dispose();
                 return marginRequirementsEntries;
             }
         }

Common/Securities/SecurityDefinitionSymbolResolver.cs

@@ -22,6 +22,7 @@ using QuantConnect.Logging;
 using QuantConnect.Interfaces;
 using QuantConnect.Configuration;
 using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
 
 namespace QuantConnect.Securities
 {
@@ -161,8 +162,7 @@ namespace QuantConnect.Securities
                 return null;
             }
 
-            var market = securityDefinition.SecurityIdentifier.Market;
-            var mapFileResolver = _mapFileProvider.Get(market);
+            var mapFileResolver = _mapFileProvider.Get(CorporateActionsKey.Create(securityDefinition.SecurityIdentifier));
             
             // Get the first ticker the symbol traded under, and then lookup the
             // trading date to get the ticker on the trading date.

Common/SecurityIdentifier.cs

@@ -21,6 +21,7 @@ using System.Numerics;
 using Newtonsoft.Json;
 using ProtoBuf;
 using QuantConnect.Configuration;
+using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Interfaces;
 using QuantConnect.Logging;
@@ -430,7 +431,7 @@ namespace QuantConnect
             var firstDate = DefaultDate;
             if (mapSymbol)
             {
-                var firstTickerDate = GetFirstTickerAndDate(mapFileProvider ?? MapFileProvider.Value, symbol, market, mappingResolveDate: mappingResolveDate);
+                var firstTickerDate = GetFirstTickerAndDate(mapFileProvider ?? MapFileProvider.Value, symbol, market, SecurityType.Equity, mappingResolveDate: mappingResolveDate);
                 firstDate = firstTickerDate.Item2;
                 symbol = firstTickerDate.Item1;
             }
@@ -497,7 +498,7 @@ namespace QuantConnect
 
             if (mapSymbol)
             {
-                var firstTickerDate = GetFirstTickerAndDate(MapFileProvider.Value, symbol, market);
+                var firstTickerDate = GetFirstTickerAndDate(MapFileProvider.Value, symbol, market, SecurityType.Equity);
                 firstDate = firstTickerDate.Item2;
                 symbol = firstTickerDate.Item1;
             }
@@ -636,11 +637,12 @@ namespace QuantConnect
         /// <param name="mapFileProvider">The IMapFileProvider instance used for resolving map files</param>
         /// <param name="tickerToday">The security's ticker as it trades today</param>
         /// <param name="market">The market the security exists in</param>
+        /// <param name="securityType">The securityType the security exists in</param>
         /// <param name="mappingResolveDate">The date to use to resolve the map file. Default value is <see cref="DateTime.Today"/></param>
         /// <returns>The security's first ticker/date if mapping data available, otherwise, the provided ticker and DefaultDate are returned</returns>
-        private static Tuple<string, DateTime> GetFirstTickerAndDate(IMapFileProvider mapFileProvider, string tickerToday, string market, DateTime? mappingResolveDate = null)
+        private static Tuple<string, DateTime> GetFirstTickerAndDate(IMapFileProvider mapFileProvider, string tickerToday, string market, SecurityType securityType, DateTime? mappingResolveDate = null)
         {
-            var resolver = mapFileProvider.Get(market);
+            var resolver = mapFileProvider.Get(new CorporateActionsKey(market, securityType));
             var mapFile = resolver.ResolveMapFile(tickerToday, mappingResolveDate ?? DateTime.Today);
 
             // if we have mapping data, use the first ticker/date from there, otherwise use provided ticker and DefaultDate

Common/Symbol.cs

@@ -17,6 +17,7 @@
 using System;
 using Newtonsoft.Json;
 using ProtoBuf;
+using QuantConnect.Securities.Future;
 using static QuantConnect.StringExtensions;
 
 namespace QuantConnect
@@ -330,7 +331,7 @@ namespace QuantConnect
         /// Creates new symbol with updated mapped symbol. Symbol Mapping: When symbols change over time (e.g. CHASE-> JPM) need to update the symbol requested.
         /// Method returns newly created symbol
         /// </summary>
-        public Symbol UpdateMappedSymbol(string mappedSymbol)
+        public Symbol UpdateMappedSymbol(string mappedSymbol, uint contractDepthOffset = 0)
         {
             // Throw for any option SecurityType that is not for equities, we don't support mapping for them (FOPs and Index Options)
             if (ID.SecurityType.IsOption() && SecurityType != SecurityType.Option)
@@ -338,6 +339,21 @@ namespace QuantConnect
                 throw new ArgumentException($"SecurityType {ID.SecurityType} can not be mapped.");
             }
 
+            if(ID.SecurityType == SecurityType.Future)
+            {
+                if (mappedSymbol == Value)
+                {
+                    // futures with no real continuous mapping
+                    return this;
+                }
+                var id = SecurityIdentifier.Parse(mappedSymbol);
+                var underlying = new Symbol(id, mappedSymbol);
+                underlying = underlying.AdjustSymbolByOffset(contractDepthOffset);
+
+                // we map the underlying
+                return new Symbol(ID, underlying.Value, underlying);
+            }
+
             // Avoid updating the current instance's underlying Symbol.
             var underlyingSymbol = Underlying;
 
@@ -348,14 +364,14 @@ namespace QuantConnect
             // This will ensure that we map all of the underlying Symbol(s) that also require mapping updates.
             if (HasUnderlying)
             {
-                underlyingSymbol = Underlying.UpdateMappedSymbol(mappedSymbol);
+                underlyingSymbol = Underlying.UpdateMappedSymbol(mappedSymbol, contractDepthOffset);
             }
 
             // If this Symbol is not a custom data type, and the security type does not support mapping,
             // then we know for a fact that this Symbol should not be mapped.
             // Custom data types should be mapped, especially if this method is called on them because
             // they can have an underlying that is also mapped.
-            if (SecurityType != SecurityType.Base && !SecurityType.RequiresMapping())
+            if (SecurityType != SecurityType.Base && !this.RequiresMapping())
             {
                 return new Symbol(ID, Value, underlyingSymbol);
             }

Common/Util/LeanData.cs

@@ -660,6 +660,10 @@ namespace QuantConnect.Util
                         ) + ".csv";
 
                 case SecurityType.Future:
+                    if (symbol.HasUnderlying)
+                    {
+                        symbol = symbol.Underlying;
+                    }
                     var expiryDate = symbol.ID.Date;
                     var monthsToAdd = FuturesExpiryUtilityFunctions.GetDeltaBetweenContractMonthAndContractExpiry(symbol.ID.Symbol, expiryDate.Date);
                     var contractYearMonth = expiryDate.AddMonths(monthsToAdd).ToStringInvariant(DateFormat.YearMonth);

Data/future/cme/factor_files/es.csv

@@ -0,0 +1,153 @@
+{"BackwardsRatioScale":[0.8802878334272878404490058911,0.8855690260287003529658011655,0.9118331468765744865680302487],"BackwardsPanamaCanalScale":[-227.50,-227.25,-164.75],"ForwardPanamaCanalScale":[0.0,0.0,0.0],"Date":"2009-11-30T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.8790348265234409892567098428,0.864884915130958502881701314,0.9056411497626623126204037878],"BackwardsPanamaCanalScale":[-229.75,-265.50,-159.00],"ForwardPanamaCanalScale":[0.0,0.0,0.0],"Date":"2009-12-12T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.8422997168493956725345341756,0.8843575984630898744265823781,0.8772497991839083159145713712],"BackwardsPanamaCanalScale":[-337.00,-218.00,-166.75],"ForwardPanamaCanalScale":[0.0,0.0,0.0],"Date":"2009-12-17T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.8837248782523369661466708199,0.8874033462207045654345450575,0.9118331468765744865680302487],"BackwardsPanamaCanalScale":[-223.25,-225.00,-164.75],"ForwardPanamaCanalScale":[5.0,5.5,0.0],"Date":"2010-02-28T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.8830177663808195166208091171,0.8690193524671183318278692465,0.9116593883839082354127941888],"BackwardsPanamaCanalScale":[-224.75,-260.25,-151.75],"ForwardPanamaCanalScale":[5.0,4.25,8.25],"Date":"2010-03-13T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.8459435453148188323200888544,0.8884031501486483825438128556,0.8804720206685140397563145167],"BackwardsPanamaCanalScale":[-332.25,-213.00,-162.75],"ForwardPanamaCanalScale":[5.25,4.75,6.5],"Date":"2010-03-18T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.8877272554183348690730597276,0.891846421691085880964713467,0.9118331468765744865680302487],"BackwardsPanamaCanalScale":[-218.25,-219.50,-164.75],"ForwardPanamaCanalScale":[8.75,7.5,0.0],"Date":"2010-05-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.8868686942760694927072496278,0.8722527359190642744063489526,0.9182918310242673561730724534],"BackwardsPanamaCanalScale":[-219.75,-256.00,-143.50],"ForwardPanamaCanalScale":[9.5,7.75,21.50],"Date":"2010-06-12T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.8497565554997536242236565441,0.8920410078755501754895741583,0.8854335014958615397636106107],"BackwardsPanamaCanalScale":[-327.00,-208.25,-156.25],"ForwardPanamaCanalScale":[10.25,11.50,10.5],"Date":"2010-06-17T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.8907869771166976960574567625,0.8934888644566311588302101769,0.9118331468765744865680302487],"BackwardsPanamaCanalScale":[-214.50,-217.50,-164.75],"ForwardPanamaCanalScale":[14.25,14.75,0.0],"Date":"2010-08-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.8905469515334356795433626447,0.8750755603071518610549131887,0.9296818303325486971225629379],"BackwardsPanamaCanalScale":[-215.25,-252.50,-130.25],"ForwardPanamaCanalScale":[14.25,12.50,21.50],"Date":"2010-09-11T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.8535713997444214294951432177,0.8974802823138157253401203422,0.8886445024079045553203236138],"BackwardsPanamaCanalScale":[-322.00,-201.50,-152.25],"ForwardPanamaCanalScale":[17.75,15.50,25.5],"Date":"2010-09-16T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.8954686095812383910677921779,0.8997220078459124442779070966,0.9118331468765744865680302487],"BackwardsPanamaCanalScale":[-209.00,-210.25,-164.75],"ForwardPanamaCanalScale":[19.75,10.50,0.0],"Date":"2010-11-30T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.8943759608294608612386158446,0.8788542956812054713640139502,0.9296818303325486971225629379],"BackwardsPanamaCanalScale":[-210.50,-247.75,-130.25],"ForwardPanamaCanalScale":[19.25,17.75,21.50],"Date":"2010-12-11T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.8592593477658191733611703827,0.9006812819832363788411105173,0.9006911969728558145429217649],"BackwardsPanamaCanalScale":[-314.50,-197.50,-137.25],"ForwardPanamaCanalScale":[22.50,20.25,25.5],"Date":"2010-12-16T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.8996726875604930314014437844,0.8964697700969478447557458335,0.9118331468765744865680302487],"BackwardsPanamaCanalScale":[-203.50,-214.50,-164.75],"ForwardPanamaCanalScale":[24.75,18.50,20.5],"Date":"2011-02-28T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.8979939865609716252403901806,0.882603217113117987085859226,0.9296818303325486971225629379],"BackwardsPanamaCanalScale":[-205.50,-242.50,-130.25],"ForwardPanamaCanalScale":[24.25,25.75,41.00],"Date":"2011-03-12T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.8625581813902989802502110449,0.904152467451325930808246643,0.9006911969728558145429217649],"BackwardsPanamaCanalScale":[-309.75,-192.75,-137.25],"ForwardPanamaCanalScale":[29.00,27.25,28.0],"Date":"2011-03-17T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9030728110962469733190607299,0.9019235785842296567390507511,0.9262731464987991115001620526],"BackwardsPanamaCanalScale":[-198.50,-206.50,-144.25],"ForwardPanamaCanalScale":[30.00,26.25,20.5],"Date":"2011-05-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9014438289987778934894888751,0.8880619150156343720079209761,0.9439116542662101567723980849],"BackwardsPanamaCanalScale":[-200.50,-234.50,-110.75],"ForwardPanamaCanalScale":[29.50,32.00,41.00],"Date":"2011-06-11T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.8669230145386344752845640008,0.909106727546949634538702789,0.9024572581433908259439863173],"BackwardsPanamaCanalScale":[-303.25,-185.75,-134.75],"ForwardPanamaCanalScale":[35.00,32.00,51.5],"Date":"2011-06-16T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9066096497820025097680909938,0.9071683039520989344278353268,0.9262731464987991115001620526],"BackwardsPanamaCanalScale":[-193.25,-198.75,-144.25],"ForwardPanamaCanalScale":[34.50,31.75,32.0],"Date":"2011-08-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9051864783481203877091961242,0.8924731106992010040961780841,0.9439116542662101567723980849],"BackwardsPanamaCanalScale":[-195.25,-228.25,-110.75],"ForwardPanamaCanalScale":[35.25,40.75,47.75],"Date":"2011-09-10T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.8710034523399537981727439451,0.912506929874389013093079827,0.9194710933350231439621842142],"BackwardsPanamaCanalScale":[-297.25,-181.00,-111.25],"ForwardPanamaCanalScale":[42.25,35.25,70.5],"Date":"2011-09-15T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9099743866265542716600138058,0.9113020451880032791994617513,0.9351833440602681117362572961],"BackwardsPanamaCanalScale":[-188.75,-193.25,-132.75],"ForwardPanamaCanalScale":[40.75,43.50,32.0],"Date":"2011-11-30T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9097025930990979948496107686,0.899300773841162104673963053,0.9495153514221441220830521632],"BackwardsPanamaCanalScale":[-189.50,-219.50,-104.00],"ForwardPanamaCanalScale":[41.25,46.75,55.00],"Date":"2011-12-10T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.8762341626563461485862891044,0.9149700922681529369241035059,0.9342136678273146542873162818],"BackwardsPanamaCanalScale":[-290.00,-177.75,-92.25],"ForwardPanamaCanalScale":[48.75,39.00,70.5],"Date":"2011-12-15T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9145720745137134989824673921,0.9200413220955397317851329821,0.9351833440602681117362572961],"BackwardsPanamaCanalScale":[-182.50,-181.50,-132.75],"ForwardPanamaCanalScale":[46.50,45.00,37.25],"Date":"2012-02-29T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.914056105010240238493955865,0.9036252015254313674493938511,0.9550647156283597971214255031],"BackwardsPanamaCanalScale":[-183.50,-213.50,-96.75],"ForwardPanamaCanalScale":[47.00,54.25,60.75],"Date":"2012-03-11T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.8808987998858844368465764748,0.9177888541050850162122245888,0.9342136678273146542873162818],"BackwardsPanamaCanalScale":[-283.50,-174.00,-92.25],"ForwardPanamaCanalScale":[55.25,44.50,83.00],"Date":"2012-03-15T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9184495229402232520657110326,0.9210600059731136824016140299,0.9388215156470160980916595568],"BackwardsPanamaCanalScale":[-176.75,-180.00,-127.50],"ForwardPanamaCanalScale":[53.75,45.00,37.25],"Date":"2012-05-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9179030017010427134226586985,0.908613013125387750342398785,0.9591235641981033808859844067],"BackwardsPanamaCanalScale":[-177.75,-206.00,-91.00],"ForwardPanamaCanalScale":[53.75,63.00,64.50],"Date":"2012-06-10T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.8849908793564995869909414897,0.9214106217804772799533311253,0.9426680901605935199369752526],"BackwardsPanamaCanalScale":[-277.00,-168.50,-79.75],"ForwardPanamaCanalScale":[62.75,52.25,87.50],"Date":"2012-06-14T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9235706604455619319945527207,0.9210600059731136824016140299,0.9388215156470160980916595568],"BackwardsPanamaCanalScale":[-169.50,-180.00,-127.50],"ForwardPanamaCanalScale":[60.75,50.75,45.00],"Date":"2012-08-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9225354093731788168361936116,0.9145963537574458070238633114,0.9618380648514942395111334192],"BackwardsPanamaCanalScale":[-171.00,-197.25,-87.25],"ForwardPanamaCanalScale":[60.25,70.00,64.50],"Date":"2012-09-16T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.8899880045928434581766610915,0.9268183706019832995819856624,0.9458914992834830874443585266],"BackwardsPanamaCanalScale":[-269.50,-160.75,-75.25],"ForwardPanamaCanalScale":[70.00,58.25,102.50],"Date":"2012-09-20T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9281959472444307808759420028,0.9248646719460629666500402373,0.9440776953445160979184324386],"BackwardsPanamaCanalScale":[-162.50,-174.25,-119.75],"ForwardPanamaCanalScale":[67.75,58.50,51.00],"Date":"2012-11-30T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9266545782970501287437569009,0.9190154560706828186885170081,0.9618380648514942395111334192],"BackwardsPanamaCanalScale":[-164.50,-190.25,-87.25],"ForwardPanamaCanalScale":[66.50,76.25,71.25],"Date":"2012-12-16T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.8944044406183726266036157752,0.9306403020271461173122206548,0.9557445357343527029385705946],"BackwardsPanamaCanalScale":[-262.25,-154.75,-60.25],"ForwardPanamaCanalScale":[74.75,60.75,111.75],"Date":"2012-12-20T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.932811377278677639669145053,0.92998445852290724378685296,0.9481411574478640868621272698],"BackwardsPanamaCanalScale":[-155.50,-166.50,-113.75],"ForwardPanamaCanalScale":[73.50,64.00,48.25],"Date":"2013-02-28T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9307446850162284926594691047,0.9230898285483857927029190165,0.9664655822697034954745690219],"BackwardsPanamaCanalScale":[-158.25,-184.00,-80.50],"ForwardPanamaCanalScale":[71.50,82.00,73.50],"Date":"2013-03-10T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.8973994502009861408625986268,0.9322833816564313329148199144,0.9620289764349340162509443483],"BackwardsPanamaCanalScale":[-257.50,-152.25,-51.00],"ForwardPanamaCanalScale":[81.75,67.00,116.75],"Date":"2013-03-14T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9363800169455787683638207576,0.9334000942637259348324808006,0.9464031883207308582409835605],"BackwardsPanamaCanalScale":[-149.75,-161.00,-116.50],"ForwardPanamaCanalScale":[79.00,69.00,54.25],"Date":"2013-05-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9337602193917873084464753225,0.9265420344112496745292876567,0.9678819949452331048963685857],"BackwardsPanamaCanalScale":[-153.25,-178.25,-78.25],"ForwardPanamaCanalScale":[77.75,89.25,79.25],"Date":"2013-06-16T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.901441139859282362825627863,0.9360474456973568470173506796,0.9651463742846907693820232347],"BackwardsPanamaCanalScale":[-250.50,-146.00,-46.00],"ForwardPanamaCanalScale":[90.00,72.25,123.00],"Date":"2013-06-20T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9395556544317515343425480671,0.9362867498822932022975572837,0.94992851449172225054236236],"BackwardsPanamaCanalScale":[-144.25,-156.00,-110.50],"ForwardPanamaCanalScale":[85.75,75.75,60.75],"Date":"2013-08-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9373582473641662669407974376,0.930702077606469714870106567,0.9713408647220728130089566027],"BackwardsPanamaCanalScale":[-147.00,-171.00,-72.50],"ForwardPanamaCanalScale":[84.25,94.50,84.00],"Date":"2013-09-15T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9061110861097307643912865317,0.9391435232465212026304179916,0.9689617853152279719807820235],"BackwardsPanamaCanalScale":[-242.25,-140.75,-39.75],"ForwardPanamaCanalScale":[97.75,78.50,130.00],"Date":"2013-09-19T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9434572235904192198896487742,0.9401909571790996758824274067,0.9537582774591256896048683953],"BackwardsPanamaCanalScale":[-137.50,-149.25,-104.00],"ForwardPanamaCanalScale":[92.25,81.25,60.75],"Date":"2013-11-30T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9409422641923233732555710749,0.9335852083971078420293147449,0.9740709647856407691401947944],"BackwardsPanamaCanalScale":[-140.50,-165.75,-67.75],"ForwardPanamaCanalScale":[90.25,103.75,89.50],"Date":"2013-12-15T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9101962291681079682447085623,0.9425705540567916784829592921,0.9729381783958122286443408534],"BackwardsPanamaCanalScale":[-234.50,-134.50,-32.75],"ForwardPanamaCanalScale":[104.50,85.50,136.00],"Date":"2013-12-19T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9468561272641549687766005364,0.9430657592719336151276551208,0.9537582774591256896048683953],"BackwardsPanamaCanalScale":[-131.00,-143.75,-104.00],"ForwardPanamaCanalScale":[99.50,100.25,66.25],"Date":"2014-02-28T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9441323495632254443048907113,0.9384904416704240860161202895,0.9771236088348265379323720431],"BackwardsPanamaCanalScale":[-134.50,-156.50,-62.25],"ForwardPanamaCanalScale":[97.50,113.00,100.50],"Date":"2014-03-16T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9136018968770241150216773997,0.9462422256641358864681461118,0.9761976027790477804990957641],"BackwardsPanamaCanalScale":[-227.75,-127.50,-26.75],"ForwardPanamaCanalScale":[114.00,92.50,144.50],"Date":"2014-03-20T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9505607721225819107753217204,0.9528357971269863625035686689,0.9566271155920352828741201187],"BackwardsPanamaCanalScale":[-123.75,-124.75,-98.50],"ForwardPanamaCanalScale":[107.00,105.50,73.50],"Date":"2014-05-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9478722536934444769349742833,0.9432576226618856032020890499,0.9830619291095133290468781715],"BackwardsPanamaCanalScale":[-127.25,-147.25,-51.25],"ForwardPanamaCanalScale":[104.50,121.00,105.00],"Date":"2014-06-15T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9182376122036801326970444368,0.9497933290441916169092542865,0.9806665294647200067730918939],"BackwardsPanamaCanalScale":[-218.25,-120.50,-18.25],"ForwardPanamaCanalScale":[126.50,99.25,160.75],"Date":"2014-06-19T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9542899238374606157999404796,0.955459656521120471290950644,0.9602788447419851345528550777],"BackwardsPanamaCanalScale":[-116.25,-119.50,-91.25],"ForwardPanamaCanalScale":[115.50,118.50,72.00],"Date":"2014-08-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9513226467126093814271972786,0.9471981244797942636854398135,0.9853774374481643680318093161],"BackwardsPanamaCanalScale":[-120.25,-139.25,-46.75],"ForwardPanamaCanalScale":[112.75,127.75,116.25],"Date":"2014-09-14T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9241071955084850427756499369,0.9530831840646140894758418254,0.9889127551329226976450519163],"BackwardsPanamaCanalScale":[-205.75,-113.75,-2.00],"ForwardPanamaCanalScale":[135.00,106.25,170.50],"Date":"2014-09-18T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.958362966444851233542496088,0.9617376325306097642536912851,0.9595513607686957518751635208],"BackwardsPanamaCanalScale":[-107.75,-106.50,-92.75],"ForwardPanamaCanalScale":[123.75,124.00,80.50],"Date":"2014-11-30T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9553081294270331607324019898,0.9504560033792789063758228549,0.9910586654667636501024603628],"BackwardsPanamaCanalScale":[-112.00,-132.50,-35.50],"ForwardPanamaCanalScale":[119.50,135.00,104.50],"Date":"2014-12-14T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9280136538756724717765665225,0.9564127323058878504871111418,0.9937482111823439595818820347],"BackwardsPanamaCanalScale":[-197.25,-106.75,7.75],"ForwardPanamaCanalScale":[140.00,113.25,178.25],"Date":"2014-12-18T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9622197930171292933848012552,0.9643248455308143882220920198,0.9635573463121799924895735846],"BackwardsPanamaCanalScale":[-99.50,-101.00,-84.25],"ForwardPanamaCanalScale":[131.75,132.00,86.25],"Date":"2015-02-28T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9585476822397096107876362091,0.9539305053454958593929536834,0.9852216031513578992810548945],"BackwardsPanamaCanalScale":[-105.25,-125.25,-47.25],"ForwardPanamaCanalScale":[127.25,139.50,110.00],"Date":"2015-03-15T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9302590292782267240673397297,0.9596634674970222107461107912,0.9975018598416168562246307057],"BackwardsPanamaCanalScale":[-192.25,-99.75,15.50],"ForwardPanamaCanalScale":[148.00,119.50,198.00],"Date":"2015-03-19T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9658901987122715197989523626,0.9680173661790118347247971705,0.9662165518515584589915246516],"BackwardsPanamaCanalScale":[-91.50,-93.00,-78.50],"ForwardPanamaCanalScale":[138.75,139.50,92.75],"Date":"2015-05-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9621910096243872711523268322,0.9560404523959036899986397446,0.9878922383496489137936452355],"BackwardsPanamaCanalScale":[-97.50,-120.75,-41.75],"ForwardPanamaCanalScale":[135.75,147.00,117.75],"Date":"2015-06-14T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9338143008579356999624973639,0.9625373997786975288185416453,1.0070317482111280101297880192],"BackwardsPanamaCanalScale":[-184.25,-93.50,35.25],"ForwardPanamaCanalScale":[155.25,127.50,198.25],"Date":"2015-06-18T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9691098327079790915316155372,0.9714869624735602642399398127,0.9692272937437133631063112816],"BackwardsPanamaCanalScale":[-84.50,-85.50,-72.00],"ForwardPanamaCanalScale":[147.00,138.50,98.25],"Date":"2015-08-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9661291936050336136609555926,0.959505622574939223489029885,0.9916061189449483458003882627],"BackwardsPanamaCanalScale":[-89.00,-113.25,-34.00],"ForwardPanamaCanalScale":[145.50,156.75,123.25],"Date":"2015-09-13T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9370210047922021458238250636,0.9661985428659337490636614994,1.0071514619234688479816817172],"BackwardsPanamaCanalScale":[-177.00,-85.50,35.50],"ForwardPanamaCanalScale":[166.00,136.25,206.50],"Date":"2015-09-17T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9732167576443298440705321049,0.9709881885228614783258132873,0.9719719198812293489459738956],"BackwardsPanamaCanalScale":[-76.25,-86.50,-66.50],"ForwardPanamaCanalScale":[154.00,145.75,110.25],"Date":"2015-11-30T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9709247947951311459166228025,0.9642921049592347760237027125,0.994404391728015157448362678],"BackwardsPanamaCanalScale":[-79.25,-103.50,-28.50],"ForwardPanamaCanalScale":[154.25,163.75,123.00],"Date":"2015-12-13T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9421886209530415317761688468,0.9705552742496373926113590197,1.0114516298939015652780196001],"BackwardsPanamaCanalScale":[-166.25,-76.75,43.75],"ForwardPanamaCanalScale":[176.50,143.25,210.00],"Date":"2015-12-17T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9764916127355497978782445071,0.9743840709959877425408939175,0.9776311450333762447185850973],"BackwardsPanamaCanalScale":[-69.25,-79.25,-54.50],"ForwardPanamaCanalScale":[163.25,151.50,121.50],"Date":"2016-02-29T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9751651701155491132371970224,0.967673043969860693094659847,0.994279888861891704943361005],"BackwardsPanamaCanalScale":[-70.50,-96.50,-28.75],"ForwardPanamaCanalScale":[163.50,170.00,132.75],"Date":"2016-03-13T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9470837127491854882747585414,0.973928604398469300132848351,1.0132124260663107448246363962],"BackwardsPanamaCanalScale":[-155.75,-69.75,47.25],"ForwardPanamaCanalScale":[188.00,150.50,218.50],"Date":"2016-03-17T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9811905579392453815069992277,0.9773074774182298137747108442,0.9834037997409608046204793801],"BackwardsPanamaCanalScale":[-60.00,-73.50,-43.25],"ForwardPanamaCanalScale":[172.00,158.25,127.50],"Date":"2016-05-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9796539871464368113200256228,0.9706921162398278111936697679,0.9991428078066639367583586322],"BackwardsPanamaCanalScale":[-61.25,-90.25,-19.00],"ForwardPanamaCanalScale":[172.75,175.50,134.50],"Date":"2016-06-12T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9524318407612727129814414401,0.9774082064099951852756169357,1.0174743612990528659831218146],"BackwardsPanamaCanalScale":[-144.25,-62.50,55.75],"ForwardPanamaCanalScale":[198.50,159.00,218.00],"Date":"2016-06-16T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9853053038687773235574119971,0.9804794093680185149373406306,0.9862490910241172915777899895],"BackwardsPanamaCanalScale":[-51.25,-66.75,-37.25],"ForwardPanamaCanalScale":[179.25,164.25,133.50],"Date":"2016-08-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9840059138512534954380278977,0.9732628669991002575456934509,0.9999854583674647376688777299],"BackwardsPanamaCanalScale":[-52.00,-84.75,-17.25],"ForwardPanamaCanalScale":[179.50,181.25,127.75],"Date":"2016-09-11T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9572676890823526759825561089,0.9814421805563826380917581078,1.0172274015026210570350676588],"BackwardsPanamaCanalScale":[-133.75,-54.00,55.25],"ForwardPanamaCanalScale":[209.00,165.25,211.00],"Date":"2016-09-15T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9886094035232864057016262603,0.9832080162586530748119343429,0.9890014140688357584380070778],"BackwardsPanamaCanalScale":[-44.00,-60.75,-31.25],"ForwardPanamaCanalScale":[184.25,171.50,130.75],"Date":"2016-11-30T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9871493307554162787291810867,0.9759185848708500862233006344,0.9967925057449574590599539245],"BackwardsPanamaCanalScale":[-45.25,-79.00,-24.00],"ForwardPanamaCanalScale":[185.25,184.75,148.75],"Date":"2016-12-11T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9619838466121425659991270598,0.9843287752050778811449691611,1.0138875366759238959660031777],"BackwardsPanamaCanalScale":[-123.25,-47.75,48.25],"ForwardPanamaCanalScale":[214.00,170.25,215.25],"Date":"2016-12-15T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9908618742535741395947719114,0.9864670209987949126413486869,0.9877595173160095948143977082],"BackwardsPanamaCanalScale":[-39.00,-53.50,-34.00],"ForwardPanamaCanalScale":[188.25,179.50,130.00],"Date":"2017-02-28T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9896614390323995244041944929,0.9774326429590805717294094785,1.0061583413694067237759534916],"BackwardsPanamaCanalScale":[-39.50,-75.50,-3.00],"ForwardPanamaCanalScale":[188.00,187.00,152.75],"Date":"2017-03-12T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9641097777648765827195671196,0.9865089053827192496524884727,1.0157998952827685321838465211],"BackwardsPanamaCanalScale":[-118.25,-42.75,52.50],"ForwardPanamaCanalScale":[218.25,173.75,221.50],"Date":"2017-03-16T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9925407676124703787384250249,0.9898212814038487259666619542,0.9874447437285921284043230117],"BackwardsPanamaCanalScale":[-35.00,-45.50,-34.75],"ForwardPanamaCanalScale":[191.00,180.25,135.75],"Date":"2017-05-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9908118433258959203429502933,0.9783631373304188121721389777,1.0078640558705627428033337899],"BackwardsPanamaCanalScale":[-36.75,-73.25,1.00],"ForwardPanamaCanalScale":[190.25,191.00,159.25],"Date":"2017-06-11T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9658312215441017588466790449,0.9879616375458998607280259544,1.0184781274470871583496972027],"BackwardsPanamaCanalScale":[-114.00,-39.25,58.75],"ForwardPanamaCanalScale":[220.00,175.75,220.50],"Date":"2017-06-15T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9936735698996396227896932004,0.9901294769247840187718409875,0.989807543261026337617109053],"BackwardsPanamaCanalScale":[-32.25,-44.75,-29.00],"ForwardPanamaCanalScale":[192.75,182.50,141.00],"Date":"2017-08-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9917309637742501837198918056,0.9799792647511587152275544457,1.0105767334536512139537361376],"BackwardsPanamaCanalScale":[-34.50,-69.25,7.50],"ForwardPanamaCanalScale":[192.25,190.50,166.75],"Date":"2017-09-10T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9665262070036832465432175681,0.988774774696143399708049021,1.0180591730548835025873155913],"BackwardsPanamaCanalScale":[-112.25,-37.25,57.75],"ForwardPanamaCanalScale":[224.25,176.75,221.25],"Date":"2017-09-14T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9943773759291142233522276386,0.9910319648012599207537071581,0.9919171503257255716666329838],"BackwardsPanamaCanalScale":[-30.50,-42.50,-23.75],"ForwardPanamaCanalScale":[190.50,183.50,138.25],"Date":"2017-11-30T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9925346355439213751167474716,0.9797807678370912332111016293,1.0136565081391052505667605265],"BackwardsPanamaCanalScale":[-32.50,-69.75,15.00],"ForwardPanamaCanalScale":[189.25,189.25,166.25],"Date":"2017-12-10T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9681747417201538610903465637,0.9891717534273272372668999892,1.0183658173841169012326732165],"BackwardsPanamaCanalScale":[-108.00,-36.25,58.50],"ForwardPanamaCanalScale":[221.75,175.75,222.50],"Date":"2017-12-14T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9935311773300822615509634509,0.9914069296006550247721535097,0.99088614884660027814982934],"BackwardsPanamaCanalScale":[-32.75,-41.50,-26.50],"ForwardPanamaCanalScale":[185.50,179.25,132.50],"Date":"2018-02-28T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9914136552496600050930862039,0.9793199114081932599123193614,1.013465827832833786072673476],"BackwardsPanamaCanalScale":[-35.50,-71.00,14.50],"ForwardPanamaCanalScale":[184.25,182.75,160.00],"Date":"2018-03-12T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9672662545833234642984817363,0.9888006168534871369996579294,1.0188436541978294185474407997],"BackwardsPanamaCanalScale":[-110.50,-37.25,59.75],"ForwardPanamaCanalScale":[218.25,169.00,214.50],"Date":"2018-03-15T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9917076824648347947957997114,0.9898626820467599234874616818,0.9888023615718931549535738701],"BackwardsPanamaCanalScale":[-37.75,-45.75,-32.25],"ForwardPanamaCanalScale":[181.50,176.50,129.25],"Date":"2018-05-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.989635494268134294385359388,0.9770418119453129678162542005,1.0112040312165312646790101269],"BackwardsPanamaCanalScale":[-40.50,-77.50,8.25],"ForwardPanamaCanalScale":[180.50,183.50,148.50],"Date":"2018-06-11T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9660379799743287678358804896,0.9863849801673629827561734231,1.0159022073434148983928469612],"BackwardsPanamaCanalScale":[-114.00,-44.00,51.75],"ForwardPanamaCanalScale":[216.00,166.00,198.00],"Date":"2018-06-14T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9902452563893106144532436934,0.9888601510180780010900432048,0.9876202362545826334330297777],"BackwardsPanamaCanalScale":[-41.75,-48.50,-35.50],"ForwardPanamaCanalScale":[177.25,169.75,133.50],"Date":"2018-08-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9883049813691451383025611097,0.9773045988769819600077055868,1.0070508718026062255562213353],"BackwardsPanamaCanalScale":[-44.25,-76.75,-3.25],"ForwardPanamaCanalScale":[175.00,175.25,135.75],"Date":"2018-09-16T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9652557629864953032465396957,0.985321205586432960666385153,1.009911896820350002181471289],"BackwardsPanamaCanalScale":[-116.25,-47.00,35.25],"ForwardPanamaCanalScale":[214.00,156.75,194.25],"Date":"2018-09-20T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9887996428763335186657207099,0.9865726917927962922115792598,0.9890607805717278848074065693],"BackwardsPanamaCanalScale":[-46.00,-55.25,-31.25],"ForwardPanamaCanalScale":[172.75,162.25,159.50],"Date":"2018-11-30T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9864343189248432202914919929,0.9745376932693172032575121856,1.0026637876254529909527048675],"BackwardsPanamaCanalScale":[-49.75,-85.00,-16.00],"ForwardPanamaCanalScale":[171.50,175.75,135.75],"Date":"2018-12-16T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9646002412459272249421277842,0.9822360837613760071634100958,1.0086315825495331284464542108],"BackwardsPanamaCanalScale":[-118.25,-56.25,31.50],"ForwardPanamaCanalScale":[212.50,154.25,197.25],"Date":"2018-12-20T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9871924193282231698002716425,0.9839072396355751206728231919,0.9984119006789514939001311405],"BackwardsPanamaCanalScale":[-50.50,-62.75,-5.25],"ForwardPanamaCanalScale":[167.00,157.25,139.25],"Date":"2019-02-28T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9851107536128873256121666251,0.9747245294835176091630446841,1.0026637876254529909527048675],"BackwardsPanamaCanalScale":[-53.25,-84.50,-16.00],"ForwardPanamaCanalScale":[166.75,169.75,133.75],"Date":"2019-03-10T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9640172250831916992697541802,0.9812476203581744614180509229,1.009851086729945662203769823],"BackwardsPanamaCanalScale":[-119.75,-58.75,34.50],"ForwardPanamaCanalScale":[206.50,149.75,198.75],"Date":"2019-03-14T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9851586119433656695400238597,0.9821477488636946215299891018,0.9912329069856541063274803973],"BackwardsPanamaCanalScale":[-56.25,-67.75,-25.50],"ForwardPanamaCanalScale":[163.50,157.75,117.75],"Date":"2019-05-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.983408270047429361341961862,0.9726013321058080046413607966,1.0019362971210993156463724134],"BackwardsPanamaCanalScale":[-58.00,-90.50,-18.00],"ForwardPanamaCanalScale":[162.75,173.50,139.00],"Date":"2019-06-16T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9619655789537249438576344586,0.9796838791305120319974643876,1.0103878133240088807327115024],"BackwardsPanamaCanalScale":[-125.75,-63.25,36.00],"ForwardPanamaCanalScale":[202.75,151.50,190.00],"Date":"2019-06-20T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9839075034013476071875601139,0.9823259647498974331107421392,0.9835586154839646705024746974],"BackwardsPanamaCanalScale":[-59.75,-67.25,-47.00],"ForwardPanamaCanalScale":[163.25,156.25,156.50],"Date":"2019-08-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.982051610940621388434437169,0.9738608520904050106043773218,1.0037561087661039070344171998],"BackwardsPanamaCanalScale":[-62.00,-86.75,-12.75],"ForwardPanamaCanalScale":[160.75,165.25,121.50],"Date":"2019-09-15T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9607449137236846871824069384,0.9802643588286594489793487833,1.0074032936627429248861721402],"BackwardsPanamaCanalScale":[-129.50,-61.50,27.25],"ForwardPanamaCanalScale":[201.75,149.50,195.25],"Date":"2019-09-19T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9838233799827941403948799189,0.9818222939424714046883643873,0.9967613162200878647458178669],"BackwardsPanamaCanalScale":[-60.00,-68.75,-8.25],"ForwardPanamaCanalScale":[161.25,156.00,156.50],"Date":"2019-11-30T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9813951044738774282850701182,0.9711827347471563968252152842,0.9979348223026716557094454348],"BackwardsPanamaCanalScale":[-64.00,-95.00,-30.25],"ForwardPanamaCanalScale":[157.75,165.75,124.50],"Date":"2019-12-15T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9604262322695927443389929217,0.979614477758747351888784866,1.0091595172362874709116253707],"BackwardsPanamaCanalScale":[-130.50,-63.50,32.50],"ForwardPanamaCanalScale":[197.25,148.75,201.00],"Date":"2019-12-19T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9831981844120687171584704313,0.9817443096458913831312008953,0.9967613162200878647458178669],"BackwardsPanamaCanalScale":[-62.00,-69.00,-8.25],"ForwardPanamaCanalScale":[165.50,151.00,203.50],"Date":"2020-02-29T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9804688231105231315146264466,0.9713355325342468871330858341,0.9988777528434300856203519974],"BackwardsPanamaCanalScale":[-67.00,-94.50,-27.25],"ForwardPanamaCanalScale":[169.25,185.00,87.50],"Date":"2020-03-16T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9590814114076049832033957321,0.979385916284701913869212121,1.0109679059309843631462157395],"BackwardsPanamaCanalScale":[-135.00,-64.25,38.25],"ForwardPanamaCanalScale":[221.75,171.00,186.75],"Date":"2020-03-19T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9845986342767528535800618848,0.9801019101316364414613870921,1.0126864566509008602903248575],"BackwardsPanamaCanalScale":[-57.75,-74.00,38.75],"ForwardPanamaCanalScale":[175.50,162.50,167.50],"Date":"2020-05-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9851046095082088200797074227,0.9790844794887782806587020389,0.9837926602494680802456772938],"BackwardsPanamaCanalScale":[-55.50,-75.25,-64.25],"ForwardPanamaCanalScale":[180.00,192.50,87.50],"Date":"2020-06-15T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9687891141878100326099163664,0.9884722653464488616784855369,1.0049963856576096368561292838],"BackwardsPanamaCanalScale":[-110.50,-42.00,24.00],"ForwardPanamaCanalScale":[276.50,180.75,206.50],"Date":"2020-06-18T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9878637599085776018923817717,0.9838539647143640776387525886,1.0006941170326665079974131158],"BackwardsPanamaCanalScale":[-47.75,-62.50,2.75],"ForwardPanamaCanalScale":[185.75,171.50,167.50],"Date":"2020-08-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9885350839339338750897031645,0.9814690008708746719900676065,0.9837926602494680802456772938],"BackwardsPanamaCanalScale":[-44.75,-67.75,-64.25],"ForwardPanamaCanalScale":[190.50,226.50,64.25],"Date":"2020-09-13T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.985674495485631791236384467,0.9915498973397836457759995826,1.0113750081744370781119848485],"BackwardsPanamaCanalScale":[-55.75,-32.25,43.75],"ForwardPanamaCanalScale":[289.50,186.25,206.50],"Date":"2020-09-17T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9907681997907039325971209365,0.9864004269136391590101664281,1.0006941170326665079974131158],"BackwardsPanamaCanalScale":[-37.50,-53.50,2.75],"ForwardPanamaCanalScale":[194.00,194.00,167.50],"Date":"2020-11-30T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9916325528630017072855974896,0.9915292800477241561165862374,0.9769449155145529891998837164],"BackwardsPanamaCanalScale":[-34.25,-33.75,-87.50],"ForwardPanamaCanalScale":[198.00,238.25,113.75],"Date":"2020-12-13T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9894940851544038944590010549,0.993178179622494345201771919,1.0113750081744370781119848485],"BackwardsPanamaCanalScale":[-42.75,-26.75,43.75],"ForwardPanamaCanalScale":[298.00,197.75,211.50],"Date":"2020-12-17T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9930193255544806628212812912,0.9925509076135302165348245255,1.0006941170326665079974131158],"BackwardsPanamaCanalScale":[-29.25,-31.00,2.75],"ForwardPanamaCanalScale":[202.75,203.00,165.50],"Date":"2021-02-28T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9936583599168587077090614273,0.9947129032313724028263471032,0.990340697597093258719383507],"BackwardsPanamaCanalScale":[-26.75,-22.00,-38.00],"ForwardPanamaCanalScale":[206.75,247.00,151.00],"Date":"2021-03-14T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.99175898342391040667198113,0.9962634170794523908446191305,1.0127428402114595788938512061],"BackwardsPanamaCanalScale":[-34.25,-15.25,48.75],"ForwardPanamaCanalScale":[312.25,205.50,149.00],"Date":"2021-03-18T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9953064850960973266274635,0.9949078886606335864421631589,1.0001663247768475910628153241],"BackwardsPanamaCanalScale":[-20.5,-22.00,0.75],"ForwardPanamaCanalScale":[212.75,216.25,165.50],"Date":"2021-05-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9958690160697375568553757643,0.9969308398825221788408035056,0.9998313564562370003935016021],"BackwardsPanamaCanalScale":[-18.00,-13.25,-0.75],"ForwardPanamaCanalScale":[215.75,255.50,151.00],"Date":"2021-06-13T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9953806287602652592045138307,0.9982459672097298991329862632,0.9967467756390015996972715497],"BackwardsPanamaCanalScale":[-20.00,-7.50,-13.75],"ForwardPanamaCanalScale":[322.00,210.25,160.00],"Date":"2021-06-17T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[0.9976813514408744617422987744,0.9980633023461708720672864099,1.0001663247768475910628153241],"BackwardsPanamaCanalScale":[-10.5,-8.75,0.75],"ForwardPanamaCanalScale":[223.25,225.00,164.75],"Date":"2021-08-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[0.9979825151311365164761264291,0.9989330637915543575920934412,0.9998313564562370003935016021],"BackwardsPanamaCanalScale":[-9.0,-4.75,-0.75],"ForwardPanamaCanalScale":[224.75,260.25,151.75],"Date":"2021-09-12T00:00:00","DataMappingMode":2}
+{"BackwardsRatioScale":[0.9977047528410681296534736606,0.9993827853215127370665469644,0.9993824041322778058503172197],"BackwardsPanamaCanalScale":[-10.25,-2.75,-2.75],"ForwardPanamaCanalScale":[332.25,213.00,162.75],"Date":"2021-09-16T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[1.0,1.0,1.0],"BackwardsPanamaCanalScale":[0.0,0.0,0.0],"ForwardPanamaCanalScale":[223.25,225.00,164.75],"Date":"2021-11-30T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[1.0,1.0,1.0],"BackwardsPanamaCanalScale":[0.0,0.0,0.0],"ForwardPanamaCanalScale":[332.25,213.00,162.75],"Date":"2021-12-16T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[1.0,1.0,1.0],"BackwardsPanamaCanalScale":[0.0,0.0,0.0],"ForwardPanamaCanalScale":[223.25,225.00,164.75],"Date":"2022-02-28T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[1.0,1.0,1.0],"BackwardsPanamaCanalScale":[0.0,0.0,0.0],"ForwardPanamaCanalScale":[332.25,213.00,162.75],"Date":"2022-03-17T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[1.0,1.0,1.0],"BackwardsPanamaCanalScale":[0.0,0.0,0.0],"ForwardPanamaCanalScale":[223.25,225.00,164.75],"Date":"2022-05-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[1.0,1.0,1.0],"BackwardsPanamaCanalScale":[0.0,0.0,0.0],"ForwardPanamaCanalScale":[332.25,213.00,162.75],"Date":"2022-06-16T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[1.0,1.0,1.0],"BackwardsPanamaCanalScale":[0.0,0.0,0.0],"ForwardPanamaCanalScale":[223.25,225.00,164.75],"Date":"2022-08-31T00:00:00","DataMappingMode":1}
+{"BackwardsRatioScale":[1.0,1.0,1.0],"BackwardsPanamaCanalScale":[0.0,0.0,0.0],"ForwardPanamaCanalScale":[332.25,213.00,162.75],"Date":"2022-09-15T13:30:00","DataMappingMode":0}
+{"BackwardsRatioScale":[1.0,1.0,1.0],"BackwardsPanamaCanalScale":[0.0,0.0,0.0],"ForwardPanamaCanalScale":[224.75,260.25,151.75],"Date":"2050-12-31T00:00:00","DataMappingMode":2}

Data/future/cme/map_files/es.csv

@@ -0,0 +1,154 @@
+18991230,es,CME
+20091130,es uik2f7cj4v0h,CME,1
+20091212,es uik2f7cj4v0h,CME,2
+20091217,es uik2f7cj4v0h,CME,0
+20100228,es ul1o3pmw3im9,CME,1
+20100313,es ul1o3pmw3im9,CME,2
+20100318,es ul1o3pmw3im9,CME,0
+20100531,es unj9s7x92681,CME,1
+20100612,es unj9s7x92681,CME,2
+20100617,es unj9s7x92681,CME,0
+20100831,es uq0vgq7m0ttt,CME,1
+20100911,es uq0vgq7m0ttt,CME,2
+20100916,es uq0vgq7m0ttt,CME,0
+20101130,es usih58hyzhfl,CME,1
+20101211,es usih58hyzhfl,CME,2
+20101216,es usih58hyzhfl,CME,0
+20110228,es uv02tqsby51d,CME,1
+20110312,es uv02tqsby51d,CME,2
+20110317,es uv02tqsby51d,CME,0
+20110531,es uxhoi92owsn5,CME,1
+20110611,es uxhoi92owsn5,CME,2
+20110616,es uxhoi92owsn5,CME,0
+20110831,es uzza6rd1vg8x,CME,1
+20110910,es uzza6rd1vg8x,CME,2
+20110915,es uzza6rd1vg8x,CME,0
+20111130,es v2gvv9neu3up,CME,1
+20111210,es v2gvv9neu3up,CME,2
+20111215,es v2gvv9neu3up,CME,0
+20120229,es v4yhjrxrsrgh,CME,1
+20120311,es v4yhjrxrsrgh,CME,2
+20120315,es v4yhjrxrsrgh,CME,0
+20120531,es v7g38a84rf29,CME,1
+20120610,es v7g38a84rf29,CME,2
+20120614,es v7g38a84rf29,CME,0
+20120831,es va4l1g2o9csh,CME,1
+20120916,es va4l1g2o9csh,CME,2
+20120920,es va4l1g2o9csh,CME,0
+20121130,es vcm6pyd180e9,CME,1
+20121216,es vcm6pyd180e9,CME,2
+20121220,es vcm6pyd180e9,CME,0
+20130228,es veww9t37ndvl,CME,1
+20130310,es veww9t37ndvl,CME,2
+20130314,es veww9t37ndvl,CME,0
+20130531,es vhle2yxr5blt,CME,1
+20130616,es vhle2yxr5blt,CME,2
+20130620,es vhle2yxr5blt,CME,0
+20130831,es vk2zrh843z7l,CME,1
+20130915,es vk2zrh843z7l,CME,2
+20130919,es vk2zrh843z7l,CME,0
+20131130,es vmklfzih2mtd,CME,1
+20131215,es vmklfzih2mtd,CME,2
+20131219,es vmklfzih2mtd,CME,0
+20140228,es vp274hsu1af5,CME,1
+20140316,es vp274hsu1af5,CME,2
+20140320,es vp274hsu1af5,CME,0
+20140531,es vrjst036zy0x,CME,1
+20140615,es vrjst036zy0x,CME,2
+20140619,es vrjst036zy0x,CME,0
+20140831,es vu1ehidjylmp,CME,1
+20140914,es vu1ehidjylmp,CME,2
+20140918,es vu1ehidjylmp,CME,0
+20141130,es vwj060nwx98h,CME,1
+20141214,es vwj060nwx98h,CME,2
+20141218,es vwj060nwx98h,CME,0
+20150228,es vz0luiy9vwu9,CME,1
+20150315,es vz0luiy9vwu9,CME,2
+20150319,es vz0luiy9vwu9,CME,0
+20150531,es w1i7j18mukg1,CME,1
+20150614,es w1i7j18mukg1,CME,2
+20150618,es w1i7j18mukg1,CME,0
+20150831,es w3zt7jizt81t,CME,1
+20150913,es w3zt7jizt81t,CME,2
+20150917,es w3zt7jizt81t,CME,0
+20151130,es w6hew1tcrvnl,CME,1
+20151213,es w6hew1tcrvnl,CME,2
+20151217,es w6hew1tcrvnl,CME,0
+20160229,es w8z0kk3pqj9d,CME,1
+20160313,es w8z0kk3pqj9d,CME,2
+20160317,es w8z0kk3pqj9d,CME,0
+20160531,es wbgm92e2p6v5,CME,1
+20160612,es wbgm92e2p6v5,CME,2
+20160616,es wbgm92e2p6v5,CME,0
+20160831,es wdy7xkofnugx,CME,1
+20160911,es wdy7xkofnugx,CME,2
+20160915,es wdy7xkofnugx,CME,0
+20161130,es wgftm2ysmi2p,CME,1
+20161211,es wgftm2ysmi2p,CME,2
+20161215,es wgftm2ysmi2p,CME,0
+20170228,es wixfal95l5oh,CME,1
+20170312,es wixfal95l5oh,CME,2
+20170316,es wixfal95l5oh,CME,0
+20170531,es wlf0z3jijta9,CME,1
+20170611,es wlf0z3jijta9,CME,2
+20170615,es wlf0z3jijta9,CME,0
+20170831,es wnwmnltvigw1,CME,1
+20170910,es wnwmnltvigw1,CME,2
+20170914,es wnwmnltvigw1,CME,0
+20171130,es wqe8c448h4ht,CME,1
+20171210,es wqe8c448h4ht,CME,2
+20171214,es wqe8c448h4ht,CME,0
+20180228,es wsvu0melfs3l,CME,1
+20180312,es wsvu0melfs3l,CME,2
+20180315,es wsvu0melfs3l,CME,0
+20180531,es wvdfp4oyefpd,CME,1
+20180611,es wvdfp4oyefpd,CME,2
+20180614,es wvdfp4oyefpd,CME,0
+20180831,es wy1xiajhwdfl,CME,1
+20180916,es wy1xiajhwdfl,CME,2
+20180920,es wy1xiajhwdfl,CME,0
+20181130,es x0jj6stuv11d,CME,1
+20181216,es x0jj6stuv11d,CME,2
+20181220,es x0jj6stuv11d,CME,0
+20190228,es x2u8qnk1aeip,CME,1
+20190310,es x2u8qnk1aeip,CME,2
+20190314,es x2u8qnk1aeip,CME,0
+20190531,es x5iqjteksc8x,CME,1
+20190616,es x5iqjteksc8x,CME,2
+20190620,es x5iqjteksc8x,CME,0
+20190831,es x80c8boxqzup,CME,1
+20190915,es x80c8boxqzup,CME,2
+20190919,es x80c8boxqzup,CME,0
+20191130,es xahxwtzapngh,CME,1
+20191215,es xahxwtzapngh,CME,2
+20191219,es xahxwtzapngh,CME,0
+20200229,es xczjlc9nob29,CME,1
+20200316,es xczjlc9nob29,CME,2
+20200319,es xczjlc9nob29,CME,0
+20200531,es xfh59uk0myo1,CME,1
+20200615,es xfh59uk0myo1,CME,2
+20200618,es xfh59uk0myo1,CME,0
+20200831,es xhyqycudlm9t,CME,1
+20200913,es xhyqycudlm9t,CME,2
+20200917,es xhyqycudlm9t,CME,0
+20201130,es xkgcmv4qk9vl,CME,1
+20201213,es xkgcmv4qk9vl,CME,2
+20201217,es xkgcmv4qk9vl,CME,0
+20210228,es xmxybdf3ixhd,CME,1
+20210314,es xmxybdf3ixhd,CME,2
+20210318,es xmxybdf3ixhd,CME,0
+20210531,es xpfjzvpghl35,CME,1
+20210613,es xpfjzvpghl35,CME,2
+20210617,es xpfjzvpghl35,CME,0
+20210831,es xrx5odztg8ox,CME,1
+20210912,es xrx5odztg8ox,CME,2
+20210916,es xrx5odztg8ox,CME,0
+20211130,es xuercwa6ewap,CME,1
+20211216,es xuercwa6ewap,CME,0
+20220228,es xwwd1ekjdjwh,CME,1
+20220317,es xwwd1ekjdjwh,CME,0
+20220531,es xzdypwuwc7i9,CME,1
+20220616,es xzdypwuwc7i9,CME,0
+20220831,es y1vkef59av41,CME,1
+20220915,es y1vkef59av41,CME,0
+20501231,es xuercwa6ewap,CME,2

Engine/DataFeeds/BacktestingOptionChainProvider.cs

@@ -17,6 +17,7 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using QuantConnect.Configuration;
+using QuantConnect.Data.Auxiliary;
 using QuantConnect.Interfaces;
 using QuantConnect.Logging;
 using QuantConnect.Util;
@@ -59,10 +60,10 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             // Resolve any mapping before requesting option contract list for equities
             // Needs to be done in order for the data file key to be accurate
             Symbol mappedSymbol;
-            if (underlyingSymbol.SecurityType.RequiresMapping())
+            if (underlyingSymbol.RequiresMapping())
             {
-                var mapFileResolver = _mapFileProvider.Get(underlyingSymbol.ID.Market);
-                var mapFile = mapFileResolver.ResolveMapFile(underlyingSymbol.ID.Symbol, date);
+                var mapFileResolver = _mapFileProvider.Get(CorporateActionsKey.Create(underlyingSymbol));
+                var mapFile = mapFileResolver.ResolveMapFile(underlyingSymbol);
                 var ticker = mapFile.GetMappedSymbol(date, underlyingSymbol.Value);
                 mappedSymbol = underlyingSymbol.UpdateMappedSymbol(ticker);
             }

Engine/DataFeeds/DataManager.cs

@@ -389,11 +389,14 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             bool isFilteredSubscription = true,
             bool isInternalFeed = false,
             bool isCustomData = false,
-            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted
+            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted,
+            DataMappingMode dataMappingMode = DataMappingMode.OpenInterest,
+            uint contractDepthOffset = 0
             )
         {
             return Add(symbol, resolution, fillForward, extendedMarketHours, isFilteredSubscription, isInternalFeed, isCustomData,
-                new List<Tuple<Type, TickType>> { new Tuple<Type, TickType>(dataType, LeanData.GetCommonTickTypeForCommonDataTypes(dataType, symbol.SecurityType))}, dataNormalizationMode)
+                new List<Tuple<Type, TickType>> { new Tuple<Type, TickType>(dataType, LeanData.GetCommonTickTypeForCommonDataTypes(dataType, symbol.SecurityType))},
+                dataNormalizationMode, dataMappingMode, contractDepthOffset)
                 .First();
         }
 
@@ -411,7 +414,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             bool isInternalFeed = false,
             bool isCustomData = false,
             List<Tuple<Type, TickType>> subscriptionDataTypes = null,
-            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted
+            DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted,
+            DataMappingMode dataMappingMode = DataMappingMode.OpenInterest,
+            uint contractDepthOffset = 0
             )
         {
             var dataTypes = subscriptionDataTypes ??
@@ -462,7 +467,6 @@ namespace QuantConnect.Lean.Engine.DataFeeds
 
             var exchangeHours = marketHoursDbEntry.ExchangeHours;
             if (symbol.ID.SecurityType.IsOption() ||
-                symbol.ID.SecurityType == SecurityType.Future ||
                 symbol.ID.SecurityType == SecurityType.Index)
             {
                 dataNormalizationMode = DataNormalizationMode.Raw;
@@ -496,7 +500,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                     isCustomData,
                     isFilteredSubscription: isFilteredSubscription,
                     tickType: tickType,
-                    dataNormalizationMode: dataNormalizationMode)).ToList();
+                    dataNormalizationMode: dataNormalizationMode,
+                    dataMappingMode: dataMappingMode,
+                    contractDepthOffset: contractDepthOffset)).ToList();
 
             for (int i = 0; i < result.Count; i++)
             {

Engine/DataFeeds/Enumerators/AuxiliaryDataEnumerator.cs

@@ -15,10 +15,11 @@
 */
 
 using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
+using QuantConnect.Util;
 using QuantConnect.Data;
+using System.Collections;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
 using QuantConnect.Data.Auxiliary;
 
 namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
@@ -32,66 +33,46 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
     {
         private readonly Queue<BaseData> _auxiliaryData;
         private bool _initialized;
+        private DateTime _startTime;
+        private IMapFileProvider _mapFileProvider;
+        private IFactorFileProvider _factorFileProvider;
+        private ITradableDateEventProvider[] _tradableDateEventProviders;
+
+        /// <summary>
+        /// The associated data configuration
+        /// </summary>
+        protected SubscriptionDataConfig Config { get; }
 
         /// <summary>
         /// Creates a new instance
         /// </summary>
         /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
-        /// <param name="factorFile">The factor file to use</param>
-        /// <param name="mapFile">The <see cref="MapFile"/> to use</param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param>
         /// <param name="tradableDateEventProviders">The tradable dates event providers</param>
         /// <param name="tradableDayNotifier">Tradable dates provider</param>
         /// <param name="startTime">Start date for the data request</param>
         public AuxiliaryDataEnumerator(
             SubscriptionDataConfig config,
-            Lazy<FactorFile> factorFile,
-            Lazy<MapFile> mapFile,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
             ITradableDateEventProvider []tradableDateEventProviders,
             ITradableDatesNotifier tradableDayNotifier,
             DateTime startTime)
         {
+            Config = config;
+            _startTime = startTime;
+            _mapFileProvider = mapFileProvider;
             _auxiliaryData = new Queue<BaseData>();
+            _factorFileProvider = factorFileProvider;
+            _tradableDateEventProviders = tradableDateEventProviders;
 
-            tradableDayNotifier.NewTradableDate += (sender, eventArgs) =>
+            if (tradableDayNotifier != null)
             {
-                if (!_initialized)
-                {
-                    Initialize(config, factorFile, mapFile, tradableDateEventProviders, startTime);
-                }
-
-                foreach (var tradableDateEventProvider in tradableDateEventProviders)
-                {
-                    var newEvents = tradableDateEventProvider.GetEvents(eventArgs).ToList();
-                    foreach (var newEvent in newEvents)
-                    {
-                        _auxiliaryData.Enqueue(newEvent);
-                    }
-                }
-            };
-        }
-
-        /// <summary>
-        /// Late initialization so it is performed in the data feed stack
-        /// and not in the algorithm thread
-        /// </summary>
-        private void Initialize(SubscriptionDataConfig config,
-            Lazy<FactorFile> factorFile,
-            Lazy<MapFile> mapFile,
-            ITradableDateEventProvider[] tradableDateEventProviders,
-            DateTime startTime)
-        {
-            foreach (var tradableDateEventProvider in tradableDateEventProviders)
-            {
-                tradableDateEventProvider.Initialize(
-                    config,
-                    factorFile?.Value,
-                    mapFile?.Value,
-                    startTime);
+                tradableDayNotifier.NewTradableDate += NewTradableDate;
             }
-            _initialized = true;
         }
 
-
         /// <summary>
         /// Advances the enumerator to the next element.
         /// </summary>
@@ -102,11 +83,47 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
             return true;
         }
 
+        /// <summary>
+        /// Handle a new tradable date, drives the <see cref="ITradableDateEventProvider"/> instances
+        /// </summary>
+        protected void NewTradableDate(object sender, NewTradableDateEventArgs eventArgs)
+        {
+            Initialize();
+            for (var i = 0; i < _tradableDateEventProviders.Length; i++)
+            {
+                foreach (var newEvent in _tradableDateEventProviders[i].GetEvents(eventArgs))
+                {
+                    _auxiliaryData.Enqueue(newEvent);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Initializes the underlying tradable data event providers
+        /// </summary>
+        protected void Initialize()
+        {
+            if (!_initialized)
+            {
+                _initialized = true;
+                // Late initialization so it is performed in the data feed stack
+                for (var i = 0; i < _tradableDateEventProviders.Length; i++)
+                {
+                    _tradableDateEventProviders[i].Initialize(Config, _factorFileProvider, _mapFileProvider, _startTime);
+                }
+            }
+        }
+
         /// <summary>
         /// Dispose of the Stream Reader and close out the source stream and file connections.
         /// </summary>
         public void Dispose()
         {
+            for (var i = 0; i < _tradableDateEventProviders.Length; i++)
+            {
+                var disposable =_tradableDateEventProviders[i] as IDisposable;
+                disposable?.DisposeSafely();
+            }
         }
 
         /// <summary>

Engine/DataFeeds/Enumerators/DelistingEventProvider.cs

@@ -19,6 +19,7 @@ using System.Collections.Generic;
 using QuantConnect.Data;
 using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Market;
+using QuantConnect.Interfaces;
 using QuantConnect.Util;
 
 namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
@@ -45,16 +46,17 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// Initializes this instance
         /// </summary>
         /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
-        /// <param name="factorFile">The factor file to use</param>
-        /// <param name="mapFile">The <see cref="MapFile"/> to use</param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
         /// <param name="startTime">Start date for the data request</param>
         public virtual void Initialize(
             SubscriptionDataConfig config,
-            FactorFile factorFile,
-            MapFile mapFile,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
             DateTime startTime)
         {
             _config = config;
+            var mapFile = mapFileProvider.ResolveMapFile(_config);
             DelistingDate = new ReferenceWrapper<DateTime>(config.Symbol.GetDelistingDate(mapFile));
         }
 

Engine/DataFeeds/Enumerators/DividendEventProvider.cs

@@ -19,6 +19,7 @@ using System.Collections.Generic;
 using QuantConnect.Data;
 using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Market;
+using QuantConnect.Interfaces;
 
 namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
 {
@@ -31,7 +32,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         // and on the next trading day we use this data to produce the dividend instance
         private decimal? _priceFactorRatio;
         private decimal _referencePrice;
-        private FactorFile _factorFile;
+        private CorporateFactorProvider _factorFile;
         private MapFile _mapFile;
         private SubscriptionDataConfig _config;
 
@@ -39,18 +40,18 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// Initializes this instance
         /// </summary>
         /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
-        /// <param name="factorFile">The factor file to use</param>
-        /// <param name="mapFile">The <see cref="MapFile"/> to use</param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
         /// <param name="startTime">Start date for the data request</param>
         public void Initialize(
             SubscriptionDataConfig config,
-            FactorFile factorFile,
-            MapFile mapFile,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
             DateTime startTime)
         {
-            _mapFile = mapFile;
-            _factorFile = factorFile;
             _config = config;
+            _mapFile = mapFileProvider.ResolveMapFile(_config);
+            _factorFile = factorFileProvider.Get(_config.Symbol) as CorporateFactorProvider;
         }
 
         /// <summary>
@@ -61,6 +62,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         public IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
         {
             if (_config.Symbol == eventArgs.Symbol
+                && _factorFile != null
                 && _mapFile.HasData(eventArgs.Date))
             {
                 if (_priceFactorRatio != null)

Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactory.cs

@@ -30,22 +30,22 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
     public class BaseDataSubscriptionEnumeratorFactory : ISubscriptionEnumeratorFactory
     {
         private readonly Func<SubscriptionRequest, IEnumerable<DateTime>> _tradableDaysProvider;
-        private readonly MapFileResolver _mapFileResolver;
+        private readonly IMapFileProvider _mapFileProvider;
         private readonly bool _isLiveMode;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="BaseDataSubscriptionEnumeratorFactory"/> class
         /// </summary>
         /// <param name="isLiveMode">True for live mode, false otherwise</param>
-        /// <param name="mapFileResolver">Used for resolving the correct map files</param>
+        /// <param name="mapFileProvider">Used for resolving the correct map files</param>
         /// <param name="factorFileProvider">Used for getting factor files</param>
         /// <param name="tradableDaysProvider">Function used to provide the tradable dates to be enumerator.
         /// Specify null to default to <see cref="SubscriptionRequest.TradableDays"/></param>
-        public BaseDataSubscriptionEnumeratorFactory(bool isLiveMode, MapFileResolver mapFileResolver, IFactorFileProvider factorFileProvider, Func<SubscriptionRequest, IEnumerable<DateTime>> tradableDaysProvider = null)
+        public BaseDataSubscriptionEnumeratorFactory(bool isLiveMode, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, Func<SubscriptionRequest, IEnumerable<DateTime>> tradableDaysProvider = null)
         {
             _isLiveMode = isLiveMode;
             _tradableDaysProvider = tradableDaysProvider ?? (request => request.TradableDays);
-            _mapFileResolver = mapFileResolver;
+            _mapFileProvider = mapFileProvider;
         }
 
         /// <summary>
@@ -77,7 +77,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
             {
                 foreach (var date in _tradableDaysProvider(request))
                 {
-                    if (sourceFactory.RequiresMapping())
+                    if (sourceFactory.RequiresMapping() && _mapFileProvider != null)
                     {
                         request.Configuration.MappedSymbol = GetMappedSymbol(request.Configuration, date);
                     }
@@ -95,8 +95,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
 
         private string GetMappedSymbol(SubscriptionDataConfig config, DateTime date)
         {
-            return _mapFileResolver.ResolveMapFile(config.Symbol, config.Type)
-                .GetMappedSymbol(date, config.MappedSymbol);
+            return _mapFileProvider.ResolveMapFile(config).GetMappedSymbol(date, config.MappedSymbol);
         }
     }
 }

Engine/DataFeeds/Enumerators/Factories/CorporateEventEnumeratorFactory.cs

@@ -40,7 +40,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
         /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
         /// <param name="factorFileProvider">Used for getting factor files</param>
         /// <param name="tradableDayNotifier">Tradable dates provider</param>
-        /// <param name="mapFileResolver">Used for resolving the correct map files</param>
+        /// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param>
         /// <param name="startTime">Start date for the data request</param>
         /// <param name="enablePriceScaling">Applies price factor</param>
         /// <returns>The new auxiliary data enumerator</returns>
@@ -49,24 +49,20 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
             SubscriptionDataConfig config,
             IFactorFileProvider factorFileProvider,
             ITradableDatesNotifier tradableDayNotifier,
-            MapFileResolver mapFileResolver,
+            IMapFileProvider mapFileProvider,
             DateTime startTime,
             bool enablePriceScaling = true)
         {
-            var lazyFactorFile =
-                new Lazy<FactorFile>(() => SubscriptionUtils.GetFactorFileToUse(config, factorFileProvider));
 
             var tradableEventProviders = new List<ITradableDateEventProvider>();
 
-            if (config.Symbol.SecurityType == SecurityType.Equity)
+            if (config.EmitSplitsAndDividends())
             {
                 tradableEventProviders.Add(new SplitEventProvider());
                 tradableEventProviders.Add(new DividendEventProvider());
             }
 
-            if (config.Symbol.SecurityType == SecurityType.Equity
-                || config.Symbol.SecurityType == SecurityType.Base
-                || config.Symbol.SecurityType == SecurityType.Option)
+            if (config.TickerShouldBeMapped())
             {
                 tradableEventProviders.Add(new MappingEventProvider());
             }
@@ -75,8 +71,8 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
 
             var enumerator = new AuxiliaryDataEnumerator(
                 config,
-                lazyFactorFile,
-                new Lazy<MapFile>(() => GetMapFileToUse(config, mapFileResolver)),
+                factorFileProvider,
+                mapFileProvider,
                 tradableEventProviders.ToArray(),
                 tradableDayNotifier,
                 startTime);
@@ -84,44 +80,15 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
             // avoid price scaling for backtesting; calculate it directly in worker
             // and allow subscription to extract the the data depending on config data mode
             var dataEnumerator = rawDataEnumerator;
-            if (enablePriceScaling)
+            if (enablePriceScaling && config.PricesShouldBeScaled())
             {
                 dataEnumerator = new PriceScaleFactorEnumerator(
                     rawDataEnumerator,
                     config,
-                    lazyFactorFile);
+                    factorFileProvider);
             }
 
             return new SynchronizingEnumerator(dataEnumerator, enumerator);
         }
-
-        private static MapFile GetMapFileToUse(
-            SubscriptionDataConfig config,
-            MapFileResolver mapFileResolver)
-        {
-            var mapFileToUse = new MapFile(config.Symbol.Value, new List<MapFileRow>());
-
-            // load up the map and factor files for equities, options, and custom data
-            if (config.TickerShouldBeMapped())
-            {
-                try
-                {
-                    var mapFile = mapFileResolver.ResolveMapFile(config.Symbol, config.Type);
-
-                    // only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
-                    if (mapFile.Any())
-                    {
-                        mapFileToUse = mapFile;
-                    }
-                }
-                catch (Exception err)
-                {
-                    Log.Error(err, "CorporateEventEnumeratorFactory.GetMapFileToUse():" +
-                        " Map File: " + config.Symbol.ID + ": ");
-                }
-            }
-
-            return mapFileToUse;
-        }
     }
 }

Engine/DataFeeds/Enumerators/Factories/SubscriptionDataReaderSubscriptionEnumeratorFactory.cs

@@ -15,15 +15,14 @@
 */
 
 using System;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
 using System.Linq;
 using QuantConnect.Data;
-using QuantConnect.Data.Auxiliary;
-using QuantConnect.Data.UniverseSelection;
-using QuantConnect.Interfaces;
-using QuantConnect.Lean.Engine.Results;
 using QuantConnect.Util;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using QuantConnect.Lean.Engine.Results;
+using QuantConnect.Data.UniverseSelection;
 
 namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
 {
@@ -44,7 +43,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
         private readonly Func<SubscriptionRequest, IEnumerable<DateTime>> _tradableDaysProvider;
         private readonly IMapFileProvider _mapFileProvider;
         private readonly bool _enablePriceScaling;
-        
+
         /// <summary>
         /// Initializes a new instance of the <see cref="SubscriptionDataReaderSubscriptionEnumeratorFactory"/> class
         /// </summary>
@@ -82,14 +81,10 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
         /// <returns>An enumerator reading the subscription request</returns>
         public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
         {
-            var mapFileResolver = request.Configuration.TickerShouldBeMapped()
-                                    ? _mapFileProvider.Get(request.Security.Symbol.ID.Market)
-                                    : MapFileResolver.Empty;
-
             var dataReader = new SubscriptionDataReader(request.Configuration,
                 request.StartTimeLocal,
                 request.EndTimeLocal,
-                mapFileResolver,
+                _mapFileProvider,
                 _factorFileProvider,
                 _tradableDaysProvider(request),
                 _isLiveMode,
@@ -122,7 +117,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
                 request.Configuration,
                 _factorFileProvider,
                 dataReader,
-                mapFileResolver,
+                _mapFileProvider,
                 request.StartTimeLocal,
                 _enablePriceScaling);
 

Engine/DataFeeds/Enumerators/ITradableDateEventProvider.cs

@@ -15,8 +15,9 @@
 */
 
 using System;
-using System.Collections.Generic;
 using QuantConnect.Data;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
 using QuantConnect.Data.Auxiliary;
 
 namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
@@ -37,12 +38,12 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// Initializes the event provider instance
         /// </summary>
         /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
-        /// <param name="factorFile">The factor file to use</param>
-        /// <param name="mapFile">The <see cref="MapFile"/> to use</param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param>
         /// <param name="startTime">Start date for the data request</param>
         void Initialize(SubscriptionDataConfig config,
-            FactorFile factorFile,
-            MapFile mapFile,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
             DateTime startTime);
     }
 }

Engine/DataFeeds/Enumerators/LiveAuxiliaryDataEnumerator.cs

@@ -0,0 +1,105 @@
+/*
+* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using QuantConnect.Data;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Auxiliary data enumerator that will trigger new tradable dates event accordingly
+    /// </summary>
+    public class LiveAuxiliaryDataEnumerator : AuxiliaryDataEnumerator
+    {
+        private DateTime _lastTime;
+        private ITimeProvider _timeProvider;
+        private SecurityCache _securityCache;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="MapFile"/> provider to use</param>
+        /// <param name="tradableDateEventProviders">The tradable dates event providers</param>
+        /// <param name="startTime">Start date for the data request</param>
+        /// <param name="timeProvider">The time provider to use</param>
+        /// <param name="securityCache">The security cache</param>
+        public LiveAuxiliaryDataEnumerator(SubscriptionDataConfig config, IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider, ITradableDateEventProvider[] tradableDateEventProviders,
+            DateTime startTime,
+            ITimeProvider timeProvider,
+            SecurityCache securityCache)
+            // tradableDayNotifier: null -> we are going to trigger the new tradables events for the base implementation
+            : base(config, factorFileProvider, mapFileProvider, tradableDateEventProviders, tradableDayNotifier:null, startTime)
+        {
+            _securityCache = securityCache;
+            _timeProvider = timeProvider;
+
+            // initialize providers right away so mapping happens before we subscribe
+            Initialize();
+        }
+
+        public override bool MoveNext()
+        {
+            var currentDate = _timeProvider.GetUtcNow().ConvertFromUtc(Config.ExchangeTimeZone).Date;
+            if (currentDate != _lastTime)
+            {
+                // when the date changes for the security we trigger a new tradable date event
+                var newDayEvent = new NewTradableDateEventArgs(currentDate, _securityCache.GetData(), Config.Symbol, null);
+
+                NewTradableDate(this, newDayEvent);
+                // update last time
+                _lastTime = currentDate;
+            }
+
+            return base.MoveNext();
+        }
+
+        /// <summary>
+        /// Helper method to create a new instance.
+        /// Knows which security types should create one and determines the appropriate delisting event provider to use
+        /// </summary>
+        public static bool TryCreate(SubscriptionDataConfig dataConfig, ITimeProvider timeProvider, IDataQueueHandler dataQueueHandler,
+            SecurityCache securityCache, IMapFileProvider mapFileProvider, IFactorFileProvider fileProvider, DateTime startTime,
+            out IEnumerator<BaseData> enumerator)
+        {
+            enumerator = null;
+            var securityType = dataConfig.SecurityType;
+            if (securityType.IsOption() || securityType == SecurityType.Future || securityType == SecurityType.Equity)
+            {
+                var providers = new List<ITradableDateEventProvider>
+                {
+                    securityType == SecurityType.Equity
+                        ? new LiveDataBasedDelistingEventProvider(dataConfig, dataQueueHandler)
+                        : new DelistingEventProvider()
+                };
+
+                if (dataConfig.TickerShouldBeMapped())
+                {
+                    providers.Add(new LiveMappingEventProvider());
+                }
+
+                enumerator = new LiveAuxiliaryDataEnumerator(dataConfig, fileProvider, mapFileProvider,
+                    providers.ToArray(), startTime, timeProvider, securityCache);
+            }
+            return enumerator != null;
+        }
+    }
+}

Engine/DataFeeds/Enumerators/LiveAuxiliaryDataSynchronizingEnumerator.cs

@@ -41,11 +41,11 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// <param name="exchangeTimeZone">The time zone the raw data is time stamped in</param>
         /// <param name="tradeBarAggregator">The trade bar aggregator enumerator</param>
         /// <param name="auxDataEnumerators">The auxiliary data enumerators</param>
-        public LiveAuxiliaryDataSynchronizingEnumerator(ITimeProvider timeProvider, DateTimeZone exchangeTimeZone, IEnumerator<BaseData> tradeBarAggregator, params IEnumerator<BaseData>[] auxDataEnumerators)
+        public LiveAuxiliaryDataSynchronizingEnumerator(ITimeProvider timeProvider, DateTimeZone exchangeTimeZone, IEnumerator<BaseData> tradeBarAggregator, List<IEnumerator<BaseData>> auxDataEnumerators)
         {
             _timeProvider = timeProvider;
             _exchangeTimeZone = exchangeTimeZone;
-            _auxDataEnumerators = auxDataEnumerators.ToList();
+            _auxDataEnumerators = auxDataEnumerators;
             _tradeBarAggregator = tradeBarAggregator;
         }
 

Engine/DataFeeds/Enumerators/LiveDataBasedDelistingEventProvider.cs

@@ -49,16 +49,16 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// Initializes this instance
         /// </summary>
         /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
-        /// <param name="factorFile">The factor file to use</param>
-        /// <param name="mapFile">The <see cref="MapFile"/> to use</param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
         /// <param name="startTime">Start date for the data request</param>
         public override void Initialize(
             SubscriptionDataConfig config,
-            FactorFile factorFile,
-            MapFile mapFile,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
             DateTime startTime)
         {
-            base.Initialize(config, factorFile, mapFile, startTime);
+            base.Initialize(config, factorFileProvider, mapFileProvider, startTime);
 
             _delistingEnumerator = _dataQueueHandler.Subscribe(_dataConfig, (sender, args) =>
             {

Engine/DataFeeds/Enumerators/LiveDelistingEventProviderEnumerator.cs

@@ -1,141 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using QuantConnect.Data;
-using System.Collections;
-using QuantConnect.Interfaces;
-using QuantConnect.Securities;
-using System.Collections.Generic;
-using QuantConnect.Data.Auxiliary;
-using QuantConnect.Util;
-
-namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
-{
-    /// <summary>
-    /// A live trading delisting event provider which uses a <see cref="DelistingEventProvider"/> internally to emit events
-    /// based on the current frontier time
-    /// </summary>
-    /// <remarks>We create the events dynamically so that we can set the current security price in the event</remarks>
-    public class LiveDelistingEventProviderEnumerator : IEnumerator<BaseData>
-    {
-        private DateTime _lastTime;
-        private readonly ITimeProvider _timeProvider;
-        private readonly Queue<BaseData> _dataToEmit;
-        private readonly SecurityCache _securityCache;
-        private readonly SubscriptionDataConfig _dataConfig;
-        private readonly DelistingEventProvider _delistingEventProvider;
-
-        /// <summary>
-        /// Gets the element in the collection at the current position of the enumerator.
-        /// </summary>
-        /// <returns>The element in the collection at the current position of the enumerator.</returns>
-        public BaseData Current { get; private set; }
-
-        /// <summary>
-        /// Gets the current element in the collection.
-        /// </summary>
-        /// <returns>The current element in the collection.</returns>
-        object IEnumerator.Current => Current;
-
-        /// <summary>
-        /// Creates a new instance
-        /// </summary>
-        private LiveDelistingEventProviderEnumerator(ITimeProvider timeProvider, SubscriptionDataConfig dataConfig,
-            SecurityCache securityCache, DelistingEventProvider delistingEventProvider, MapFile mapFile)
-        {
-            _dataConfig = dataConfig;
-            _timeProvider = timeProvider;
-            _securityCache = securityCache;
-            _dataToEmit = new Queue<BaseData>();
-            _delistingEventProvider = delistingEventProvider;
-            _delistingEventProvider.Initialize(dataConfig, null, mapFile, DateTime.UtcNow);
-        }
-
-        /// <summary>
-        /// Advances the enumerator to the next element of the collection.
-        /// </summary>
-        /// <returns> true if the enumerator was successfully advanced to the next element; false if the enumerator has passed the end of the collection.</returns>
-        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created.</exception>
-        public bool MoveNext()
-        {
-            var currentDate = _timeProvider.GetUtcNow().ConvertFromUtc(_dataConfig.ExchangeTimeZone).Date;
-            if (currentDate != _lastTime)
-            {
-                // when the date changes for the security we trigger a new tradable date event
-                var newDayEvent = new NewTradableDateEventArgs(currentDate, _securityCache.GetData(), _dataConfig.Symbol, null);
-                foreach (var delistingEvent in _delistingEventProvider.GetEvents(newDayEvent))
-                {
-                    _dataToEmit.Enqueue(delistingEvent);
-                }
-
-                // update last time
-                _lastTime = currentDate;
-            }
-
-            if (_dataToEmit.Count > 0)
-            {
-                // emit event if any
-                Current = _dataToEmit.Dequeue();
-                return true;
-            }
-
-            Current = null;
-            return false;
-        }
-
-        /// <summary>
-        /// Sets the enumerator to its initial position, which is before the first element in the collection.
-        /// </summary>
-        /// <exception cref="T:System.InvalidOperationException">The collection was modified after the enumerator was created.</exception>
-        public void Reset()
-        {
-            _dataToEmit.Clear();
-        }
-
-        /// <summary>
-        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
-        /// </summary>
-        public void Dispose()
-        {
-            var liveProvider = _delistingEventProvider as LiveDataBasedDelistingEventProvider;
-            liveProvider.DisposeSafely();
-        }
-
-        /// <summary>
-        /// Helper method to create a new instance.
-        /// Knows which security types should create one and determines the appropriate delisting event provider to use
-        /// </summary>
-        public static bool TryCreate(SubscriptionDataConfig dataConfig, ITimeProvider timeProvider, IDataQueueHandler dataQueueHandler,
-            SecurityCache securityCache, IMapFileProvider mapFileProvider, out IEnumerator<BaseData> enumerator)
-        {
-            enumerator = null;
-            var securityType = dataConfig.SecurityType;
-            if (securityType.IsOption() || securityType == SecurityType.Future || securityType == SecurityType.Equity)
-            {
-                var delistingEventProvider = new DelistingEventProvider();
-                MapFile mapFile = null;
-                if (securityType == SecurityType.Equity)
-                {
-                    delistingEventProvider = new LiveDataBasedDelistingEventProvider(dataConfig, dataQueueHandler);
-                    mapFile = mapFileProvider.Get(dataConfig.Symbol.ID.Market).ResolveMapFile(dataConfig.Symbol, dataConfig.Type);
-                }
-                enumerator = new LiveDelistingEventProviderEnumerator(timeProvider, dataConfig, securityCache, delistingEventProvider, mapFile);
-                return true;
-            }
-            return false;
-        }
-    }
-}

Engine/DataFeeds/Enumerators/LiveMappingEventProvider.cs

@@ -0,0 +1,37 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Event provider who will emit <see cref="SymbolChangedEvent"/> events
+    /// </summary>
+    /// <remarks>Only special behavior is that it will refresh map file on each new tradable date event</remarks>
+    public class LiveMappingEventProvider : MappingEventProvider
+    {
+        public override IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        {
+            // refresh map file instance
+            InitializeMapFile();
+
+            return base.GetEvents(eventArgs);
+        }
+    }
+}

Engine/DataFeeds/Enumerators/MappingEventProvider.cs

@@ -15,10 +15,11 @@
 */
 
 using System;
-using System.Collections.Generic;
 using QuantConnect.Data;
-using QuantConnect.Data.Auxiliary;
+using QuantConnect.Interfaces;
 using QuantConnect.Data.Market;
+using System.Collections.Generic;
+using QuantConnect.Data.Auxiliary;
 
 namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
 {
@@ -27,24 +28,27 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
     /// </summary>
     public class MappingEventProvider : ITradableDateEventProvider
     {
-        private MapFile _mapFile;
+        private IMapFileProvider _mapFileProvider;
         private SubscriptionDataConfig _config;
+        private MapFile _mapFile;
 
         /// <summary>
         /// Initializes this instance
         /// </summary>
         /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
-        /// <param name="factorFile">The factor file to use</param>
-        /// <param name="mapFile">The <see cref="MapFile"/> to use</param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
         /// <param name="startTime">Start date for the data request</param>
-        public void Initialize(
+        public virtual void Initialize(
             SubscriptionDataConfig config,
-            FactorFile factorFile,
-            MapFile mapFile,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
             DateTime startTime)
         {
-            _mapFile = mapFile;
+            _mapFileProvider = mapFileProvider;
             _config = config;
+            InitializeMapFile();
+
             if (_mapFile.HasData(startTime.Date))
             {
                 // initialize mapped symbol using request start date
@@ -57,24 +61,34 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// </summary>
         /// <param name="eventArgs">The new tradable day event arguments</param>
         /// <returns>New mapping event if any</returns>
-        public IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
+        public virtual IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
         {
             if (_config.Symbol == eventArgs.Symbol
                 && _mapFile.HasData(eventArgs.Date))
             {
-                // check to see if the symbol was remapped
+                var old = _config.MappedSymbol;
                 var newSymbol = _mapFile.GetMappedSymbol(eventArgs.Date, _config.MappedSymbol);
-                if (newSymbol != _config.MappedSymbol)
+                _config.MappedSymbol = newSymbol;
+
+                // check to see if the symbol was remapped
+                if (old != _config.MappedSymbol)
                 {
                     var changed = new SymbolChangedEvent(
                         _config.Symbol,
                         eventArgs.Date,
-                        _config.MappedSymbol,
-                        newSymbol);
-                    _config.MappedSymbol = newSymbol;
+                        old,
+                        _config.MappedSymbol);
                     yield return changed;
                 }
             }
         }
+
+        /// <summary>
+        /// Initializes the map file to use
+        /// </summary>
+        protected void InitializeMapFile()
+        {
+            _mapFile = _mapFileProvider.ResolveMapFile(_config);
+        }
     }
 }

Engine/DataFeeds/Enumerators/PriceScaleFactorEnumerator.cs

@@ -19,6 +19,7 @@ using System.Collections;
 using System.Collections.Generic;
 using QuantConnect.Data;
 using QuantConnect.Data.Auxiliary;
+using QuantConnect.Interfaces;
 
 namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
 {
@@ -31,8 +32,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
     {
         private readonly IEnumerator<BaseData> _rawDataEnumerator;
         private readonly SubscriptionDataConfig _config;
-        private readonly Lazy<FactorFile> _factorFile;
+        private readonly IFactorFileProvider _factorFileProvider;
         private DateTime _lastTradableDate;
+        private IFactorProvider _factorFile;
 
         /// <summary>
         /// Explicit interface implementation for <see cref="Current"/>
@@ -54,16 +56,16 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// <param name="rawDataEnumerator">The underlying raw data enumerator</param>
         /// <param name="config">The <see cref="SubscriptionDataConfig"/> to enumerate for.
         /// Will determine the <see cref="DataNormalizationMode"/> to use.</param>
-        /// <param name="factorFile">The <see cref="FactorFile"/> instance to use</param>
+        /// <param name="factorFileProvider">The <see cref="IFactorFileProvider"/> instance to use</param>
         public PriceScaleFactorEnumerator(
             IEnumerator<BaseData> rawDataEnumerator,
             SubscriptionDataConfig config,
-            Lazy<FactorFile> factorFile)
+            IFactorFileProvider factorFileProvider)
         {
             _lastTradableDate = DateTime.MinValue;
             _config = config;
             _rawDataEnumerator = rawDataEnumerator;
-            _factorFile = factorFile;
+            _factorFileProvider = factorFileProvider;
         }
 
         /// <summary>
@@ -88,16 +90,17 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
 
             if (underlyingReturnValue
                 && Current != null
-                && _factorFile != null
+                && _factorFileProvider != null
                 && _config.DataNormalizationMode != DataNormalizationMode.Raw)
             {
                 if (Current.Time.Date > _lastTradableDate)
                 {
+                    _factorFile = _factorFileProvider.Get(_config.Symbol);
                     _lastTradableDate = Current.Time.Date;
-                    UpdateScaleFactor(_lastTradableDate);
+                    _config.PriceScaleFactor = _factorFile.GetPriceScale(_lastTradableDate, _config.DataNormalizationMode, _config.ContractDepthOffset, _config.DataMappingMode);
                 }
 
-                Current = Current.Normalize(_config);
+                Current = Current.Normalize(_config.PriceScaleFactor, _config.DataNormalizationMode, _config.SumOfDividends);
             }
 
             return underlyingReturnValue;
@@ -111,26 +114,5 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         {
             throw new NotImplementedException("Reset method not implemented. Assumes loop will only be used once.");
         }
-
-        private void UpdateScaleFactor(DateTime date)
-        {
-            switch (_config.DataNormalizationMode)
-            {
-                case DataNormalizationMode.Raw:
-                    return;
-
-                case DataNormalizationMode.TotalReturn:
-                case DataNormalizationMode.SplitAdjusted:
-                    _config.PriceScaleFactor = _factorFile.Value.GetSplitFactor(date);
-                    break;
-
-                case DataNormalizationMode.Adjusted:
-                    _config.PriceScaleFactor = _factorFile.Value.GetPriceScaleFactor(date);
-                    break;
-
-                default:
-                    throw new ArgumentOutOfRangeException();
-            }
-        }
     }
 }

Engine/DataFeeds/Enumerators/SplitEventProvider.cs

@@ -19,6 +19,7 @@ using System.Collections.Generic;
 using QuantConnect.Data;
 using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Market;
+using QuantConnect.Interfaces;
 
 namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
 {
@@ -31,7 +32,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         // and on the next trading day we use this data to produce the split instance
         private decimal? _splitFactor;
         private decimal _referencePrice;
-        private FactorFile _factorFile;
+        private CorporateFactorProvider _factorFile;
         private MapFile _mapFile;
         private SubscriptionDataConfig _config;
 
@@ -39,18 +40,18 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         /// Initializes this instance
         /// </summary>
         /// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
-        /// <param name="factorFile">The factor file to use</param>
-        /// <param name="mapFile">The <see cref="MapFile"/> to use</param>
+        /// <param name="factorFileProvider">The factor file provider to use</param>
+        /// <param name="mapFileProvider">The <see cref="Data.Auxiliary.MapFile"/> provider to use</param>
         /// <param name="startTime">Start date for the data request</param>
         public void Initialize(
             SubscriptionDataConfig config,
-            FactorFile factorFile,
-            MapFile mapFile,
+            IFactorFileProvider factorFileProvider,
+            IMapFileProvider mapFileProvider,
             DateTime startTime)
         {
-            _mapFile = mapFile;
-            _factorFile = factorFile;
             _config = config;
+            _mapFile = mapFileProvider.ResolveMapFile(_config);
+            _factorFile = factorFileProvider.Get(_config.Symbol) as CorporateFactorProvider;
         }
 
         /// <summary>
@@ -61,6 +62,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
         public IEnumerable<BaseData> GetEvents(NewTradableDateEventArgs eventArgs)
         {
             if (_config.Symbol == eventArgs.Symbol
+                && _factorFile != null
                 && _mapFile.HasData(eventArgs.Date))
             {
                 var factor = _splitFactor;

Engine/DataFeeds/FileSystemDataFeed.cs

@@ -18,6 +18,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using QuantConnect.Data;
+using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Market;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Interfaces;
@@ -149,11 +150,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             {
                 factory = new OptionChainUniverseSubscriptionEnumeratorFactory((req) =>
                 {
-                    var mapFileResolver = req.Security.Symbol.SecurityType == SecurityType.Option
-                        ? _mapFileProvider.Get(req.Configuration.Market)
-                        : null;
-
-                    var underlyingFactory = new BaseDataSubscriptionEnumeratorFactory(false, mapFileResolver, _factorFileProvider);
+                    var underlyingFactory = new BaseDataSubscriptionEnumeratorFactory(false, _mapFileProvider, _factorFileProvider);
                     return ConfigureEnumerator(req, true, underlyingFactory.CreateEnumerator(req, _dataProvider));
                 });
             }

Engine/DataFeeds/LiveTradingDataFeed.cs

@@ -20,6 +20,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using QuantConnect.Configuration;
 using QuantConnect.Data;
+using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Custom;
 using QuantConnect.Data.Custom.Tiingo;
 using QuantConnect.Data.Market;
@@ -50,6 +51,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         private IDataQueueHandler _dataQueueHandler;
         private BaseDataExchange _customExchange;
         private SubscriptionCollection _subscriptions;
+        private IFactorFileProvider _factorFileProvider;
         private CancellationTokenSource _cancellationTokenSource = new CancellationTokenSource();
         private IDataChannelProvider _channelProvider;
 
@@ -85,6 +87,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             _timeProvider = dataFeedTimeProvider.TimeProvider;
             _dataProvider = dataProvider;
             _mapFileProvider = mapFileProvider;
+            _factorFileProvider = factorFileProvider;
             _channelProvider = dataChannelProvider;
             _frontierTimeProvider = dataFeedTimeProvider.FrontierTimeProvider;
             _customExchange = new BaseDataExchange("CustomDataExchange") {SleepInterval = 10};
@@ -217,27 +220,26 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                 }
                 else
                 {
-                    EventHandler handler = (sender, args) => subscription?.OnNewDataAvailable();
-                    enumerator = _dataQueueHandler.Subscribe(request.Configuration, handler);
-
-                    var securityType = request.Configuration.SecurityType;
                     var auxEnumerators = new List<IEnumerator<BaseData>>();
 
-                    if (securityType == SecurityType.Equity)
+                    if (LiveAuxiliaryDataEnumerator.TryCreate(request.Configuration, _timeProvider, _dataQueueHandler,
+                        request.Security.Cache, _mapFileProvider, _factorFileProvider, request.StartTimeLocal, out var auxDataEnumator))
+                    {
+                        auxEnumerators.Add(auxDataEnumator);
+                    }
+
+                    EventHandler handler = (_, _) => subscription?.OnNewDataAvailable();
+                    enumerator = _dataQueueHandler.Subscribe(request.Configuration, handler);
+
+                    if (request.Configuration.EmitSplitsAndDividends())
                     {
                         auxEnumerators.Add(_dataQueueHandler.Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Dividend)), handler));
                         auxEnumerators.Add(_dataQueueHandler.Subscribe(new SubscriptionDataConfig(request.Configuration, typeof(Split)), handler));
                     }
 
-                    IEnumerator<BaseData> delistingEnumerator;
-                    if (LiveDelistingEventProviderEnumerator.TryCreate(request.Configuration, _timeProvider, _dataQueueHandler, request.Security.Cache, _mapFileProvider, out delistingEnumerator))
-                    {
-                        auxEnumerators.Add(delistingEnumerator);
-                    }
-
                     if (auxEnumerators.Count > 0)
                     {
-                        enumerator = new LiveAuxiliaryDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, auxEnumerators.ToArray());
+                        enumerator = new LiveAuxiliaryDataSynchronizingEnumerator(_timeProvider, request.Configuration.ExchangeTimeZone, enumerator, auxEnumerators);
                     }
                 }
 

Engine/DataFeeds/NullDataFeed.cs

@@ -27,11 +27,20 @@ namespace QuantConnect.Lean.Engine.DataFeeds
     /// </summary>
     public class NullDataFeed : IDataFeed
     {
+        /// <summary>
+        /// Allows specifying if this implementation should throw always or not
+        /// </summary>
+        public bool ShouldThrow { get; set; } = true;
+
         /// <inheritdoc />
         public bool IsActive
         {
             get
             {
+                if (!ShouldThrow)
+                {
+                    return true;
+                }
                 throw new NotImplementedException("Unexpected usage of null data feed implementation.");
             }
         }
@@ -49,24 +58,40 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             IDataChannelProvider channelProvider
             )
         {
+            if (!ShouldThrow)
+            {
+                return;
+            }
             throw new NotImplementedException("Unexpected usage of null data feed implementation.");
         }
 
         /// <inheritdoc />
         public Subscription CreateSubscription(SubscriptionRequest request)
         {
+            if (!ShouldThrow)
+            {
+                return null;
+            }
             throw new NotImplementedException("Unexpected usage of null data feed implementation.");
         }
 
         /// <inheritdoc />
         public void RemoveSubscription(Subscription subscription)
         {
+            if (!ShouldThrow)
+            {
+                return;
+            }
             throw new NotImplementedException("Unexpected usage of null data feed implementation.");
         }
 
         /// <inheritdoc />
         public void Exit()
         {
+            if (!ShouldThrow)
+            {
+                return;
+            }
             throw new NotImplementedException("Unexpected usage of null data feed implementation.");
         }
     }

Engine/DataFeeds/Queues/FakeDataQueue.cs

@@ -31,7 +31,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Queues
     /// <summary>
     /// This is an implementation of <see cref="IDataQueueHandler"/> used for testing
     /// </summary>
-    public class FakeDataQueue : IDataQueueHandler
+    public class FakeDataQueue : IDataQueueHandler, IDataQueueUniverseProvider
     {
         private int _count;
         private readonly Random _random = new Random();
@@ -205,5 +205,15 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Queues
             }
             return offsetProvider;
         }
+
+        public IEnumerable<Symbol> LookupSymbols(Symbol symbol, bool includeExpired, string securityCurrency = null)
+        {
+            yield break;
+        }
+
+        public bool CanPerformSelection()
+        {
+            return true;
+        }
     }
 }

Engine/DataFeeds/SubscriptionData.cs

@@ -15,7 +15,6 @@
 
 using System;
 using QuantConnect.Data;
-using QuantConnect.Data.Market;
 using QuantConnect.Securities;
 
 namespace QuantConnect.Lean.Engine.DataFeeds
@@ -80,20 +79,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                 data.Time = data.Time.ExchangeRoundDownInTimeZone(configuration.Increment, exchangeHours, configuration.DataTimeZone, configuration.ExtendedMarketHours);
             }
 
-            if (factor.HasValue && (factor.Value != 1 || configuration.SumOfDividends != 0))
+            if (factor.HasValue && (configuration.SecurityType != SecurityType.Equity || (factor.Value != 1 || configuration.SumOfDividends != 0)))
             {
-                var sumOfDividends = configuration.SumOfDividends;
-
-                var normalizedData = data.Clone(data.IsFillForward);
-
-                if (normalizationMode == DataNormalizationMode.Adjusted || normalizationMode == DataNormalizationMode.SplitAdjusted)
-                {
-                    normalizedData.Adjust(factor.Value);
-                }
-                else if (normalizationMode == DataNormalizationMode.TotalReturn)
-                {
-                    normalizedData.Scale(p => p * factor.Value + sumOfDividends, 1/factor.Value);
-                }
+                var normalizedData = data.Clone(data.IsFillForward).Normalize(factor.Value, normalizationMode, configuration.SumOfDividends);
 
                 return new PrecalculatedSubscriptionData(configuration, data, normalizedData, normalizationMode, emitTimeUtc);
             }

Engine/DataFeeds/SubscriptionDataReader.cs

@@ -61,9 +61,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         private DateTime _periodStart;
         private readonly DateTime _periodFinish;
 
-        private readonly MapFileResolver _mapFileResolver;
+        private readonly IMapFileProvider _mapFileProvider;
         private readonly IFactorFileProvider _factorFileProvider;
-        private FactorFile _factorFile;
+        private IFactorProvider _factorFile;
         private MapFile _mapFile;
 
         private bool _pastDelistedDate;
@@ -141,7 +141,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         public SubscriptionDataReader(SubscriptionDataConfig config,
             DateTime periodStart,
             DateTime periodFinish,
-            MapFileResolver mapFileResolver,
+            IMapFileProvider mapFileProvider,
             IFactorFileProvider factorFileProvider,
             IEnumerable<DateTime> tradeableDates,
             bool isLiveMode,
@@ -154,7 +154,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             //Save Start and End Dates:
             _periodStart = periodStart;
             _periodFinish = periodFinish;
-            _mapFileResolver = mapFileResolver;
+            _mapFileProvider = mapFileProvider;
             _factorFileProvider = factorFileProvider;
             _dataCacheProvider = dataCacheProvider;
 
@@ -208,21 +208,18 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                 }
             }
 
-            _factorFile = new FactorFile(_config.Symbol.Value, new List<FactorFileRow>());
-            _mapFile = new MapFile(_config.Symbol.Value, new List<MapFileRow>());
-
             // load up the map files for equities, options, and custom data if it supports it.
             // Only load up factor files for equities
             if (_dataFactory.RequiresMapping())
             {
                 try
                 {
-                    var mapFile = _mapFileResolver.ResolveMapFile(_config.Symbol, _config.Type);
+                    var mapFile = _mapFileProvider.ResolveMapFile(_config);
 
                     // only take the resolved map file if it has data, otherwise we'll use the empty one we defined above
                     if (mapFile.Any()) _mapFile = mapFile;
 
-                    if (!_config.IsCustomData && !_config.SecurityType.IsOption())
+                    if (_config.PricesShouldBeScaled())
                     {
                         var factorFile = _factorFileProvider.Get(_config.Symbol);
                         _hasScaleFactors = factorFile != null;
@@ -261,6 +258,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                 }
             }
 
+            _factorFile ??= _config.Symbol.GetEmptyFactorFile();
+            _mapFile ??= new MapFile(_config.Symbol.Value, Enumerable.Empty<MapFileRow>());
+
             _delistingDate = _config.Symbol.GetDelistingDate(_mapFile);
 
             // adding a day so we stop at EOD

Engine/DataFeeds/SubscriptionUtils.cs

@@ -54,7 +54,6 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             return new Subscription(request, dataEnumerator, timeZoneOffsetProvider);
         }
 
-
         /// <summary>
         /// Setups a new <see cref="Subscription"/> which will consume a blocking <see cref="EnqueueableEnumerator{T}"/>
         /// that will be feed by a worker task
@@ -70,14 +69,13 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             IFactorFileProvider factorFileProvider,
             bool enablePriceScale)
         {
-            var factorFile = GetFactorFileToUse(request.Configuration, factorFileProvider);
             var exchangeHours = request.Security.Exchange.Hours;
             var enqueueable = new EnqueueableEnumerator<SubscriptionData>(true);
             var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Security.Exchange.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
             var subscription = new Subscription(request, enqueueable, timeZoneOffsetProvider);
             var config = subscription.Configuration;
+            enablePriceScale = enablePriceScale && config.PricesShouldBeScaled();
             var lastTradableDate = DateTime.MinValue;
-            decimal? currentScale = null;
 
             Func<int, bool> produce = (workBatchSize) =>
             {
@@ -107,16 +105,18 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                         // if the config is changed at any point it can emit adjusted data as well
                         // See SubscriptionData.Create() and PrecalculatedSubscriptionData for more
                         var requestMode = config.DataNormalizationMode;
-                        var mode = requestMode != DataNormalizationMode.Raw
-                            ? requestMode
-                            : DataNormalizationMode.Adjusted;
+                        if (config.SecurityType == SecurityType.Equity)
+                        {
+                            requestMode = requestMode != DataNormalizationMode.Raw ? requestMode : DataNormalizationMode.Adjusted;
+                        }
 
                         // We update our price scale factor when the date changes for non fill forward bars or if we haven't initialized yet.
                         // We don't take into account auxiliary data because we don't scale it and because the underlying price data could be fill forwarded
                         if (enablePriceScale && data?.Time.Date > lastTradableDate && data.DataType != MarketDataType.Auxiliary && (!data.IsFillForward || lastTradableDate == DateTime.MinValue))
                         {
+                            var factorFile = factorFileProvider.Get(request.Configuration.Symbol);
                             lastTradableDate = data.Time.Date;
-                            currentScale = GetScaleFactor(factorFile, mode, data.Time.Date);
+                            request.Configuration.PriceScaleFactor = factorFile.GetPriceScale(data.Time.Date, requestMode, config.ContractDepthOffset, config.DataMappingMode);
                         }
 
                         SubscriptionData subscriptionData = SubscriptionData.Create(
@@ -124,8 +124,8 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                             exchangeHours,
                             subscription.OffsetProvider,
                             data,
-                            mode,
-                            enablePriceScale ? currentScale : null);
+                            requestMode,
+                            enablePriceScale ? request.Configuration.PriceScaleFactor : null);
 
                         // drop the data into the back of the enqueueable
                         enqueueable.Enqueue(subscriptionData);
@@ -165,58 +165,5 @@ namespace QuantConnect.Lean.Engine.DataFeeds
 
             return subscription;
         }
-
-        /// <summary>
-        /// Gets <see cref="FactorFile"/> for configuration
-        /// </summary>
-        /// <param name="config">Subscription configuration</param>
-        /// <param name="factorFileProvider">The factor file provider</param>
-        /// <returns></returns>
-        public static FactorFile GetFactorFileToUse(
-            SubscriptionDataConfig config,
-            IFactorFileProvider factorFileProvider)
-        {
-            var factorFileToUse = new FactorFile(config.Symbol.Value, new List<FactorFileRow>());
-
-            if (!config.IsCustomData
-                && config.SecurityType == SecurityType.Equity)
-            {
-                try
-                {
-                    var factorFile = factorFileProvider.Get(config.Symbol);
-                    if (factorFile != null)
-                    {
-                        factorFileToUse = factorFile;
-                    }
-                }
-                catch (Exception err)
-                {
-                    Log.Error(err, "SubscriptionUtils.GetFactorFileToUse(): Factors File: "
-                        + config.Symbol.ID + ": ");
-                }
-            }
-
-            return factorFileToUse;
-        }
-
-        private static decimal GetScaleFactor(FactorFile factorFile, DataNormalizationMode mode, DateTime date)
-        {
-            switch (mode)
-            {
-                case DataNormalizationMode.Raw:
-                    return 1;
-
-                case DataNormalizationMode.TotalReturn:
-                case DataNormalizationMode.SplitAdjusted:
-                    return factorFile.GetSplitFactor(date);
-
-                case DataNormalizationMode.Adjusted:
-                    return factorFile.GetPriceScaleFactor(date);
-
-                default:
-                    throw new ArgumentOutOfRangeException();
-            }
-        }
-
     }
 }

Engine/DataFeeds/TimeSliceFactory.cs

@@ -282,8 +282,8 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                             }
                         }
 
-                        // special handling of futures data to build the futures chain
-                        if (symbol.SecurityType == SecurityType.Future)
+                        // special handling of futures data to build the futures chain. Don't push canonical continuous contract
+                        if (symbol.SecurityType == SecurityType.Future && !symbol.IsCanonical())
                         {
                             // internal feeds, like open interest, will not create the chain but will update it if it exists
                             // this is because the open interest could arrive at some closed market hours in which there is no other data and we don't

Engine/DataFeeds/UniverseSelection.cs

@@ -516,7 +516,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                     universeSettings.FillForward,
                     universeSettings.ExtendedMarketHours,
                     dataNormalizationMode: universeSettings.DataNormalizationMode,
-                    subscriptionDataTypes: universeSettings.SubscriptionDataTypes);
+                    subscriptionDataTypes: universeSettings.SubscriptionDataTypes,
+                    dataMappingMode: universeSettings.DataMappingMode,
+                    contractDepthOffset: (uint)Math.Abs(universeSettings.ContractDepthOffset));
 
                 security = _securityService.CreateSecurity(symbol, configs, universeSettings.Leverage, symbol.ID.SecurityType.IsOption(), underlying);
 

Engine/HistoricalData/SubscriptionDataReaderHistoryProvider.cs

@@ -111,21 +111,13 @@ namespace QuantConnect.Lean.Engine.HistoricalData
                 new SecurityCache()
             );
 
-            var mapFileResolver = MapFileResolver.Empty;
-            if (config.TickerShouldBeMapped())
-            {
-                mapFileResolver = _mapFileProvider.Get(config.Market);
-                var mapFile = mapFileResolver.ResolveMapFile(config.Symbol.ID.Symbol, config.Symbol.ID.Date);
-                config.MappedSymbol = mapFile.GetMappedSymbol(startTimeLocal, config.MappedSymbol);
-            }
-
             // Tradable dates are defined with the data time zone to access the right source
             var tradableDates = Time.EachTradeableDayInTimeZone(request.ExchangeHours, startTimeLocal, endTimeLocal, request.DataTimeZone, request.IncludeExtendedMarketHours);
 
             var dataReader = new SubscriptionDataReader(config,
                 startTimeLocal,
                 endTimeLocal,
-                mapFileResolver,
+                _mapFileProvider,
                 _factorFileProvider,
                 tradableDates,
                 false,
@@ -152,7 +144,7 @@ namespace QuantConnect.Lean.Engine.HistoricalData
                 config,
                 _factorFileProvider,
                 dataReader,
-                mapFileResolver,
+                _mapFileProvider,
                 startTimeLocal);
 
             // optionally apply fill forward behavior

Engine/Results/LiveTradingResultHandler.cs

@@ -1256,7 +1256,7 @@ namespace QuantConnect.Lean.Engine.Results
 
             foreach (var kvp in Algorithm.Securities
                 // we send non internal, non canonical and tradable securities. When securities are removed they are marked as non tradable
-                .Where(pair => pair.Value.IsTradable && !pair.Value.IsInternalFeed() && !pair.Key.IsCanonical() && (!onlyInvested || pair.Value.Invested))
+                .Where(pair => pair.Value.IsTradable && !pair.Value.IsInternalFeed() && (!pair.Key.IsCanonical() || pair.Key.SecurityType == QuantConnect.SecurityType.Future) && (!onlyInvested || pair.Value.Invested))
                 .OrderBy(x => x.Key.Value))
             {
                 var security = kvp.Value;

Tests/Algorithm/AlgorithmAddDataTests.cs

@@ -191,12 +191,12 @@ namespace QuantConnect.Tests.Algorithm
 
         [TestCase("EURUSD", typeof(IndexedLinkedData), SecurityType.Cfd, false, true)]
         [TestCase("BTCUSD", typeof(IndexedLinkedData), SecurityType.Crypto, false, true)]
-        [TestCase("CL", typeof(IndexedLinkedData), SecurityType.Future, false, true)]
+        [TestCase("CL", typeof(IndexedLinkedData), SecurityType.Future, true, true)]
         [TestCase("EURUSD", typeof(IndexedLinkedData), SecurityType.Forex, false, true)]
         [TestCase("AAPL", typeof(IndexedLinkedData), SecurityType.Equity, true, true)]
         [TestCase("EURUSD", typeof(UnlinkedData), SecurityType.Cfd, false, false)]
         [TestCase("BTCUSD", typeof(UnlinkedData), SecurityType.Crypto, false, false)]
-        [TestCase("CL", typeof(UnlinkedData), SecurityType.Future, false, false)]
+        [TestCase("CL", typeof(UnlinkedData), SecurityType.Future, true, false)]
         [TestCase("AAPL", typeof(UnlinkedData), SecurityType.Equity, true, false)]
         [TestCase("EURUSD", typeof(UnlinkedData), SecurityType.Forex, false, false)]
         public void AddDataSecuritySymbolWithUnderlying(string ticker, Type customDataType, SecurityType securityType, bool securityShouldBeMapped, bool customDataShouldBeMapped)
@@ -328,7 +328,7 @@ namespace QuantConnect.Tests.Algorithm
 
         [TestCase("EURUSD", typeof(UnlinkedData), SecurityType.Cfd, false, false)]
         [TestCase("BTCUSD", typeof(UnlinkedData), SecurityType.Crypto, false, false)]
-        [TestCase("CL", typeof(UnlinkedData), SecurityType.Future, false, false)]
+        [TestCase("CL", typeof(UnlinkedData), SecurityType.Future, true, false)]
         [TestCase("AAPL", typeof(UnlinkedData), SecurityType.Equity, true, false)]
         [TestCase("EURUSD", typeof(UnlinkedData), SecurityType.Forex, false, false)]
         public void AddDataSecurityTickerNoUnderlying(string ticker, Type customDataType, SecurityType securityType, bool securityShouldBeMapped, bool customDataShouldBeMapped)

Tests/Algorithm/AlgorithmAddSecurityTests.cs

@@ -18,6 +18,7 @@ using System;
 using NUnit.Framework;
 using QuantConnect.Algorithm;
 using QuantConnect.Data.Custom.AlphaStreams;
+using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Securities.Cfd;
 using QuantConnect.Securities.Crypto;
 using QuantConnect.Securities.Equity;
@@ -33,6 +34,7 @@ namespace QuantConnect.Tests.Algorithm
     public class AlgorithmAddSecurityTests
     {
         private QCAlgorithm _algo;
+        private NullDataFeed _dataFeed;
 
         /// <summary>
         /// Instatiate a new algorithm before each test.
@@ -42,7 +44,11 @@ namespace QuantConnect.Tests.Algorithm
         public void Setup()
         {
             _algo = new QCAlgorithm();
-            _algo.SubscriptionManager.SetDataManager(new DataManagerStub(_algo));
+            _dataFeed = new NullDataFeed
+            {
+                ShouldThrow = false
+            };
+            _algo.SubscriptionManager.SetDataManager(new DataManagerStub(_dataFeed, _algo));
         }
 
         [Test, TestCaseSource(nameof(TestAddSecurityWithSymbol))]
@@ -86,9 +92,7 @@ namespace QuantConnect.Tests.Algorithm
 
             if (symbol.IsCanonical())
             {
-                // Throws NotImplementedException because we are using NullDataFeed
-                // We need to call this to add the pending universe additions
-                Assert.Throws<NotImplementedException>(() => _algo.OnEndOfTimeStep());
+                Assert.DoesNotThrow(() => _algo.OnEndOfTimeStep());
 
                 Assert.IsTrue(_algo.UniverseManager.ContainsKey(symbol));
             }

Tests/Algorithm/Framework/Execution/ImmediateExecutionModelTests.cs

@@ -46,7 +46,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((OrderTicket)null)
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
 
             var algorithm = new QCAlgorithm();
             algorithm.SetPandasConverter();
@@ -113,7 +113,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((SubmitOrderRequest request) => new OrderTicket(algorithm.Transactions, request))
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
             orderProcessor.Setup(m => m.GetOpenOrders(It.IsAny<Func<Order, bool>>()))
                 .Returns(new List<Order> { new MarketOrder(Symbols.AAPL, openOrdersQuantity, DateTime.MinValue) });
             orderProcessor.Setup(m => m.GetOpenOrderTickets(It.IsAny<Func<OrderTicket, bool>>()))
@@ -167,7 +167,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((SubmitOrderRequest request) => new OrderTicket(algorithm.Transactions, request))
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
             orderProcessor.Setup(m => m.GetOpenOrders(It.IsAny<Func<Order, bool>>()))
                 .Returns(new List<Order> { new MarketOrder(Symbols.AAPL, 100, DateTime.MinValue) });
             orderProcessor.Setup(m => m.GetOpenOrderTickets(It.IsAny<Func<OrderTicket, bool>>()))
@@ -211,7 +211,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((SubmitOrderRequest request) => new OrderTicket(algorithm.Transactions, request))
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
             algorithm.Transactions.SetOrderProcessor(orderProcessor.Object);
 
             var model = GetExecutionModel(language);

Tests/Algorithm/Framework/Execution/SpreadExecutionModelTests.cs

@@ -43,7 +43,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((OrderTicket)null)
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
 
             var algorithm = new QCAlgorithm();
             algorithm.SetPandasConverter();
@@ -97,7 +97,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((SubmitOrderRequest request) => new OrderTicket(algorithm.Transactions, request))
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
             orderProcessor.Setup(m => m.GetOpenOrders(It.IsAny<Func<Order, bool>>()))
                 .Returns(new List<Order>());
             algorithm.Transactions.SetOrderProcessor(orderProcessor.Object);
@@ -149,7 +149,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((SubmitOrderRequest request) => new OrderTicket(algorithm.Transactions, request))
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
             orderProcessor.Setup(m => m.GetOpenOrders(It.IsAny<Func<Order, bool>>()))
                 .Returns(new List<Order>());
             algorithm.Transactions.SetOrderProcessor(orderProcessor.Object);

Tests/Algorithm/Framework/Execution/StandardDeviationExecutionModelTests.cs

@@ -45,7 +45,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((OrderTicket)null)
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
 
             var algorithm = new QCAlgorithm();
             algorithm.SetPandasConverter();
@@ -109,7 +109,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((SubmitOrderRequest request) => new OrderTicket(algorithm.Transactions, request))
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
             orderProcessor.Setup(m => m.GetOpenOrders(It.IsAny<Func<Order, bool>>()))
                 .Returns(new List<Order>());
             algorithm.Transactions.SetOrderProcessor(orderProcessor.Object);

Tests/Algorithm/Framework/Execution/VolumeWeightedAveragePriceExecutionModelTests.cs

@@ -46,7 +46,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((OrderTicket)null)
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
 
             var algorithm = new QCAlgorithm();
             algorithm.SetPandasConverter();
@@ -113,7 +113,7 @@ namespace QuantConnect.Tests.Algorithm.Framework.Execution
             var orderProcessor = new Mock<IOrderProcessor>();
             orderProcessor.Setup(m => m.Process(It.IsAny<SubmitOrderRequest>()))
                 .Returns((SubmitOrderRequest request) => new OrderTicket(algorithm.Transactions, request))
-                .Callback((SubmitOrderRequest request) => actualOrdersSubmitted.Add(request));
+                .Callback((OrderRequest request) => actualOrdersSubmitted.Add((SubmitOrderRequest)request));
             orderProcessor.Setup(m => m.GetOpenOrders(It.IsAny<Func<Order, bool>>()))
                 .Returns(new List<Order>());
             algorithm.Transactions.SetOrderProcessor(orderProcessor.Object);

Tests/Algorithm/Framework/Portfolio/EqualWeightingAlphaStreamsPortfolioConstructionModelTests.cs

@@ -20,7 +20,6 @@ using QuantConnect.Data;
 using QuantConnect.Util;
 using QuantConnect.Algorithm;
 using System.Collections.Generic;
-using QuantConnect.Data.Auxiliary;
 using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Tests.Engine.DataFeeds;
@@ -35,22 +34,16 @@ namespace QuantConnect.Tests.Algorithm.Framework.Portfolio
     public class EqualWeightingAlphaStreamsPortfolioConstructionModelTests
     {
         private ZipDataCacheProvider _cacheProvider;
-        private DefaultDataProvider _dataProvider;
         private QCAlgorithm _algorithm;
 
         [SetUp]
         public virtual void SetUp()
         {
             _algorithm = new QCAlgorithm();
-            _dataProvider = new DefaultDataProvider();
-            var mapFileProvider = new LocalDiskMapFileProvider();
-            mapFileProvider.Initialize(_dataProvider);
-            var factorFileProvider = new LocalZipFactorFileProvider();
-            factorFileProvider.Initialize(mapFileProvider, _dataProvider);
             var historyProvider = new SubscriptionDataReaderHistoryProvider();
-            _cacheProvider = new ZipDataCacheProvider(_dataProvider);
+            _cacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider);
             historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null,
-                _dataProvider, _cacheProvider, mapFileProvider, factorFileProvider,
+                TestGlobals.DataProvider, _cacheProvider, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider,
                 null, true, new DataPermissionManager()));
             _algorithm.SetHistoryProvider(historyProvider);
             _algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(_algorithm));
@@ -62,7 +55,6 @@ namespace QuantConnect.Tests.Algorithm.Framework.Portfolio
         public virtual void TearDown()
         {
             _cacheProvider.DisposeSafely();
-            _dataProvider.DisposeSafely();
         }
 
         [TestCase(Language.CSharp)]

Tests/AlgorithmRunner.cs

@@ -275,7 +275,8 @@ namespace QuantConnect.Tests
             public override IEnumerable<Slice> GetHistory(IEnumerable<HistoryRequest> requests, DateTimeZone sliceTimeZone)
             {
                 requests = requests.ToList();
-                if (requests.Any(r => RegressionSetupHandlerWrapper.Algorithm.UniverseManager.ContainsKey(r.Symbol)))
+                if (requests.Any(r => RegressionSetupHandlerWrapper.Algorithm.UniverseManager.ContainsKey(r.Symbol)
+                    && (r.Symbol.SecurityType != SecurityType.Future || !r.Symbol.IsCanonical())))
                 {
                     throw new Exception("History requests should not be submitted for universe symbols");
                 }

Tests/Brokerages/InteractiveBrokers/InteractiveBrokersSymbolMapperTests.cs

@@ -16,10 +16,9 @@
 using System;
 using IBApi;
 using NUnit.Framework;
-using QuantConnect.Brokerages.InteractiveBrokers;
-using QuantConnect.Data.Auxiliary;
-using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Securities;
+using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Brokerages.InteractiveBrokers;
 using IB = QuantConnect.Brokerages.InteractiveBrokers.Client;
 
 namespace QuantConnect.Tests.Brokerages.InteractiveBrokers
@@ -71,7 +70,7 @@ namespace QuantConnect.Tests.Brokerages.InteractiveBrokers
         [TestCase("NB", "BAC")]
         public void MapCorrectBrokerageSymbol(string ticker, string ibSymbol)
         {
-            var mapper = new InteractiveBrokersSymbolMapper(new LocalDiskMapFileProvider());
+            var mapper = new InteractiveBrokersSymbolMapper(TestGlobals.MapFileProvider);
 
             var symbol = Symbol.Create(ticker, SecurityType.Equity, Market.USA);
             var brokerageSymbol = mapper.GetBrokerageSymbol(symbol);
@@ -81,7 +80,7 @@ namespace QuantConnect.Tests.Brokerages.InteractiveBrokers
         [Test]
         public void ThrowsOnNullOrEmptyOrInvalidSymbol()
         {
-            var mapper = new InteractiveBrokersSymbolMapper(new LocalDiskMapFileProvider());
+            var mapper = new InteractiveBrokersSymbolMapper(TestGlobals.MapFileProvider);
 
             Assert.Throws<ArgumentException>(() => mapper.GetLeanSymbol(null, SecurityType.Forex, Market.FXCM));
 
@@ -161,7 +160,7 @@ namespace QuantConnect.Tests.Brokerages.InteractiveBrokers
                 Currency = "USD"
             };
 
-            var mapper = new InteractiveBrokersSymbolMapper(new LocalDiskMapFileProvider());
+            var mapper = new InteractiveBrokersSymbolMapper(TestGlobals.MapFileProvider);
             var actualContract = mapper.ParseMalformedContractFutureSymbol(malformedContract, SymbolPropertiesDatabase.FromDataFolder());
 
             Assert.AreEqual(expectedContract.Symbol, actualContract.Symbol);

Tests/Common/Data/Auxiliary/FactorFileRowTests.cs

@@ -29,7 +29,7 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
         public void ToCsv()
         {
             var row = new FactorFileRow(new DateTime(2000, 01, 01), 1m, 2m, 123m);
-            var actual = row.ToCsv("source");
+            var actual = row.GetFileFormat("source");
             var expected = "20000101,1,2,123,source";
             Assert.AreEqual(expected, actual);
         }
@@ -40,7 +40,7 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
             var row = new FactorFileRow(new DateTime(2018, 08, 23), 1m, 2m, 123m);
             var dividend = new Dividend(Symbols.SPY, row.Date.AddDays(1), 1m, 123m);
             var updated = row.Apply(dividend, SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork));
-            Assert.AreEqual("20180823,0.9918699,2,123", updated.ToCsv());
+            Assert.AreEqual("20180823,0.9918699,2,123", updated.GetFileFormat());
         }
 
         [Test]
@@ -49,7 +49,7 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
             var row = new FactorFileRow(new DateTime(2018, 08, 23), 1m, 2m, 123m);
             var dividend = new Split(Symbols.SPY, row.Date.AddDays(1), 123m, 2m, SplitType.SplitOccurred);
             var updated = row.Apply(dividend, SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork));
-            Assert.AreEqual("20180823,1,4,123", updated.ToCsv());
+            Assert.AreEqual("20180823,1,4,123", updated.GetFileFormat());
         }
     }
 }

Tests/Common/Data/Auxiliary/FactorFileTests.cs

@@ -22,7 +22,6 @@ using NUnit.Framework;
 using QuantConnect.Data;
 using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Market;
-using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Logging;
 using QuantConnect.Securities;
 using QuantConnect.Util;
@@ -39,7 +38,7 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
             var Market = "usa";
             var _symbol = new Symbol(SecurityIdentifier.GenerateEquity(PermTick, Market), PermTick);
 
-            var factorFile = TestGlobals.FactorFileProvider.Get(_symbol);
+            var factorFile = TestGlobals.FactorFileProvider.Get(_symbol) as CorporateFactorProvider;
 
             Assert.AreEqual(41, factorFile.SortedFactorFileData.Count);
 
@@ -72,13 +71,12 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
                 "20501231,1.0000000,1"
             };
 
-            DateTime? factorFileMinimumDate;
-            var factorFile = FactorFileRow.Parse(lines, out factorFileMinimumDate).ToList();
+            var factorFile = PriceScalingExtensions.SafeRead("PermTick", lines, SecurityType.Equity);
 
-            Assert.AreEqual(5, factorFile.Count);
+            Assert.AreEqual(5, factorFile.Count());
 
-            Assert.IsNotNull(factorFileMinimumDate);
-            Assert.AreEqual(new DateTime(2013, 12, 04), factorFileMinimumDate.Value);
+            Assert.IsNotNull(factorFile.FactorFileMinimumDate);
+            Assert.AreEqual(new DateTime(2013, 12, 04), factorFile.FactorFileMinimumDate.Value);
         }
 
         [Test]
@@ -94,12 +92,12 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
             };
 
             DateTime? factorFileMinimumDate;
-            var factorFile = FactorFileRow.Parse(lines, out factorFileMinimumDate).ToList();
+            var factorFile = PriceScalingExtensions.SafeRead("PermTick", lines, SecurityType.Equity);
 
-            Assert.AreEqual(3, factorFile.Count);
+            Assert.AreEqual(3, factorFile.Count());
 
-            Assert.IsNotNull(factorFileMinimumDate);
-            Assert.AreEqual(new DateTime(2016, 3, 29), factorFileMinimumDate.Value);
+            Assert.IsNotNull(factorFile.FactorFileMinimumDate);
+            Assert.AreEqual(new DateTime(2016, 3, 29), factorFile.FactorFileMinimumDate.Value);
         }
 
         [Test]
@@ -109,19 +107,18 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
 
             const string symbol = "n/a";
             var file = GetTestFactorFile(symbol, reference);
-
             // time price factors should be the price factor * split factor
 
-            Assert.AreEqual(1, file.GetPriceScaleFactor(reference));
-            Assert.AreEqual(1, file.GetPriceScaleFactor(reference.AddDays(-6)));
-            Assert.AreEqual(.9, file.GetPriceScaleFactor(reference.AddDays(-7)));
-            Assert.AreEqual(.9, file.GetPriceScaleFactor(reference.AddDays(-13)));
-            Assert.AreEqual(.8, file.GetPriceScaleFactor(reference.AddDays(-14)));
-            Assert.AreEqual(.8, file.GetPriceScaleFactor(reference.AddDays(-20)));
-            Assert.AreEqual(.8m * .5m, file.GetPriceScaleFactor(reference.AddDays(-21)));
-            Assert.AreEqual(.8m * .5m, file.GetPriceScaleFactor(reference.AddDays(-22)));
-            Assert.AreEqual(.8m * .5m, file.GetPriceScaleFactor(reference.AddDays(-89)));
-            Assert.AreEqual(.8m * .25m, file.GetPriceScaleFactor(reference.AddDays(-91)));
+            Assert.AreEqual(1, file.GetPriceScaleFactor(reference, DataNormalizationMode.Adjusted));
+            Assert.AreEqual(1, file.GetPriceScaleFactor(reference.AddDays(-6), DataNormalizationMode.Adjusted));
+            Assert.AreEqual(.9, file.GetPriceScaleFactor(reference.AddDays(-7), DataNormalizationMode.Adjusted));
+            Assert.AreEqual(.9, file.GetPriceScaleFactor(reference.AddDays(-13), DataNormalizationMode.Adjusted));
+            Assert.AreEqual(.8, file.GetPriceScaleFactor(reference.AddDays(-14), DataNormalizationMode.Adjusted));
+            Assert.AreEqual(.8, file.GetPriceScaleFactor(reference.AddDays(-20), DataNormalizationMode.Adjusted));
+            Assert.AreEqual(.8m * .5m, file.GetPriceScaleFactor(reference.AddDays(-21), DataNormalizationMode.Adjusted));
+            Assert.AreEqual(.8m * .5m, file.GetPriceScaleFactor(reference.AddDays(-22), DataNormalizationMode.Adjusted));
+            Assert.AreEqual(.8m * .5m, file.GetPriceScaleFactor(reference.AddDays(-89), DataNormalizationMode.Adjusted));
+            Assert.AreEqual(.8m * .25m, file.GetPriceScaleFactor(reference.AddDays(-91), DataNormalizationMode.Adjusted));
         }
 
         [Test]
@@ -254,13 +251,15 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
 
             foreach (var item in factorFileAfterDividend.Reverse().Zip(actual.Reverse(), (a,e) => new{actual=a, expected=e}))
             {
-                Log.Trace($"expected: {item.expected} actual: {item.actual}  diff: {100* (1 - item.actual.PriceFactor/item.expected.PriceFactor):0.0000}%");
+                var expected = (FactorFileRow)item.expected;
+                var actualRow = (FactorFileRow)item.actual;
+                Log.Trace($"expected: {item.expected} actual: {item.actual}  diff: {100* (1 - actualRow.PriceFactor/expected.PriceFactor):0.0000}%");
                 Assert.AreEqual(item.expected.Date, item.actual.Date);
-                Assert.AreEqual(item.expected.ReferencePrice, item.actual.ReferencePrice);
-                Assert.AreEqual(item.expected.SplitFactor, item.actual.SplitFactor);
+                Assert.AreEqual(expected.ReferencePrice, actualRow.ReferencePrice);
+                Assert.AreEqual(expected.SplitFactor, actualRow.SplitFactor);
 
-                var delta = (double)item.expected.PriceFactor * 1e-5;
-                Assert.AreEqual((double)item.expected.PriceFactor, (double)item.actual.PriceFactor, delta);
+                var delta = (double)expected.PriceFactor * 1e-5;
+                Assert.AreEqual((double)expected.PriceFactor, (double)actualRow.PriceFactor, delta);
             }
         }
 
@@ -279,16 +278,18 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
             Assert.AreEqual(factorFileAfterSplit.Count(), actual.Count());
             Assert.True(actual.First().Date == new DateTime(1998, 01, 02),
                 $"Factor file first row changed from 1998-01-02 to {actual.First().Date:yyyy-MM-dd} after applying new event");
-            Assert.True(actual.First().SplitFactor == 25m, "Factor File split factor is not computed correctly");
+            Assert.True(((FactorFileRow)actual.First()).SplitFactor == 25m, "Factor File split factor is not computed correctly");
             foreach (var item in actual.Reverse().Zip(factorFileAfterSplit.Reverse(), (a, e) => new { actual = a, expected = e }))
             {
-                Log.Trace($"expected: {item.expected} actual: {item.actual}  diff: {100 * (1 - item.actual.PriceFactor / item.expected.PriceFactor):0.0000}%");
+                var expected = (FactorFileRow)item.expected;
+                var actualRow = (FactorFileRow)item.actual;
+                Log.Trace($"expected: {item.expected} actual: {item.actual}  diff: {100 * (1 - actualRow.PriceFactor / expected.PriceFactor):0.0000}%");
                 Assert.AreEqual(item.expected.Date, item.actual.Date);
-                Assert.AreEqual(item.expected.ReferencePrice, item.actual.ReferencePrice);
-                Assert.AreEqual(item.expected.SplitFactor, item.actual.SplitFactor);
+                Assert.AreEqual(expected.ReferencePrice, actualRow.ReferencePrice);
+                Assert.AreEqual(expected.SplitFactor, actualRow.SplitFactor);
 
-                var delta = (double)item.expected.PriceFactor * 1e-5;
-                Assert.AreEqual((double)item.expected.PriceFactor, (double)item.actual.PriceFactor, delta);
+                var delta = (double)expected.PriceFactor * 1e-5;
+                Assert.AreEqual((double)expected.PriceFactor, (double)actualRow.PriceFactor, delta);
             }
         }
 
@@ -308,16 +309,18 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
             Assert.AreEqual(factorFileAfterSplit.Count(), actual.Count());
             Assert.True(actual.First().Date == new DateTime(1998, 01, 02),
                 $"Factor file first row changed from 1998-01-02 to {actual.First().Date:yyyy-MM-dd} after applying new event");
-            Assert.True(actual.First().SplitFactor == 25m, "Factor File split factor is not computed correctly");
+            Assert.True(((FactorFileRow)actual.First()).SplitFactor == 25m, "Factor File split factor is not computed correctly");
             foreach (var item in actual.Reverse().Zip(factorFileAfterSplit.Reverse(), (a, e) => new { actual = a, expected = e }))
             {
-                Log.Trace($"expected: {item.expected} actual: {item.actual}  diff: {100 * (1 - item.actual.PriceFactor / item.expected.PriceFactor):0.0000}%");
+                var expectedRow = (FactorFileRow)item.expected;
+                var actualRow = (FactorFileRow)item.actual;
+                Log.Trace($"expected: {item.expected} actual: {item.actual}  diff: {100 * (1 - actualRow.PriceFactor / expectedRow.PriceFactor):0.0000}%");
                 Assert.AreEqual(item.expected.Date, item.actual.Date);
-                Assert.AreEqual(item.expected.ReferencePrice, item.actual.ReferencePrice);
-                Assert.AreEqual(item.expected.SplitFactor, item.actual.SplitFactor);
+                Assert.AreEqual(expectedRow.ReferencePrice, actualRow.ReferencePrice);
+                Assert.AreEqual(expectedRow.SplitFactor, actualRow.SplitFactor);
 
-                var delta = (double)item.expected.PriceFactor * 1e-5;
-                Assert.AreEqual((double)item.expected.PriceFactor, (double)item.actual.PriceFactor, delta);
+                var delta = (double)expectedRow.PriceFactor * 1e-5;
+                Assert.AreEqual((double)expectedRow.PriceFactor, (double)actualRow.PriceFactor, delta);
             }
         }
 
@@ -330,7 +333,7 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
             var expected = GetTestFactorFile("AAPL", reference);
 
             // remove the last entry that contains a split and dividend at the same time
-            var factorFile = new FactorFile("AAPL", expected.SortedFactorFileData.Where(kvp => kvp.Value.PriceFactor >= .8m).Select(kvp => kvp.Value));
+            var factorFile = new CorporateFactorProvider("AAPL", expected.SortedFactorFileData.Where(kvp => kvp.Value.Single().PriceFactor >= .8m).Select(kvp => kvp.Value.Single()));
             var actual = factorFile.Apply(new List<BaseData>
             {
                 new Split(Symbols.AAPL, reference.AddDays(-364), 100m, 1 / 2m, SplitType.SplitOccurred),
@@ -339,12 +342,14 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
 
             foreach (var item in actual.Reverse().Zip(expected.Reverse(), (a, e) => new {actual = a, expected = e}))
             {
-                Log.Trace($"expected: {item.expected} actual: {item.actual}  diff: {100 * (1 - item.actual.PriceFactor / item.expected.PriceFactor):0.0000}%");
+                var expectedRow = (FactorFileRow)item.expected;
+                var actualRow = (FactorFileRow)item.actual;
+                Log.Trace($"expected: {item.expected} actual: {item.actual}  diff: {100 * (1 - actualRow.PriceFactor / expectedRow.PriceFactor):0.0000}%");
                 Assert.AreEqual(item.expected.Date, item.actual.Date);
-                Assert.AreEqual(item.expected.ReferencePrice, item.actual.ReferencePrice);
-                Assert.AreEqual(item.expected.SplitFactor, item.actual.SplitFactor);
+                Assert.AreEqual(expectedRow.ReferencePrice, actualRow.ReferencePrice);
+                Assert.AreEqual(expectedRow.SplitFactor, actualRow.SplitFactor);
 
-                Assert.AreEqual(item.expected.PriceFactor.RoundToSignificantDigits(4), item.actual.PriceFactor.RoundToSignificantDigits(4));
+                Assert.AreEqual(expectedRow.PriceFactor.RoundToSignificantDigits(4), actualRow.PriceFactor.RoundToSignificantDigits(4));
             }
         }
 
@@ -358,19 +363,19 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
                 "20501231,1.0000000,1"
             };
 
-            var factorFile = FactorFile.Parse("bno", lines);
+            var factorFile = PriceScalingExtensions.SafeRead("bno", lines, SecurityType.Equity) as CorporateFactorProvider;
 
-            var firstRow = factorFile.SortedFactorFileData[new DateTime(1998, 01, 02)];
+            var firstRow = factorFile.SortedFactorFileData[new DateTime(1998, 01, 02)].Single();
             Assert.AreEqual(1m, firstRow.PriceFactor);
             Assert.AreEqual(0.5m, firstRow.SplitFactor);
             Assert.AreEqual(0m, firstRow.ReferencePrice);
 
-            var secondRow = factorFile.SortedFactorFileData[new DateTime(2013, 08, 28)];
+            var secondRow = factorFile.SortedFactorFileData[new DateTime(2013, 08, 28)].Single();
             Assert.AreEqual(1m, secondRow.PriceFactor);
             Assert.AreEqual(0.5m, secondRow.SplitFactor);
             Assert.AreEqual(0m, firstRow.ReferencePrice);
 
-            var thirdRow = factorFile.SortedFactorFileData[Time.EndOfTime];
+            var thirdRow = factorFile.SortedFactorFileData[Time.EndOfTime].Single();
             Assert.AreEqual(1m, thirdRow.PriceFactor);
             Assert.AreEqual(1m, thirdRow.SplitFactor);
             Assert.AreEqual(0m, firstRow.ReferencePrice);
@@ -386,7 +391,7 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
                 "20501231,1.0000000,1"
             };
 
-            var factorFile = FactorFile.Parse("bno", lines);
+            var factorFile = PriceScalingExtensions.SafeRead("bno", lines, SecurityType.Equity) as CorporateFactorProvider;
             Assert.AreEqual(new DateTime(2013, 08, 28), factorFile.MostRecentFactorChange);
         }
 
@@ -397,13 +402,13 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
         {
             var lines = contents.Split('\n').Where(l => !string.IsNullOrWhiteSpace(l));
 
-            var factorFile = FactorFile.Parse("bno", lines);
+            var factorFile = PriceScalingExtensions.SafeRead("bno", lines, SecurityType.Equity) as CorporateFactorProvider;
             Assert.IsEmpty(factorFile.GetSplitsAndDividends(Symbols.SPY, SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork)));
         }
 
-        private static FactorFile GetTestFactorFile(string symbol, DateTime reference)
+        private static CorporateFactorProvider GetTestFactorFile(string symbol, DateTime reference)
         {
-            var file = new FactorFile(symbol, new List<FactorFileRow>
+            var file = new CorporateFactorProvider(symbol, new List<FactorFileRow>
             {
                 new FactorFileRow(reference, 1, 1),
                 new FactorFileRow(reference.AddDays(-7), .9m, 1, 100m),       // dividend
@@ -415,12 +420,12 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
             return file;
         }
 
-        private static FactorFile GetFactorFile(string permtick)
+        private static IFactorProvider GetFactorFile(string permtick)
         {
             return TestGlobals.FactorFileProvider.Get(permtick);
         }
 
-        private static FactorFile GetFactorFile_LODE20191127()
+        private static CorporateFactorProvider GetFactorFile_LODE20191127()
         {
             const string factorFileContents = @"
 19980102,1,5,8.5,qq
@@ -431,10 +436,10 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
             var reader = new StreamReader(factorFileContents.ToStream());
             var enumerable = new StreamReaderEnumerable(reader).Where(line => line.Length > 0);
             var factorFileRows = FactorFileRow.Parse(enumerable, out factorFileMinimumDate);
-            return new FactorFile("lode", factorFileRows, factorFileMinimumDate);
+            return new CorporateFactorProvider("lode", factorFileRows, factorFileMinimumDate);
         }
 
-        private static FactorFile GetFactorFile_LODE20191129()
+        private static CorporateFactorProvider GetFactorFile_LODE20191129()
         {
             const string factorFileContents = @"
 19980102,1,25,8.5,qq
@@ -446,10 +451,10 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
             var reader = new StreamReader(factorFileContents.ToStream());
             var enumerable = new StreamReaderEnumerable(reader).Where(line => line.Length > 0);
             var factorFileRows = FactorFileRow.Parse(enumerable, out factorFileMinimumDate);
-            return new FactorFile("lode", factorFileRows, factorFileMinimumDate);
+            return new CorporateFactorProvider("lode", factorFileRows, factorFileMinimumDate);
         }
 
-        private static FactorFile GetFactorFile_AAPL2018_05_11()
+        private static CorporateFactorProvider GetFactorFile_AAPL2018_05_11()
         {
             const string factorFileContents = @"
 19980102,0.8893653,0.0357143,16.25
@@ -487,11 +492,11 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
             var reader = new StreamReader(factorFileContents.ToStream());
             var enumerable = new StreamReaderEnumerable(reader).Where(line => line.Length > 0);
             var factorFileRows = FactorFileRow.Parse(enumerable, out factorFileMinimumDate);
-            return new FactorFile("aapl", factorFileRows, factorFileMinimumDate);
+            return new CorporateFactorProvider("aapl", factorFileRows, factorFileMinimumDate);
         }
 
         // AAPL experiences a 0.73 dividend distribution on 2018.05.11
-        private static FactorFile GetFactorFile_AAPL2018_05_08()
+        private static CorporateFactorProvider GetFactorFile_AAPL2018_05_08()
         {
             const string factorFileContents = @"
 19980102,0.8927948,0.0357143,16.25
@@ -528,7 +533,7 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
             var reader = new StreamReader(factorFileContents.ToStream());
             var enumerable = new StreamReaderEnumerable(reader).Where(line => line.Length > 0);
             var factorFileRows = FactorFileRow.Parse(enumerable, out factorFileMinimumDate);
-            return new FactorFile("aapl", factorFileRows, factorFileMinimumDate);
+            return new CorporateFactorProvider("aapl", factorFileRows, factorFileMinimumDate);
         }
     }
 }

Tests/Common/Data/Auxiliary/LocalDiskFactorFileProviderTests.cs

@@ -18,6 +18,7 @@ using System;
 using System.Globalization;
 using System.IO;
 using NUnit.Framework;
+using QuantConnect.Data.Auxiliary;
 using QuantConnect.Interfaces;
 
 namespace QuantConnect.Tests.Common.Data.Auxiliary
@@ -51,8 +52,9 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
         [Test]
         public void ReturnsNullForNotFound()
         {
-            var factorFile = FactorFileProvider.Get(Symbol.Create("not-a-ticker", SecurityType.Equity, QuantConnect.Market.USA));
-            Assert.IsNull(factorFile);
+            var factorFile = FactorFileProvider.Get(Symbol.Create("not-a-ticker", SecurityType.Equity, QuantConnect.Market.USA)) as CorporateFactorProvider;
+            Assert.IsNotNull(factorFile);
+            Assert.IsEmpty(factorFile);
         }
 
         [Test, Ignore("This test is meant to be run manually")]

Tests/Common/Data/Auxiliary/LocalDiskMapFileProviderTests.cs

@@ -26,7 +26,8 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
         public void RetrievesFromDisk()
         {
             var provider = new LocalDiskMapFileProvider();
-            var mapFiles = provider.Get(QuantConnect.Market.USA);
+            provider.Initialize(TestGlobals.DataProvider);
+            var mapFiles = provider.Get(CorporateActionsKey.EquityUsa);
             Assert.IsNotEmpty(mapFiles);
         }
 
@@ -34,8 +35,9 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
         public void CachesValueAndReturnsSameReference()
         {
             var provider = new LocalDiskMapFileProvider();
-            var mapFiles1 = provider.Get(QuantConnect.Market.USA);
-            var mapFiles2 = provider.Get(QuantConnect.Market.USA);
+            provider.Initialize(TestGlobals.DataProvider);
+            var mapFiles1 = provider.Get(CorporateActionsKey.EquityUsa);
+            var mapFiles2 = provider.Get(CorporateActionsKey.EquityUsa);
             Assert.IsTrue(ReferenceEquals(mapFiles1, mapFiles2));
         }
     }

Tests/Common/Data/Auxiliary/LocalZipMapFileProviderTests.cs

@@ -59,7 +59,7 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
             var dataProviderTest = new DefaultDataProviderTest();
             fileProviderTest.Initialize(dataProviderTest);
 
-            var mapFileResolver = fileProviderTest.Get(QuantConnect.Market.USA);
+            var mapFileResolver = fileProviderTest.Get(CorporateActionsKey.EquityUsa);
 
             fileProviderTest.Enabled = false;
             dataProviderTest.DisposeSafely();
@@ -74,15 +74,15 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
             var dataProviderTest = new DefaultDataProviderTest();
             fileProviderTest.Initialize(dataProviderTest);
 
-            fileProviderTest.Get(QuantConnect.Market.USA);
+            fileProviderTest.Get(CorporateActionsKey.EquityUsa);
             Assert.AreEqual(1, dataProviderTest.FetchCount);
             Thread.Sleep(50);
-            fileProviderTest.Get(QuantConnect.Market.USA);
+            fileProviderTest.Get(CorporateActionsKey.EquityUsa);
             Assert.AreEqual(1, dataProviderTest.FetchCount);
 
             Thread.Sleep(1000);
 
-            fileProviderTest.Get(QuantConnect.Market.USA);
+            fileProviderTest.Get(CorporateActionsKey.EquityUsa);
             Assert.AreEqual(2, dataProviderTest.FetchCount);
 
             fileProviderTest.Enabled = false;

Tests/Common/Data/Auxiliary/MapFileTests.cs

@@ -106,7 +106,7 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
                 rowParts[1],
                 rowParts[2]);
             // Act
-            var actualMapFileRow = MapFileRow.Parse(mapFileRow, QuantConnect.Market.USA);
+            var actualMapFileRow = MapFileRow.Parse(mapFileRow, QuantConnect.Market.USA, SecurityType.Equity);
             // Assert
             Assert.AreEqual(expectedExchange, actualMapFileRow.PrimaryExchange);
             Assert.AreEqual(expectedMapFileRow, actualMapFileRow);
@@ -122,7 +122,7 @@ namespace QuantConnect.Tests.Common.Data.Auxiliary
                 DateTime.ParseExact(rowParts[0], DateFormat.EightCharacter, CultureInfo.InvariantCulture),
                 rowParts[1]);
             // Act
-            var actualMapFileRow = MapFileRow.Parse(mapFileRow, QuantConnect.Market.USA);
+            var actualMapFileRow = MapFileRow.Parse(mapFileRow, QuantConnect.Market.USA, SecurityType.Equity);
             // Assert
             Assert.AreEqual(Exchange.UNKNOWN, actualMapFileRow.PrimaryExchange);
             Assert.AreEqual(expectedMapFileRow, actualMapFileRow);

Tests/Common/Securities/SubscriptionDataConfigTests.cs

@@ -55,5 +55,17 @@ namespace QuantConnect.Tests.Common.Securities
                 Assert.DoesNotThrow(() => config.Consolidators.Remove(consolidator));
             }
         }
+
+        [TestCase(1, 0, DataMappingMode.OpenInterest, DataMappingMode.OpenInterest)]
+        [TestCase(0, 0, DataMappingMode.OpenInterest, DataMappingMode.FirstDayMonth)]
+        public void NotEqualsMappingAndOffset(int offsetA, int offsetB, DataMappingMode mappingModeA, DataMappingMode mappingModeB)
+        {
+            var configA = new SubscriptionDataConfig(typeof(TradeBar), Symbols.SPY, Resolution.Minute, TimeZones.NewYork,
+                TimeZones.NewYork, false, false, false, dataMappingMode: mappingModeA, contractDepthOffset: (uint)offsetA);
+            var configB = new SubscriptionDataConfig(configA, dataMappingMode: mappingModeB, contractDepthOffset: (uint)offsetB);
+
+            Assert.AreNotEqual(configA, configB);
+            Assert.AreNotEqual(configA.GetHashCode(), configB.GetHashCode());
+        }
     }
-}
+}