Update moq test package

- Add support for live mapping, refreshing mapfiles
- Fix future expiration functions
- Adding unit tests

Algorithm.CSharp/AddBetaIndicatorRegressionAlgorithm.cs

@@ -1,151 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using System.Collections.Generic;
-using QuantConnect.Data;
-using QuantConnect.Indicators;
-using QuantConnect.Interfaces;
-using QuantConnect.Orders;
-
-namespace QuantConnect.Algorithm.CSharp
-{
-    /// <summary>
-    /// Regression test to explain how Beta indicator works
-    /// </summary>
-    public class AddBetaIndicatorRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
-    {
-        private BetaIndicator _beta;
-        private SimpleMovingAverage _sma;
-        private decimal _lastSMAValue;
-
-        public override void Initialize()
-        {
-            SetStartDate(2013, 10, 07);
-            SetEndDate(2013, 10, 15);
-            SetCash(10000);
-
-            AddEquity("IBM");
-            AddEquity("SPY");
-
-            EnableAutomaticIndicatorWarmUp = true;
-            _beta = B("IBM", "SPY", 3, Resolution.Daily);
-            _sma = SMA("SPY", 3, Resolution.Daily);
-            _lastSMAValue = 0;
-
-            if (!_beta.IsReady)
-            {
-                throw new Exception("_beta indicator was expected to be ready");
-            }
-        }
-
-        public override void OnData(Slice data)
-        {
-            if (!Portfolio.Invested)
-            {
-                var price = data["IBM"].Close;
-                Buy("IBM", 10);
-                LimitOrder("IBM", 10, price * 0.1m);
-                StopMarketOrder("IBM", 10, price / 0.1m);
-            }
-            
-            if (_beta.Current.Value < 0m || _beta.Current.Value > 2.80m)
-            {
-                throw new Exception($"_beta value was expected to be between 0 and 2.80 but was {_beta.Current.Value}");
-            }
-
-            Log($"Beta between IBM and SPY is: {_beta.Current.Value}");
-        }
-
-        public override void OnOrderEvent(OrderEvent orderEvent)
-        {
-            var order = Transactions.GetOrderById(orderEvent.OrderId);
-            var goUpwards = _lastSMAValue < _sma.Current.Value;
-            _lastSMAValue = _sma.Current.Value;
-
-            if (order.Status == OrderStatus.Filled)
-            {
-                if (order.Type == OrderType.Limit && Math.Abs(_beta.Current.Value - 1) < 0.2m && goUpwards)
-                {
-                    Transactions.CancelOpenOrders(order.Symbol);
-                }
-            }
-
-            if (order.Status == OrderStatus.Canceled)
-            {
-                Log(orderEvent.ToString());
-            }
-        }
-
-        /// <summary>
-        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
-        /// </summary>
-        public bool CanRunLocally { get; } = true;
-
-        /// <summary>
-        /// This is used by the regression test system to indicate which languages this algorithm is written in.
-        /// </summary>
-        public virtual Language[] Languages { get; } = { Language.CSharp};
-
-        /// <summary>
-        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
-        /// </summary>
-        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
-        {
-            {"Total Trades", "1"},
-            {"Average Win", "0%"},
-            {"Average Loss", "0%"},
-            {"Compounding Annual Return", "12.939%"},
-            {"Drawdown", "0.300%"},
-            {"Expectancy", "0"},
-            {"Net Profit", "0.289%"},
-            {"Sharpe Ratio", "4.233"},
-            {"Probabilistic Sharpe Ratio", "68.349%"},
-            {"Loss Rate", "0%"},
-            {"Win Rate", "0%"},
-            {"Profit-Loss Ratio", "0"},
-            {"Alpha", "0.035"},
-            {"Beta", "0.122"},
-            {"Annual Standard Deviation", "0.024"},
-            {"Annual Variance", "0.001"},
-            {"Information Ratio", "-3.181"},
-            {"Tracking Error", "0.142"},
-            {"Treynor Ratio", "0.842"},
-            {"Total Fees", "$1.00"},
-            {"Estimated Strategy Capacity", "$35000000.00"},
-            {"Lowest Capacity Asset", "IBM R735QTJ8XC9X"},
-            {"Fitness Score", "0.022"},
-            {"Kelly Criterion Estimate", "0"},
-            {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "8.508"},
-            {"Return Over Maximum Drawdown", "58.894"},
-            {"Portfolio Turnover", "0.022"},
-            {"Total Insights Generated", "0"},
-            {"Total Insights Closed", "0"},
-            {"Total Insights Analysis Completed", "0"},
-            {"Long Insight Count", "0"},
-            {"Short Insight Count", "0"},
-            {"Long/Short Ratio", "100%"},
-            {"Estimated Monthly Alpha Value", "$0"},
-            {"Total Accumulated Estimated Alpha Value", "$0"},
-            {"Mean Population Estimated Insight Value", "$0"},
-            {"Mean Population Direction", "0%"},
-            {"Mean Population Magnitude", "0%"},
-            {"Rolling Averaged Population Direction", "0%"},
-            {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "bd88c6a0e10c7e146b05377205101a12"}
-        };
-    }
-}

Algorithm.CSharp/AddOptionContractExpiresRegressionAlgorithm.cs

@@ -139,8 +139,8 @@ namespace QuantConnect.Algorithm.CSharp
             {"Tracking Error", "0.049"},
             {"Treynor Ratio", "1.372"},
             {"Total Fees", "$2.00"},
-            {"Estimated Strategy Capacity", "$67000000.00"},
-            {"Lowest Capacity Asset", "AOL VRKS95ENLBYE|AOL R735QTJ8XC9X"},
+            {"Estimated Strategy Capacity", "$45000000.00"},
+            {"Lowest Capacity Asset", "AOL R735QTJ8XC9X"},
             {"Fitness Score", "0"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
@@ -160,7 +160,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "4f50b8360ea317ef974801649088bd06"}
+            {"OrderListHash", "b006bb7864c0b2f1a6552fb2aa7f03b8"}
         };
     }
 }

Algorithm.CSharp/AdjustedVolumeRegressionAlgorithm.cs

@@ -30,7 +30,7 @@ namespace QuantConnect.Algorithm.CSharp
     {
         private Symbol _aapl;
         private const string Ticker = "AAPL";
-        private CorporateFactorProvider _factorFile;
+        private FactorFile _factorFile;
         private readonly IEnumerator<decimal> _expectedAdjustedVolume = new List<decimal> { 6164842, 3044047, 3680347, 3468303, 2169943, 2652523,
             1499707, 1518215, 1655219, 1510487 }.GetEnumerator();
         private readonly IEnumerator<decimal> _expectedAdjustedAskSize = new List<decimal> { 215600, 5600, 25200, 8400, 5600, 5600, 2800,
@@ -56,7 +56,7 @@ namespace QuantConnect.Algorithm.CSharp
             factorFileProvider.Initialize(mapFileProvider, dataProvider);
 
 
-            _factorFile = factorFileProvider.Get(_aapl) as CorporateFactorProvider;
+            _factorFile = factorFileProvider.Get(_aapl);
         }
 
         /// <summary>
@@ -83,7 +83,7 @@ namespace QuantConnect.Algorithm.CSharp
                 if (_expectedAdjustedVolume.MoveNext() && _expectedAdjustedVolume.Current != aaplData.Volume)
                 {
                     // Our values don't match lets try and give a reason why
-                    var dayFactor = _factorFile.GetPriceScale(aaplData.Time, DataNormalizationMode.SplitAdjusted);
+                    var dayFactor = _factorFile.GetSplitFactor(aaplData.Time);
                     var probableAdjustedVolume = aaplData.Volume / dayFactor;
 
                     if (_expectedAdjustedVolume.Current == probableAdjustedVolume)
@@ -107,7 +107,7 @@ namespace QuantConnect.Algorithm.CSharp
                 if (_expectedAdjustedAskSize.MoveNext() && _expectedAdjustedAskSize.Current != aaplQuoteData.LastAskSize)
                 {
                     // Our values don't match lets try and give a reason why
-                    var dayFactor = _factorFile.GetPriceScale(aaplQuoteData.Time, DataNormalizationMode.SplitAdjusted);
+                    var dayFactor = _factorFile.GetSplitFactor(aaplQuoteData.Time);
                     var probableAdjustedAskSize = aaplQuoteData.LastAskSize / dayFactor;
 
                     if (_expectedAdjustedAskSize.Current == probableAdjustedAskSize)
@@ -126,7 +126,7 @@ namespace QuantConnect.Algorithm.CSharp
                 if (_expectedAdjustedBidSize.MoveNext() && _expectedAdjustedBidSize.Current != aaplQuoteData.LastBidSize)
                 {
                     // Our values don't match lets try and give a reason why
-                    var dayFactor = _factorFile.GetPriceScale(aaplQuoteData.Time, DataNormalizationMode.SplitAdjusted);
+                    var dayFactor = _factorFile.GetSplitFactor(aaplQuoteData.Time);
                     var probableAdjustedBidSize = aaplQuoteData.LastBidSize / dayFactor;
 
                     if (_expectedAdjustedBidSize.Current == probableAdjustedBidSize)

Algorithm.CSharp/AlphaStreamsBasicTemplateAlgorithm.cs

@@ -13,12 +13,15 @@
  * limitations under the License.
 */
 
+using System.Linq;
+using QuantConnect.Data;
 using QuantConnect.Orders;
 using QuantConnect.Interfaces;
+using QuantConnect.Brokerages;
 using QuantConnect.Securities;
 using System.Collections.Generic;
+using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Data.Custom.AlphaStreams;
-using QuantConnect.Algorithm.Framework.Alphas;
 using QuantConnect.Algorithm.Framework.Execution;
 using QuantConnect.Algorithm.Framework.Portfolio;
 
@@ -29,6 +32,8 @@ namespace QuantConnect.Algorithm.CSharp
     /// </summary>
     public class AlphaStreamsBasicTemplateAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
+        private Dictionary<Symbol, HashSet<Symbol>> _symbolsPerAlpha = new Dictionary<Symbol, HashSet<Symbol>>();
+
         /// <summary>
         /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
         /// </summary>
@@ -37,10 +42,10 @@ namespace QuantConnect.Algorithm.CSharp
             SetStartDate(2018, 04, 04);
             SetEndDate(2018, 04, 06);
 
-            SetAlpha(new AlphaStreamAlphaModule());
             SetExecution(new ImmediateExecutionModel());
             Settings.MinimumOrderMarginPortfolioPercentage = 0.01m;
             SetPortfolioConstruction(new EqualWeightingAlphaStreamsPortfolioConstructionModel());
+
             SetSecurityInitializer(new BrokerageModelSecurityInitializer(BrokerageModel,
                 new FuncSecuritySeeder(GetLastKnownPrices)));
 
@@ -50,11 +55,77 @@ namespace QuantConnect.Algorithm.CSharp
             }
         }
 
+        /// <summary>
+        /// OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
+        /// </summary>
+        /// <param name="data">Slice object keyed by symbol containing the stock data</param>
+        public override void OnData(Slice data)
+        {
+            foreach (var portfolioState in data.Get<AlphaStreamsPortfolioState>().Values)
+            {
+                ProcessPortfolioState(portfolioState);
+            }
+        }
+
         public override void OnOrderEvent(OrderEvent orderEvent)
         {
             Log($"OnOrderEvent: {orderEvent}");
         }
 
+        public override void OnSecuritiesChanged(SecurityChanges changes)
+        {
+            changes.FilterCustomSecurities = false;
+            foreach (var addedSecurity in changes.AddedSecurities)
+            {
+                if (addedSecurity.Symbol.IsCustomDataType<AlphaStreamsPortfolioState>())
+                {
+                    if (!_symbolsPerAlpha.ContainsKey(addedSecurity.Symbol))
+                    {
+                        _symbolsPerAlpha[addedSecurity.Symbol] = new HashSet<Symbol>();
+                    }
+                    // warmup alpha state, adding target securities
+                    ProcessPortfolioState(addedSecurity.Cache.GetData<AlphaStreamsPortfolioState>());
+                }
+            }
+
+            Log($"OnSecuritiesChanged: {changes}");
+        }
+
+        private bool UsedBySomeAlpha(Symbol asset)
+        {
+            return _symbolsPerAlpha.Any(pair => pair.Value.Contains(asset));
+        }
+
+        private void ProcessPortfolioState(AlphaStreamsPortfolioState portfolioState)
+        {
+            if (portfolioState == null)
+            {
+                return;
+            }
+
+            var alphaId = portfolioState.Symbol;
+            if (!_symbolsPerAlpha.TryGetValue(alphaId, out var currentSymbols))
+            {
+                _symbolsPerAlpha[alphaId] = currentSymbols = new HashSet<Symbol>();
+            }
+
+            var newSymbols = new HashSet<Symbol>(currentSymbols.Count);
+            foreach (var symbol in portfolioState.PositionGroups?.SelectMany(positionGroup => positionGroup.Positions).Select(state => state.Symbol) ?? Enumerable.Empty<Symbol>())
+            {
+                // only add it if it's not used by any alpha (already added check)
+                if (newSymbols.Add(symbol) && !UsedBySomeAlpha(symbol))
+                {
+                    AddSecurity(symbol, resolution: UniverseSettings.Resolution, extendedMarketHours: UniverseSettings.ExtendedMarketHours);
+                }
+            }
+            _symbolsPerAlpha[alphaId] = newSymbols;
+
+            foreach (var symbol in currentSymbols.Where(symbol => !UsedBySomeAlpha(symbol)))
+            {
+                RemoveSecurity(symbol);
+            }
+        }
+
         /// <summary>
         /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
         /// </summary>

Algorithm.CSharp/AlphaStreamsUniverseSelectionTemplateAlgorithm.cs

@@ -14,11 +14,9 @@
 */
 
 using System;
-using QuantConnect.Interfaces;
 using System.Collections.Generic;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Data.Custom.AlphaStreams;
-using QuantConnect.Algorithm.Framework.Alphas;
 using QuantConnect.Algorithm.Framework.Execution;
 using QuantConnect.Algorithm.Framework.Portfolio;
 using QuantConnect.Algorithm.Framework.Selection;
@@ -28,7 +26,7 @@ namespace QuantConnect.Algorithm.CSharp
     /// <summary>
     /// Example algorithm consuming an alpha streams portfolio state and trading based on it
     /// </summary>
-    public class AlphaStreamsUniverseSelectionTemplateAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    public class AlphaStreamsUniverseSelectionTemplateAlgorithm : AlphaStreamsBasicTemplateAlgorithm
     {
         /// <summary>
         /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
@@ -38,7 +36,6 @@ namespace QuantConnect.Algorithm.CSharp
             SetStartDate(2018, 04, 04);
             SetEndDate(2018, 04, 06);
 
-            SetAlpha(new AlphaStreamAlphaModule());
             SetExecution(new ImmediateExecutionModel());
             Settings.MinimumOrderMarginPortfolioPercentage = 0.01m;
             SetPortfolioConstruction(new EqualWeightingAlphaStreamsPortfolioConstructionModel());
@@ -68,20 +65,10 @@ namespace QuantConnect.Algorithm.CSharp
             }
         }
 
-        /// <summary>
-        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
-        /// </summary>
-        public bool CanRunLocally { get; } = true;
-
-        /// <summary>
-        /// This is used by the regression test system to indicate which languages this algorithm is written in.
-        /// </summary>
-        public Language[] Languages { get; } = { Language.CSharp };
-
         /// <summary>
         /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
         /// </summary>
-        public virtual Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        public override Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
             {"Total Trades", "2"},
             {"Average Win", "0%"},

Algorithm.CSharp/BacktestingBrokerageRegressionAlgorithm.cs

@@ -339,7 +339,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "f67306bc706a2cf66288f1cadf6148ed"}
+            {"OrderListHash", "7f99e1a8ce4675a1e8bbe1ba45967ccd"}
         };
     }
 }

Algorithm.CSharp/BasicTemplateIndexAlgorithm.cs

@@ -41,7 +41,7 @@ namespace QuantConnect.Algorithm.CSharp
         public override void Initialize()
         {
             SetStartDate(2021, 1, 4);
-            SetEndDate(2021, 1, 18);
+            SetEndDate(2021, 1, 15);
             SetCash(1000000);
 
             // Use indicator for signal; but it cannot be traded
@@ -114,31 +114,31 @@ namespace QuantConnect.Algorithm.CSharp
             {"Total Trades", "4"},
             {"Average Win", "0%"},
             {"Average Loss", "-53.10%"},
-            {"Compounding Annual Return", "-92.544%"},
+            {"Compounding Annual Return", "-96.172%"},
             {"Drawdown", "10.100%"},
             {"Expectancy", "-1"},
             {"Net Profit", "-9.915%"},
-            {"Sharpe Ratio", "-3.845"},
-            {"Probabilistic Sharpe Ratio", "0.053%"},
+            {"Sharpe Ratio", "-4.068"},
+            {"Probabilistic Sharpe Ratio", "0.055%"},
             {"Loss Rate", "100%"},
             {"Win Rate", "0%"},
             {"Profit-Loss Ratio", "0"},
-            {"Alpha", "-0.558"},
-            {"Beta", "0.313"},
-            {"Annual Standard Deviation", "0.112"},
-            {"Annual Variance", "0.013"},
-            {"Information Ratio", "-6.652"},
-            {"Tracking Error", "0.125"},
-            {"Treynor Ratio", "-1.379"},
+            {"Alpha", "-0.745"},
+            {"Beta", "0.432"},
+            {"Annual Standard Deviation", "0.126"},
+            {"Annual Variance", "0.016"},
+            {"Information Ratio", "-7.972"},
+            {"Tracking Error", "0.132"},
+            {"Treynor Ratio", "-1.189"},
             {"Total Fees", "$0.00"},
-            {"Estimated Strategy Capacity", "$13000000.00"},
+            {"Estimated Strategy Capacity", "$14000000.00"},
             {"Lowest Capacity Asset", "SPX XL80P3GHDZXQ|SPX 31"},
-            {"Fitness Score", "0.039"},
+            {"Fitness Score", "0.044"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "-1.763"},
-            {"Return Over Maximum Drawdown", "-9.371"},
-            {"Portfolio Turnover", "0.278"},
+            {"Sortino Ratio", "-1.96"},
+            {"Return Over Maximum Drawdown", "-10.171"},
+            {"Portfolio Turnover", "0.34"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
             {"Total Insights Analysis Completed", "0"},
@@ -152,7 +152,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "0668385036aba3e95127607dfc2f1a59"}
+            {"OrderListHash", "52521ab779446daf4d38a7c9bbbdd893"}
         };
     }
 }

Algorithm.CSharp/BasicTemplateOptionsDailyAlgorithm.cs

@@ -1,171 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
-*/
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using QuantConnect.Data;
-using QuantConnect.Data.Market;
-using QuantConnect.Orders;
-using QuantConnect.Interfaces;
-
-namespace QuantConnect.Algorithm.CSharp
-{
-    /// <summary>
-    /// This example demonstrates how to add options for a given underlying equity security.
-    /// It also shows how you can prefilter contracts easily based on strikes and expirations, and how you
-    /// can inspect the option chain to pick a specific option contract to trade.
-    /// </summary>
-    /// <meta name="tag" content="using data" />
-    /// <meta name="tag" content="options" />
-    /// <meta name="tag" content="filter selection" />
-    public class BasicTemplateOptionsDailyAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
-    {
-        private const string UnderlyingTicker = "GOOG";
-        public Symbol OptionSymbol;
-        private bool _optionExpired;
-
-        public override void Initialize()
-        {
-            SetStartDate(2015, 12, 23);
-            SetEndDate(2016, 1, 20);
-            SetCash(100000);
-
-            var equity = AddEquity(UnderlyingTicker, Resolution.Daily);
-            var option = AddOption(UnderlyingTicker, Resolution.Daily);
-            OptionSymbol = option.Symbol;
-
-            option.SetFilter(x => x.CallsOnly().Strikes(0, 1).Expiration(0, 30));
-
-            // use the underlying equity as the benchmark
-            SetBenchmark(equity.Symbol);
-        }
-
-        /// <summary>
-        /// Event - v3.0 DATA EVENT HANDLER: (Pattern) Basic template for user to override for receiving all subscription data in a single event
-        /// </summary>
-        /// <param name="slice">The current slice of data keyed by symbol string</param>
-        public override void OnData(Slice slice)
-        {
-            if (!Portfolio.Invested)
-            {
-                OptionChain chain;
-                if (slice.OptionChains.TryGetValue(OptionSymbol, out chain))
-                {
-                    // Grab us the contract nearest expiry that is not today
-                    var contractsByExpiration = chain.Where(x => x.Expiry != Time.Date).OrderBy(x => x.Expiry);
-                    var contract = contractsByExpiration.FirstOrDefault();
-
-                    if (contract != null)
-                    {
-                        // if found, trade it
-                        MarketOrder(contract.Symbol, 1);
-                    }
-                }
-            }
-        }
-
-        /// <summary>
-        /// Order fill event handler. On an order fill update the resulting information is passed to this method.
-        /// </summary>
-        /// <param name="orderEvent">Order event details containing details of the evemts</param>
-        /// <remarks>This method can be called asynchronously and so should only be used by seasoned C# experts. Ensure you use proper locks on thread-unsafe objects</remarks>
-        public override void OnOrderEvent(OrderEvent orderEvent)
-        {
-            Log(orderEvent.ToString());
-
-            // Check for our expected OTM option expiry
-            if (orderEvent.Message == "OTM")
-            {
-                // Assert it is at midnight (5AM UTC)
-                if (orderEvent.UtcTime != new DateTime(2016, 1, 16, 5, 0, 0))
-                {
-                    throw new ArgumentException($"Expiry event was not at the correct time, {orderEvent.UtcTime}");
-                }
-
-                _optionExpired = true;
-            }
-        }
-
-        public override void OnEndOfAlgorithm()
-        {
-            // Assert we had our option expire and fill a liquidation order
-            if (_optionExpired != true)
-            {
-                throw new ArgumentException("Algorithm did not process the option expiration like expected");
-            }
-        }
-
-        /// <summary>
-        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
-        /// </summary>
-        public bool CanRunLocally { get; } = true;
-
-        /// <summary>
-        /// This is used by the regression test system to indicate which languages this algorithm is written in.
-        /// </summary>
-        public Language[] Languages { get; } = { Language.CSharp, Language.Python };
-
-        /// <summary>
-        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
-        /// </summary>
-        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
-        {
-            {"Total Trades", "2"},
-            {"Average Win", "0%"},
-            {"Average Loss", "-1.31%"},
-            {"Compounding Annual Return", "-15.304%"},
-            {"Drawdown", "1.300%"},
-            {"Expectancy", "-1"},
-            {"Net Profit", "-1.311%"},
-            {"Sharpe Ratio", "-3.31"},
-            {"Probabilistic Sharpe Ratio", "0.035%"},
-            {"Loss Rate", "100%"},
-            {"Win Rate", "0%"},
-            {"Profit-Loss Ratio", "0"},
-            {"Alpha", "0"},
-            {"Beta", "0"},
-            {"Annual Standard Deviation", "0.034"},
-            {"Annual Variance", "0.001"},
-            {"Information Ratio", "-3.31"},
-            {"Tracking Error", "0.034"},
-            {"Treynor Ratio", "0"},
-            {"Total Fees", "$1.00"},
-            {"Estimated Strategy Capacity", "$18000.00"},
-            {"Lowest Capacity Asset", "GOOCV W78ZFMML01JA|GOOCV VP83T1ZUHROL"},
-            {"Fitness Score", "0"},
-            {"Kelly Criterion Estimate", "0"},
-            {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "-1.496"},
-            {"Return Over Maximum Drawdown", "-11.673"},
-            {"Portfolio Turnover", "0"},
-            {"Total Insights Generated", "0"},
-            {"Total Insights Closed", "0"},
-            {"Total Insights Analysis Completed", "0"},
-            {"Long Insight Count", "0"},
-            {"Short Insight Count", "0"},
-            {"Long/Short Ratio", "100%"},
-            {"Estimated Monthly Alpha Value", "$0"},
-            {"Total Accumulated Estimated Alpha Value", "$0"},
-            {"Mean Population Estimated Insight Value", "$0"},
-            {"Mean Population Direction", "0%"},
-            {"Mean Population Magnitude", "0%"},
-            {"Rolling Averaged Population Direction", "0%"},
-            {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "c6d089f1fb86379c74a7413a9c2f8553"}
-        };
-    }
-}

Algorithm.CSharp/BasicTemplateOptionsFilterUniverseAlgorithm.cs

@@ -41,7 +41,7 @@ namespace QuantConnect.Algorithm.CSharp
         public override void Initialize()
         {
             SetStartDate(2015, 12, 24);
-            SetEndDate(2015, 12, 28);
+            SetEndDate(2015, 12, 24);
             SetCash(100000);
 
             var equity = AddEquity(UnderlyingTicker);
@@ -104,14 +104,14 @@ namespace QuantConnect.Algorithm.CSharp
         {
             {"Total Trades", "2"},
             {"Average Win", "0%"},
-            {"Average Loss", "-0.40%"},
-            {"Compounding Annual Return", "-21.622%"},
-            {"Drawdown", "0.300%"},
-            {"Expectancy", "-1"},
-            {"Net Profit", "-0.311%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "0%"},
+            {"Drawdown", "0%"},
+            {"Expectancy", "0"},
+            {"Net Profit", "0%"},
             {"Sharpe Ratio", "0"},
             {"Probabilistic Sharpe Ratio", "0%"},
-            {"Loss Rate", "100%"},
+            {"Loss Rate", "0%"},
             {"Win Rate", "0%"},
             {"Profit-Loss Ratio", "0"},
             {"Alpha", "0"},
@@ -124,12 +124,12 @@ namespace QuantConnect.Algorithm.CSharp
             {"Total Fees", "$1.00"},
             {"Estimated Strategy Capacity", "$0"},
             {"Lowest Capacity Asset", "GOOCV VP83T1ZUHROL"},
-            {"Fitness Score", "0.188"},
+            {"Fitness Score", "0"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "79228162514264337593543950335"},
-            {"Return Over Maximum Drawdown", "-73.268"},
-            {"Portfolio Turnover", "0.376"},
+            {"Sortino Ratio", "0"},
+            {"Return Over Maximum Drawdown", "0"},
+            {"Portfolio Turnover", "0"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
             {"Total Insights Analysis Completed", "0"},
@@ -143,7 +143,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "452e7a36e0a95e33d3457a908add3ead"}
+            {"OrderListHash", "92d8a50efe230524512404dab66b19dd"}
         };
     }
 }

Algorithm.CSharp/BasicTemplateOptionsFrameworkAlgorithm.cs

@@ -36,7 +36,7 @@ namespace QuantConnect.Algorithm.CSharp
             UniverseSettings.Resolution = Resolution.Minute;
 
             SetStartDate(2014, 06, 05);
-            SetEndDate(2014, 06, 09);
+            SetEndDate(2014, 06, 06);
             SetCash(100000);
 
             // set framework models
@@ -142,47 +142,47 @@ namespace QuantConnect.Algorithm.CSharp
         public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
             {"Total Trades", "4"},
-            {"Average Win", "0.14%"},
-            {"Average Loss", "-0.28%"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
             {"Compounding Annual Return", "0%"},
-            {"Drawdown", "385.400%"},
-            {"Expectancy", "-0.249"},
-            {"Net Profit", "-386.489%"},
-            {"Sharpe Ratio", "-0.033"},
-            {"Probabilistic Sharpe Ratio", "1.235%"},
-            {"Loss Rate", "50%"},
-            {"Win Rate", "50%"},
-            {"Profit-Loss Ratio", "0.50"},
-            {"Alpha", "-95.983"},
-            {"Beta", "263.726"},
-            {"Annual Standard Deviation", "30.617"},
-            {"Annual Variance", "937.371"},
-            {"Information Ratio", "-0.044"},
-            {"Tracking Error", "30.604"},
-            {"Treynor Ratio", "-0.004"},
+            {"Drawdown", "0%"},
+            {"Expectancy", "0"},
+            {"Net Profit", "0%"},
+            {"Sharpe Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "0%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "0"},
+            {"Tracking Error", "0"},
+            {"Treynor Ratio", "0"},
             {"Total Fees", "$3.00"},
-            {"Estimated Strategy Capacity", "$0"},
-            {"Lowest Capacity Asset", "AAPL R735QTJ8XC9X"},
-            {"Fitness Score", "0.168"},
+            {"Estimated Strategy Capacity", "$74000.00"},
+            {"Lowest Capacity Asset", "AAPL 2ZQGWTSSZ0WLI|AAPL R735QTJ8XC9X"},
+            {"Fitness Score", "0"},
             {"Kelly Criterion Estimate", "0.327"},
             {"Kelly Criterion Probability Value", "1"},
             {"Sortino Ratio", "79228162514264337593543950335"},
-            {"Return Over Maximum Drawdown", "0"},
-            {"Portfolio Turnover", "0.224"},
-            {"Total Insights Generated", "28"},
+            {"Return Over Maximum Drawdown", "79228162514264337593543950335"},
+            {"Portfolio Turnover", "0"},
+            {"Total Insights Generated", "26"},
             {"Total Insights Closed", "24"},
             {"Total Insights Analysis Completed", "24"},
-            {"Long Insight Count", "28"},
+            {"Long Insight Count", "26"},
             {"Short Insight Count", "0"},
             {"Long/Short Ratio", "100%"},
-            {"Estimated Monthly Alpha Value", "$13.64796"},
+            {"Estimated Monthly Alpha Value", "$31.01809"},
             {"Total Accumulated Estimated Alpha Value", "$1.89555"},
             {"Mean Population Estimated Insight Value", "$0.07898125"},
             {"Mean Population Direction", "50%"},
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "50.0482%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "87603bd45898dd9c456745fa51f989a5"}
+            {"OrderListHash", "ce06ddfa4b2ffeb666a8910ac8836992"}
         };
     }
 }

Algorithm.CSharp/BasicTemplateOptionsHourlyAlgorithm.cs

@@ -1,158 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
-*/
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using QuantConnect.Data;
-using QuantConnect.Data.Market;
-using QuantConnect.Orders;
-using QuantConnect.Interfaces;
-
-namespace QuantConnect.Algorithm.CSharp
-{
-    /// <summary>
-    /// This example demonstrates how to add options for a given underlying equity security.
-    /// It also shows how you can prefilter contracts easily based on strikes and expirations, and how you
-    /// can inspect the option chain to pick a specific option contract to trade.
-    /// </summary>
-    /// <meta name="tag" content="using data" />
-    /// <meta name="tag" content="options" />
-    /// <meta name="tag" content="filter selection" />
-    public class BasicTemplateOptionsHourlyAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
-    {
-        private const string UnderlyingTicker = "AAPL";
-        public Symbol OptionSymbol;
-
-        public override void Initialize()
-        {
-            SetStartDate(2014, 6, 6);
-            SetEndDate(2014, 6, 9);
-            SetCash(100000);
-
-            var equity = AddEquity(UnderlyingTicker, Resolution.Hour);
-            var option = AddOption(UnderlyingTicker, Resolution.Hour);
-            OptionSymbol = option.Symbol;
-
-            // set our strike/expiry filter for this option chain
-            option.SetFilter(u => u.Strikes(-2, +2)
-                                   // Expiration method accepts TimeSpan objects or integer for days.
-                                   // The following statements yield the same filtering criteria
-                                   .Expiration(0, 180));
-            // .Expiration(TimeSpan.Zero, TimeSpan.FromDays(180)));
-
-            // use the underlying equity as the benchmark
-            SetBenchmark(equity.Symbol);
-        }
-
-        /// <summary>
-        /// Event - v3.0 DATA EVENT HANDLER: (Pattern) Basic template for user to override for receiving all subscription data in a single event
-        /// </summary>
-        /// <param name="slice">The current slice of data keyed by symbol string</param>
-        public override void OnData(Slice slice)
-        {
-            if (!Portfolio.Invested && IsMarketOpen(OptionSymbol))
-            {
-                OptionChain chain;
-                if (slice.OptionChains.TryGetValue(OptionSymbol, out chain))
-                {
-                    // we find at the money (ATM) put contract with farthest expiration
-                    var atmContract = chain
-                        .OrderByDescending(x => x.Expiry)
-                        .ThenBy(x => Math.Abs(chain.Underlying.Price - x.Strike))
-                        .ThenByDescending(x => x.Right)
-                        .FirstOrDefault();
-
-                    if (atmContract != null)
-                    {
-                        // if found, trade it
-                        MarketOrder(atmContract.Symbol, 1);
-                        MarketOnCloseOrder(atmContract.Symbol, -1);
-                    }
-                }
-            }
-        }
-
-        /// <summary>
-        /// Order fill event handler. On an order fill update the resulting information is passed to this method.
-        /// </summary>
-        /// <param name="orderEvent">Order event details containing details of the evemts</param>
-        /// <remarks>This method can be called asynchronously and so should only be used by seasoned C# experts. Ensure you use proper locks on thread-unsafe objects</remarks>
-        public override void OnOrderEvent(OrderEvent orderEvent)
-        {
-            Log(orderEvent.ToString());
-        }
-
-        /// <summary>
-        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
-        /// </summary>
-        public bool CanRunLocally { get; } = true;
-
-        /// <summary>
-        /// This is used by the regression test system to indicate which languages this algorithm is written in.
-        /// </summary>
-        public Language[] Languages { get; } = { Language.CSharp, Language.Python };
-
-        /// <summary>
-        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
-        /// </summary>
-        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
-        {
-            {"Total Trades", "4"},
-            {"Average Win", "0%"},
-            {"Average Loss", "-0.07%"},
-            {"Compounding Annual Return", "-12.496%"},
-            {"Drawdown", "0.200%"},
-            {"Expectancy", "-1"},
-            {"Net Profit", "-0.134%"},
-            {"Sharpe Ratio", "-8.839"},
-            {"Probabilistic Sharpe Ratio", "0%"},
-            {"Loss Rate", "100%"},
-            {"Win Rate", "0%"},
-            {"Profit-Loss Ratio", "0"},
-            {"Alpha", "0.083"},
-            {"Beta", "-0.054"},
-            {"Annual Standard Deviation", "0.008"},
-            {"Annual Variance", "0"},
-            {"Information Ratio", "-18.699"},
-            {"Tracking Error", "0.155"},
-            {"Treynor Ratio", "1.296"},
-            {"Total Fees", "$4.00"},
-            {"Estimated Strategy Capacity", "$1000.00"},
-            {"Lowest Capacity Asset", "AAPL 2ZTXYMUAHCIAU|AAPL R735QTJ8XC9X"},
-            {"Fitness Score", "0.04"},
-            {"Kelly Criterion Estimate", "0"},
-            {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "79228162514264337593543950335"},
-            {"Return Over Maximum Drawdown", "-118.28"},
-            {"Portfolio Turnover", "0.081"},
-            {"Total Insights Generated", "0"},
-            {"Total Insights Closed", "0"},
-            {"Total Insights Analysis Completed", "0"},
-            {"Long Insight Count", "0"},
-            {"Short Insight Count", "0"},
-            {"Long/Short Ratio", "100%"},
-            {"Estimated Monthly Alpha Value", "$0"},
-            {"Total Accumulated Estimated Alpha Value", "$0"},
-            {"Mean Population Estimated Insight Value", "$0"},
-            {"Mean Population Direction", "0%"},
-            {"Mean Population Magnitude", "0%"},
-            {"Rolling Averaged Population Direction", "0%"},
-            {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "81e8a822d43de2165c1d3f52964ec312"}
-        };
-    }
-}

Algorithm.CSharp/ContinuousBackMonthRawFutureRegressionAlgorithm.cs

@@ -1,184 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using System.Linq;
-using QuantConnect.Data;
-using QuantConnect.Orders;
-using QuantConnect.Interfaces;
-using QuantConnect.Securities;
-using QuantConnect.Data.Market;
-using System.Collections.Generic;
-using QuantConnect.Securities.Future;
-
-namespace QuantConnect.Algorithm.CSharp
-{
-    /// <summary>
-    /// Continuous Back Month Raw Futures Regression algorithm. Asserting and showcasing the behavior of adding a continuous future
-    /// </summary>
-    public class ContinuousBackMonthRawFutureRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
-    {
-        private List<SymbolChangedEvent> _mappings = new();
-        private Future _continuousContract;
-        private DateTime _lastDateLog;
-
-        /// <summary>
-        /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
-        /// </summary>
-        public override void Initialize()
-        {
-            SetStartDate(2013, 7, 1);
-            SetEndDate(2014, 1, 1);
-
-            _continuousContract = AddFuture(Futures.Indices.SP500EMini,
-                dataNormalizationMode: DataNormalizationMode.Raw,
-                dataMappingMode: DataMappingMode.FirstDayMonth,
-                contractDepthOffset: 1
-            );
-        }
-
-        /// <summary>
-        /// OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
-        /// </summary>
-        /// <param name="data">Slice object keyed by symbol containing the stock data</param>
-        public override void OnData(Slice data)
-        {
-            if (data.Keys.Count != 1)
-            {
-                throw new Exception($"We are getting data for more than one symbols! {string.Join(",", data.Keys.Select(symbol => symbol))}");
-            }
-
-            foreach (var changedEvent in data.SymbolChangedEvents.Values)
-            {
-                if (changedEvent.Symbol == _continuousContract.Symbol)
-                {
-                    _mappings.Add(changedEvent);
-                    Log($"SymbolChanged event: {changedEvent}");
-
-                    var currentExpiration = changedEvent.Symbol.Underlying.ID.Date;
-                    // +4 months cause we are actually using the back month, es is quarterly contract
-                    var frontMonthExpiration = FuturesExpiryFunctions.FuturesExpiryFunction(_continuousContract.Symbol)(Time.AddMonths(1 + 4));
-
-                    if (currentExpiration != frontMonthExpiration.Date)
-                    {
-                        throw new Exception($"Unexpected current mapped contract expiration {currentExpiration}" +
-                            $" @ {Time} it should be AT front month expiration {frontMonthExpiration}");
-                    }
-                }
-            }
-
-            if (_lastDateLog.Month != Time.Month)
-            {
-                _lastDateLog = Time;
-
-                Log($"{Time}- {Securities[_continuousContract.Symbol].GetLastData()}");
-                if (Portfolio.Invested)
-                {
-                    Liquidate();
-                }
-                else if(_continuousContract.HasData)
-                {
-                    // This works because we set this contract as tradable, even if it's a canonical security
-                    Buy(_continuousContract.Symbol, 1);
-                }
-
-                if(Time.Month == 1 && Time.Year == 2013)
-                {
-                    var response = History(new[] { _continuousContract.Symbol }, 60 * 24 * 90);
-                    if (!response.Any())
-                    {
-                        throw new Exception("Unexpected empty history response");
-                    }
-                }
-            }
-        }
-
-        public override void OnOrderEvent(OrderEvent orderEvent)
-        {
-            if (orderEvent.Status == OrderStatus.Filled)
-            {
-                Log($"{orderEvent}");
-            }
-        }
-
-        public override void OnEndOfAlgorithm()
-        {
-            var expectedMappingCounts = 2;
-            if (_mappings.Count != expectedMappingCounts)
-            {
-                throw new Exception($"Unexpected symbol changed events: {_mappings.Count}, was expecting {expectedMappingCounts}");
-            }
-        }
-
-        /// <summary>
-        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
-        /// </summary>
-        public bool CanRunLocally { get; } = true;
-
-        /// <summary>
-        /// This is used by the regression test system to indicate which languages this algorithm is written in.
-        /// </summary>
-        public Language[] Languages { get; } = { Language.CSharp };
-
-        /// <summary>
-        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
-        /// </summary>
-        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
-        {
-            {"Total Trades", "2"},
-            {"Average Win", "1.11%"},
-            {"Average Loss", "0%"},
-            {"Compounding Annual Return", "2.199%"},
-            {"Drawdown", "1.700%"},
-            {"Expectancy", "0"},
-            {"Net Profit", "1.109%"},
-            {"Sharpe Ratio", "0.717"},
-            {"Probabilistic Sharpe Ratio", "38.157%"},
-            {"Loss Rate", "0%"},
-            {"Win Rate", "100%"},
-            {"Profit-Loss Ratio", "0"},
-            {"Alpha", "-0.007"},
-            {"Beta", "0.099"},
-            {"Annual Standard Deviation", "0.022"},
-            {"Annual Variance", "0"},
-            {"Information Ratio", "-2.732"},
-            {"Tracking Error", "0.076"},
-            {"Treynor Ratio", "0.156"},
-            {"Total Fees", "$3.70"},
-            {"Estimated Strategy Capacity", "$3900000.00"},
-            {"Lowest Capacity Asset", "ES 1S1"},
-            {"Fitness Score", "0.007"},
-            {"Kelly Criterion Estimate", "0"},
-            {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "0.484"},
-            {"Return Over Maximum Drawdown", "1.736"},
-            {"Portfolio Turnover", "0.011"},
-            {"Total Insights Generated", "0"},
-            {"Total Insights Closed", "0"},
-            {"Total Insights Analysis Completed", "0"},
-            {"Long Insight Count", "0"},
-            {"Short Insight Count", "0"},
-            {"Long/Short Ratio", "100%"},
-            {"Estimated Monthly Alpha Value", "$0"},
-            {"Total Accumulated Estimated Alpha Value", "$0"},
-            {"Mean Population Estimated Insight Value", "$0"},
-            {"Mean Population Direction", "0%"},
-            {"Mean Population Magnitude", "0%"},
-            {"Rolling Averaged Population Direction", "0%"},
-            {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "7d6fb409115f2f8d403c7eb261b9b3b6"}
-        };
-    }
-}

Algorithm.CSharp/ContinuousFutureBackMonthRegressionAlgorithm.cs

@@ -1,197 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using System.Linq;
-using QuantConnect.Data;
-using QuantConnect.Orders;
-using QuantConnect.Interfaces;
-using QuantConnect.Securities;
-using QuantConnect.Data.Market;
-using System.Collections.Generic;
-using QuantConnect.Securities.Future;
-
-namespace QuantConnect.Algorithm.CSharp
-{
-    /// <summary>
-    /// Continuous Futures Back Month #1 Regression algorithm. Asserting and showcasing the behavior of adding a continuous future
-    /// </summary>
-    public class ContinuousFutureBackMonthRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
-    {
-        private List<SymbolChangedEvent> _mappings = new();
-        private Future _continuousContract;
-        private DateTime _lastDateLog;
-
-        /// <summary>
-        /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
-        /// </summary>
-        public override void Initialize()
-        {
-            SetStartDate(2013, 7, 1);
-            SetEndDate(2014, 1, 1);
-
-            try
-            {
-                AddFuture(Futures.Indices.SP500EMini,
-                    dataNormalizationMode: DataNormalizationMode.BackwardsPanamaCanal,
-                    dataMappingMode: DataMappingMode.OpenInterest,
-                    contractDepthOffset: 5
-                );
-                throw new Exception("Expected out of rage exception. We don't support that many back months");
-            }
-            catch (ArgumentOutOfRangeException)
-            {
-                // expected
-            }
-
-            _continuousContract = AddFuture(Futures.Indices.SP500EMini,
-                dataNormalizationMode: DataNormalizationMode.BackwardsPanamaCanal,
-                dataMappingMode: DataMappingMode.OpenInterest,
-                contractDepthOffset: 1
-            );
-        }
-
-        /// <summary>
-        /// OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
-        /// </summary>
-        /// <param name="data">Slice object keyed by symbol containing the stock data</param>
-        public override void OnData(Slice data)
-        {
-            if (data.Keys.Count != 1)
-            {
-                throw new Exception($"We are getting data for more than one symbols! {string.Join(",", data.Keys.Select(symbol => symbol))}");
-            }
-
-            foreach (var changedEvent in data.SymbolChangedEvents.Values)
-            {
-                if (changedEvent.Symbol == _continuousContract.Symbol)
-                {
-                    _mappings.Add(changedEvent);
-                    Log($"SymbolChanged event: {changedEvent}");
-
-                    var backMonthExpiration = changedEvent.Symbol.Underlying.ID.Date;
-                    var frontMonthExpiration = FuturesExpiryFunctions.FuturesExpiryFunction(_continuousContract.Symbol)(Time.AddMonths(1));
-
-                    if (backMonthExpiration <= frontMonthExpiration.Date)
-                    {
-                        throw new Exception($"Unexpected current mapped contract expiration {backMonthExpiration}" +
-                            $" @ {Time} it should be AFTER front month expiration {frontMonthExpiration}");
-                    }
-                }
-            }
-
-            if (_lastDateLog.Month != Time.Month)
-            {
-                _lastDateLog = Time;
-
-                Log($"{Time}- {Securities[_continuousContract.Symbol].GetLastData()}");
-                if (Portfolio.Invested)
-                {
-                    Liquidate();
-                }
-                else if(_continuousContract.HasData)
-                {
-                    // This works because we set this contract as tradable, even if it's a canonical security
-                    Buy(_continuousContract.Symbol, 1);
-                }
-
-                if(Time.Month == 1 && Time.Year == 2013)
-                {
-                    var response = History(new[] { _continuousContract.Symbol }, 60 * 24 * 90);
-                    if (!response.Any())
-                    {
-                        throw new Exception("Unexpected empty history response");
-                    }
-                }
-            }
-        }
-
-        public override void OnOrderEvent(OrderEvent orderEvent)
-        {
-            if (orderEvent.Status == OrderStatus.Filled)
-            {
-                Log($"{orderEvent}");
-            }
-        }
-
-        public override void OnEndOfAlgorithm()
-        {
-            var expectedMappingCounts = 2;
-            if (_mappings.Count != expectedMappingCounts)
-            {
-                throw new Exception($"Unexpected symbol changed events: {_mappings.Count}, was expecting {expectedMappingCounts}");
-            }
-        }
-
-        /// <summary>
-        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
-        /// </summary>
-        public bool CanRunLocally { get; } = true;
-
-        /// <summary>
-        /// This is used by the regression test system to indicate which languages this algorithm is written in.
-        /// </summary>
-        public Language[] Languages { get; } = { Language.CSharp };
-
-        /// <summary>
-        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
-        /// </summary>
-        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
-        {
-            {"Total Trades", "3"},
-            {"Average Win", "1.11%"},
-            {"Average Loss", "0%"},
-            {"Compounding Annual Return", "2.092%"},
-            {"Drawdown", "1.700%"},
-            {"Expectancy", "0"},
-            {"Net Profit", "1.055%"},
-            {"Sharpe Ratio", "0.682"},
-            {"Probabilistic Sharpe Ratio", "36.937%"},
-            {"Loss Rate", "0%"},
-            {"Win Rate", "100%"},
-            {"Profit-Loss Ratio", "0"},
-            {"Alpha", "-0.007"},
-            {"Beta", "0.099"},
-            {"Annual Standard Deviation", "0.022"},
-            {"Annual Variance", "0"},
-            {"Information Ratio", "-2.742"},
-            {"Tracking Error", "0.076"},
-            {"Treynor Ratio", "0.149"},
-            {"Total Fees", "$5.55"},
-            {"Estimated Strategy Capacity", "$190000.00"},
-            {"Lowest Capacity Asset", "ES 1S1"},
-            {"Fitness Score", "0.01"},
-            {"Kelly Criterion Estimate", "0"},
-            {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "0.479"},
-            {"Return Over Maximum Drawdown", "1.652"},
-            {"Portfolio Turnover", "0.015"},
-            {"Total Insights Generated", "0"},
-            {"Total Insights Closed", "0"},
-            {"Total Insights Analysis Completed", "0"},
-            {"Long Insight Count", "0"},
-            {"Short Insight Count", "0"},
-            {"Long/Short Ratio", "100%"},
-            {"Estimated Monthly Alpha Value", "$0"},
-            {"Total Accumulated Estimated Alpha Value", "$0"},
-            {"Mean Population Estimated Insight Value", "$0"},
-            {"Mean Population Direction", "0%"},
-            {"Mean Population Magnitude", "0%"},
-            {"Rolling Averaged Population Direction", "0%"},
-            {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "9f7574803b8ebfac8f912019c943d27e"}
-        };
-    }
-}

Algorithm.CSharp/ContinuousFutureRegressionAlgorithm.cs

@@ -44,7 +44,7 @@ namespace QuantConnect.Algorithm.CSharp
 
             _continuousContract = AddFuture(Futures.Indices.SP500EMini,
                 dataNormalizationMode: DataNormalizationMode.BackwardsRatio,
-                dataMappingMode: DataMappingMode.LastTradingDay,
+                dataMappingMode: DataMappingMode.FirstDayMonth,
                 contractDepthOffset: 0
             );
         }
@@ -66,15 +66,6 @@ namespace QuantConnect.Algorithm.CSharp
                 {
                     _mappings.Add(changedEvent);
                     Log($"SymbolChanged event: {changedEvent}");
-
-                    var currentExpiration = changedEvent.Symbol.Underlying.ID.Date;
-                    var frontMonthExpiration = FuturesExpiryFunctions.FuturesExpiryFunction(_continuousContract.Symbol)(Time.AddMonths(1));
-
-                    if (currentExpiration != frontMonthExpiration.Date)
-                    {
-                        throw new Exception($"Unexpected current mapped contract expiration {currentExpiration}" +
-                            $" @ {Time} it should be AT front month expiration {frontMonthExpiration}");
-                    }
                 }
             }
 
@@ -136,34 +127,34 @@ namespace QuantConnect.Algorithm.CSharp
         /// </summary>
         public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
-            {"Total Trades", "3"},
-            {"Average Win", "1.03%"},
+            {"Total Trades", "2"},
+            {"Average Win", "1.13%"},
             {"Average Loss", "0%"},
-            {"Compounding Annual Return", "1.970%"},
-            {"Drawdown", "1.400%"},
+            {"Compounding Annual Return", "2.249%"},
+            {"Drawdown", "1.600%"},
             {"Expectancy", "0"},
-            {"Net Profit", "0.994%"},
-            {"Sharpe Ratio", "0.7"},
-            {"Probabilistic Sharpe Ratio", "37.553%"},
+            {"Net Profit", "1.134%"},
+            {"Sharpe Ratio", "0.727"},
+            {"Probabilistic Sharpe Ratio", "38.512%"},
             {"Loss Rate", "0%"},
             {"Win Rate", "100%"},
             {"Profit-Loss Ratio", "0"},
             {"Alpha", "-0.006"},
-            {"Beta", "0.091"},
-            {"Annual Standard Deviation", "0.02"},
+            {"Beta", "0.099"},
+            {"Annual Standard Deviation", "0.022"},
             {"Annual Variance", "0"},
-            {"Information Ratio", "-2.745"},
+            {"Information Ratio", "-2.728"},
             {"Tracking Error", "0.076"},
-            {"Treynor Ratio", "0.153"},
-            {"Total Fees", "$5.55"},
-            {"Estimated Strategy Capacity", "$48000000.00"},
+            {"Treynor Ratio", "0.159"},
+            {"Total Fees", "$3.70"},
+            {"Estimated Strategy Capacity", "$810000000.00"},
             {"Lowest Capacity Asset", "ES 1S1"},
-            {"Fitness Score", "0.01"},
+            {"Fitness Score", "0.007"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "0.492"},
-            {"Return Over Maximum Drawdown", "1.708"},
-            {"Portfolio Turnover", "0.016"},
+            {"Sortino Ratio", "0.495"},
+            {"Return Over Maximum Drawdown", "1.776"},
+            {"Portfolio Turnover", "0.011"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
             {"Total Insights Analysis Completed", "0"},
@@ -177,7 +168,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "fb3bb82d84fc6c390a40f36d0d1faf59"}
+            {"OrderListHash", "b622c3dcec2c38a81e336d59a3e4c92d"}
         };
     }
 }

Algorithm.CSharp/DelistedFutureLiquidateRegressionAlgorithm.cs

@@ -94,14 +94,14 @@ namespace QuantConnect.Algorithm.CSharp
         public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
             {"Total Trades", "2"},
-            {"Average Win", "1.64%"},
+            {"Average Win", "1.63%"},
             {"Average Loss", "0%"},
-            {"Compounding Annual Return", "7.329%"},
+            {"Compounding Annual Return", "7.292%"},
             {"Drawdown", "1.500%"},
             {"Expectancy", "0"},
-            {"Net Profit", "1.642%"},
-            {"Sharpe Ratio", "2.36"},
-            {"Probabilistic Sharpe Ratio", "94.555%"},
+            {"Net Profit", "1.634%"},
+            {"Sharpe Ratio", "2.351"},
+            {"Probabilistic Sharpe Ratio", "94.365%"},
             {"Loss Rate", "0%"},
             {"Win Rate", "100%"},
             {"Profit-Loss Ratio", "0"},
@@ -109,17 +109,17 @@ namespace QuantConnect.Algorithm.CSharp
             {"Beta", "0.158"},
             {"Annual Standard Deviation", "0.03"},
             {"Annual Variance", "0.001"},
-            {"Information Ratio", "-4.44"},
+            {"Information Ratio", "-4.444"},
             {"Tracking Error", "0.075"},
-            {"Treynor Ratio", "0.441"},
-            {"Total Fees", "$1.85"},
+            {"Treynor Ratio", "0.439"},
+            {"Total Fees", "$3.70"},
             {"Estimated Strategy Capacity", "$170000000.00"},
             {"Lowest Capacity Asset", "ES VMKLFZIH2MTD"},
             {"Fitness Score", "0.019"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "1.369"},
-            {"Return Over Maximum Drawdown", "9.749"},
+            {"Sortino Ratio", "1.362"},
+            {"Return Over Maximum Drawdown", "9.699"},
             {"Portfolio Turnover", "0.023"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
@@ -134,7 +134,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "4c5e32aedcd5bb67642d1629628fe615"}
+            {"OrderListHash", "00d6dc8775da38f7f79defad06de240a"}
         };
     }
 }

Algorithm.CSharp/DelistedIndexOptionDivestedRegression.cs

@@ -137,13 +137,13 @@ namespace QuantConnect.Algorithm.CSharp
             {"Loss Rate", "100%"},
             {"Win Rate", "0%"},
             {"Profit-Loss Ratio", "0"},
-            {"Alpha", "-0.559"},
-            {"Beta", "-0.807"},
+            {"Alpha", "-0.592"},
+            {"Beta", "-0.737"},
             {"Annual Standard Deviation", "1.582"},
             {"Annual Variance", "2.502"},
             {"Information Ratio", "-0.905"},
-            {"Tracking Error", "1.593"},
-            {"Treynor Ratio", "1.181"},
+            {"Tracking Error", "1.592"},
+            {"Treynor Ratio", "1.292"},
             {"Total Fees", "$0.00"},
             {"Estimated Strategy Capacity", "$1000000.00"},
             {"Lowest Capacity Asset", "SPX 31KC0UJFONTBI|SPX 31"},
@@ -166,7 +166,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "721fddfd1327f7adcc2883d1412708c9"}
+            {"OrderListHash", "4ae4c1d8e4054c41908fd36e893699a6"}
         };
     }
 }

Algorithm.CSharp/DelistingEventsAlgorithm.cs

@@ -140,34 +140,34 @@ namespace QuantConnect.Algorithm.CSharp
         /// </summary>
         public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
-            {"Total Trades", "2"},
+            {"Total Trades", "3"},
             {"Average Win", "0%"},
-            {"Average Loss", "-5.58%"},
-            {"Compounding Annual Return", "-87.694%"},
-            {"Drawdown", "5.600%"},
+            {"Average Loss", "-3.23%"},
+            {"Compounding Annual Return", "-79.990%"},
+            {"Drawdown", "4.300%"},
             {"Expectancy", "-1"},
-            {"Net Profit", "-5.578%"},
-            {"Sharpe Ratio", "-4.683"},
-            {"Probabilistic Sharpe Ratio", "0.008%"},
+            {"Net Profit", "-4.312%"},
+            {"Sharpe Ratio", "-5.637"},
+            {"Probabilistic Sharpe Ratio", "0.005%"},
             {"Loss Rate", "100%"},
             {"Win Rate", "0%"},
             {"Profit-Loss Ratio", "0"},
-            {"Alpha", "-0.622"},
-            {"Beta", "-0.877"},
-            {"Annual Standard Deviation", "0.142"},
-            {"Annual Variance", "0.02"},
-            {"Information Ratio", "-3.844"},
-            {"Tracking Error", "0.186"},
-            {"Treynor Ratio", "0.759"},
-            {"Total Fees", "$36.70"},
-            {"Estimated Strategy Capacity", "$65000.00"},
+            {"Alpha", "-0.5"},
+            {"Beta", "-0.346"},
+            {"Annual Standard Deviation", "0.092"},
+            {"Annual Variance", "0.008"},
+            {"Information Ratio", "-4.312"},
+            {"Tracking Error", "0.131"},
+            {"Treynor Ratio", "1.493"},
+            {"Total Fees", "$55.05"},
+            {"Estimated Strategy Capacity", "$43000.00"},
             {"Lowest Capacity Asset", "AAA SEVKGI6HF885"},
-            {"Fitness Score", "0.003"},
+            {"Fitness Score", "0.002"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "-10.959"},
-            {"Return Over Maximum Drawdown", "-15.72"},
-            {"Portfolio Turnover", "0.224"},
+            {"Sortino Ratio", "-15.687"},
+            {"Return Over Maximum Drawdown", "-18.549"},
+            {"Portfolio Turnover", "0.334"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
             {"Total Insights Analysis Completed", "0"},
@@ -181,7 +181,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "2d66947eafcca81ba9a2cd3bb351eee2"}
+            {"OrderListHash", "e357cfa77fd5e5b974c68d550fa66490"}
         };
     }
 }

Algorithm.CSharp/DelistingFutureOptionRegressionAlgorithm.cs

@@ -108,9 +108,9 @@ namespace QuantConnect.Algorithm.CSharp
             {"Average Loss", "-0.02%"},
             {"Compounding Annual Return", "-0.111%"},
             {"Drawdown", "0.100%"},
-            {"Expectancy", "-0.678"},
-            {"Net Profit", "-0.111%"},
-            {"Sharpe Ratio", "-0.967"},
+            {"Expectancy", "-0.679"},
+            {"Net Profit", "-0.112%"},
+            {"Sharpe Ratio", "-0.966"},
             {"Probabilistic Sharpe Ratio", "0.000%"},
             {"Loss Rate", "80%"},
             {"Win Rate", "20%"},
@@ -121,8 +121,8 @@ namespace QuantConnect.Algorithm.CSharp
             {"Annual Variance", "0"},
             {"Information Ratio", "-1.075"},
             {"Tracking Error", "0.107"},
-            {"Treynor Ratio", "1.353"},
-            {"Total Fees", "$14.80"},
+            {"Treynor Ratio", "1.354"},
+            {"Total Fees", "$37.00"},
             {"Estimated Strategy Capacity", "$860000000.00"},
             {"Lowest Capacity Asset", "DC V5E8P9SH0U0X"},
             {"Fitness Score", "0"},
@@ -144,7 +144,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "d10e8665214344369e3e8f1c49dbdd67"}
+            {"OrderListHash", "de309ab56d2fcd80ff03df2802d9feda"}
         };
     }
 }

Algorithm.CSharp/FutureOptionBuySellCallIntradayRegressionAlgorithm.cs

@@ -119,32 +119,32 @@ namespace QuantConnect.Algorithm.CSharp
         public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
             {"Total Trades", "6"},
-            {"Average Win", "2.94%"},
+            {"Average Win", "2.93%"},
             {"Average Loss", "-4.15%"},
-            {"Compounding Annual Return", "-5.589%"},
-            {"Drawdown", "5.600%"},
-            {"Expectancy", "-0.145"},
-            {"Net Profit", "-2.760%"},
-            {"Sharpe Ratio", "-0.45"},
-            {"Probabilistic Sharpe Ratio", "9.306%"},
+            {"Compounding Annual Return", "-5.673%"},
+            {"Drawdown", "5.700%"},
+            {"Expectancy", "-0.148"},
+            {"Net Profit", "-2.802%"},
+            {"Sharpe Ratio", "-0.456"},
+            {"Probabilistic Sharpe Ratio", "9.156%"},
             {"Loss Rate", "50%"},
             {"Win Rate", "50%"},
-            {"Profit-Loss Ratio", "0.71"},
+            {"Profit-Loss Ratio", "0.70"},
             {"Alpha", "-0.036"},
             {"Beta", "-0.012"},
             {"Annual Standard Deviation", "0.08"},
             {"Annual Variance", "0.006"},
-            {"Information Ratio", "-0.149"},
+            {"Information Ratio", "-0.15"},
             {"Tracking Error", "0.387"},
-            {"Treynor Ratio", "2.943"},
-            {"Total Fees", "$3.70"},
-            {"Estimated Strategy Capacity", "$280000000.00"},
-            {"Lowest Capacity Asset", "ES XFH59UPBIJ7O|ES XFH59UK0MYO1"},
+            {"Treynor Ratio", "3.008"},
+            {"Total Fees", "$14.80"},
+            {"Estimated Strategy Capacity", "$180000000.00"},
+            {"Lowest Capacity Asset", "ES XFH59UK0MYO1"},
             {"Fitness Score", "0.017"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "-0.096"},
-            {"Return Over Maximum Drawdown", "-0.993"},
+            {"Sortino Ratio", "-0.097"},
+            {"Return Over Maximum Drawdown", "-1.002"},
             {"Portfolio Turnover", "0.043"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
@@ -159,7 +159,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "18f8a17034aa12be40581baecca96788"}
+            {"OrderListHash", "fc9eb9b0a644e4890d5ec3d40367d0e1"}
         };
     }
 }

Algorithm.CSharp/FutureOptionCallITMExpiryRegressionAlgorithm.cs

@@ -126,7 +126,7 @@ namespace QuantConnect.Algorithm.CSharp
 
         private void AssertFutureOptionOrderExercise(OrderEvent orderEvent, Security future, Security optionContract)
         {
-            var expectedLiquidationTimeUtc = new DateTime(2020, 6, 20, 4, 0, 0);
+            var expectedLiquidationTimeUtc = new DateTime(2020, 6, 19, 20, 0, 0);
 
             if (orderEvent.Direction == OrderDirection.Sell && future.Holdings.Quantity != 0)
             {
@@ -203,13 +203,13 @@ namespace QuantConnect.Algorithm.CSharp
         public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
             {"Total Trades", "3"},
-            {"Average Win", "1.26%"},
-            {"Average Loss", "-7.41%"},
-            {"Compounding Annual Return", "-12.424%"},
+            {"Average Win", "1.22%"},
+            {"Average Loss", "-7.42%"},
+            {"Compounding Annual Return", "-12.482%"},
             {"Drawdown", "6.300%"},
-            {"Expectancy", "-0.415"},
-            {"Net Profit", "-6.252%"},
-            {"Sharpe Ratio", "-1.226"},
+            {"Expectancy", "-0.417"},
+            {"Net Profit", "-6.282%"},
+            {"Sharpe Ratio", "-1.225"},
             {"Probabilistic Sharpe Ratio", "0.002%"},
             {"Loss Rate", "50%"},
             {"Win Rate", "50%"},
@@ -218,12 +218,12 @@ namespace QuantConnect.Algorithm.CSharp
             {"Beta", "0.004"},
             {"Annual Standard Deviation", "0.07"},
             {"Annual Variance", "0.005"},
-            {"Information Ratio", "-0.283"},
+            {"Information Ratio", "-0.284"},
             {"Tracking Error", "0.379"},
-            {"Treynor Ratio", "-23.811"},
-            {"Total Fees", "$1.85"},
-            {"Estimated Strategy Capacity", "$270000000.00"},
-            {"Lowest Capacity Asset", "ES XFH59UPBIJ7O|ES XFH59UK0MYO1"},
+            {"Treynor Ratio", "-23.801"},
+            {"Total Fees", "$7.40"},
+            {"Estimated Strategy Capacity", "$180000000.00"},
+            {"Lowest Capacity Asset", "ES XFH59UK0MYO1"},
             {"Fitness Score", "0.008"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
@@ -243,7 +243,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "b738fdaf1dae6849884df9e51eb6482b"}
+            {"OrderListHash", "c59d790b89d76f1ad3bb7738b28567c9"}
         };
     }
 }

Algorithm.CSharp/FutureOptionCallITMGreeksExpiryRegressionAlgorithm.cs

@@ -168,32 +168,32 @@ namespace QuantConnect.Algorithm.CSharp
         public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
             {"Total Trades", "3"},
-            {"Average Win", "8.93%"},
-            {"Average Loss", "-34.88%"},
-            {"Compounding Annual Return", "-50.632%"},
-            {"Drawdown", "29.100%"},
-            {"Expectancy", "-0.372"},
-            {"Net Profit", "-29.072%"},
-            {"Sharpe Ratio", "-0.978"},
+            {"Average Win", "8.71%"},
+            {"Average Loss", "-34.89%"},
+            {"Compounding Annual Return", "-50.850%"},
+            {"Drawdown", "29.200%"},
+            {"Expectancy", "-0.375"},
+            {"Net Profit", "-29.224%"},
+            {"Sharpe Ratio", "-0.977"},
             {"Probabilistic Sharpe Ratio", "0.012%"},
             {"Loss Rate", "50%"},
             {"Win Rate", "50%"},
-            {"Profit-Loss Ratio", "0.26"},
-            {"Alpha", "-0.339"},
+            {"Profit-Loss Ratio", "0.25"},
+            {"Alpha", "-0.341"},
             {"Beta", "0.017"},
-            {"Annual Standard Deviation", "0.347"},
-            {"Annual Variance", "0.12"},
-            {"Information Ratio", "-0.714"},
-            {"Tracking Error", "0.505"},
-            {"Treynor Ratio", "-19.672"},
-            {"Total Fees", "$9.25"},
-            {"Estimated Strategy Capacity", "$50000000.00"},
-            {"Lowest Capacity Asset", "ES XFH59UPBIJ7O|ES XFH59UK0MYO1"},
-            {"Fitness Score", "0.055"},
+            {"Annual Standard Deviation", "0.348"},
+            {"Annual Variance", "0.121"},
+            {"Information Ratio", "-0.715"},
+            {"Tracking Error", "0.506"},
+            {"Treynor Ratio", "-19.652"},
+            {"Total Fees", "$37.00"},
+            {"Estimated Strategy Capacity", "$33000000.00"},
+            {"Lowest Capacity Asset", "ES XFH59UK0MYO1"},
+            {"Fitness Score", "0.056"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
             {"Sortino Ratio", "-0.155"},
-            {"Return Over Maximum Drawdown", "-1.743"},
+            {"Return Over Maximum Drawdown", "-1.741"},
             {"Portfolio Turnover", "0.152"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
@@ -208,7 +208,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "ed0cbd8487dd45519e5d0225e51ba29c"}
+            {"OrderListHash", "ca0898608da51d972723b1065a3f0d47"}
         };
     }
 }

Algorithm.CSharp/FutureOptionCallOTMExpiryRegressionAlgorithm.cs

@@ -179,11 +179,11 @@ namespace QuantConnect.Algorithm.CSharp
         {
             {"Total Trades", "2"},
             {"Average Win", "0%"},
-            {"Average Loss", "-4.02%"},
-            {"Compounding Annual Return", "-8.099%"},
+            {"Average Loss", "-4.03%"},
+            {"Compounding Annual Return", "-8.103%"},
             {"Drawdown", "4.000%"},
             {"Expectancy", "-1"},
-            {"Net Profit", "-4.027%"},
+            {"Net Profit", "-4.029%"},
             {"Sharpe Ratio", "-1.175"},
             {"Probabilistic Sharpe Ratio", "0.009%"},
             {"Loss Rate", "100%"},
@@ -195,8 +195,8 @@ namespace QuantConnect.Algorithm.CSharp
             {"Annual Variance", "0.002"},
             {"Information Ratio", "-0.206"},
             {"Tracking Error", "0.376"},
-            {"Treynor Ratio", "-23.481"},
-            {"Total Fees", "$1.85"},
+            {"Treynor Ratio", "-23.48"},
+            {"Total Fees", "$3.70"},
             {"Estimated Strategy Capacity", "$200000000.00"},
             {"Lowest Capacity Asset", "ES XFH59UPHGV9G|ES XFH59UK0MYO1"},
             {"Fitness Score", "0"},
@@ -218,7 +218,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "cf1c12b839e49456dc2793f0e63c7803"}
+            {"OrderListHash", "5dc2591837f882d173d2d4852b3b0626"}
         };
     }
 }

Algorithm.CSharp/FutureOptionPutITMExpiryRegressionAlgorithm.cs

@@ -127,7 +127,7 @@ namespace QuantConnect.Algorithm.CSharp
 
         private void AssertFutureOptionOrderExercise(OrderEvent orderEvent, Security future, Security optionContract)
         {
-            var expectedLiquidationTimeUtc = new DateTime(2020, 6, 20, 4, 0, 0);
+            var expectedLiquidationTimeUtc = new DateTime(2020, 6, 19, 20, 0, 0);
 
             if (orderEvent.Direction == OrderDirection.Buy && future.Holdings.Quantity != 0)
             {
@@ -204,31 +204,31 @@ namespace QuantConnect.Algorithm.CSharp
         public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
             {"Total Trades", "3"},
-            {"Average Win", "4.18%"},
-            {"Average Loss", "-8.26%"},
-            {"Compounding Annual Return", "-8.884%"},
-            {"Drawdown", "4.400%"},
-            {"Expectancy", "-0.247"},
-            {"Net Profit", "-4.427%"},
-            {"Sharpe Ratio", "-1.283"},
+            {"Average Win", "4.15%"},
+            {"Average Loss", "-8.27%"},
+            {"Compounding Annual Return", "-8.944%"},
+            {"Drawdown", "4.500%"},
+            {"Expectancy", "-0.249"},
+            {"Net Profit", "-4.457%"},
+            {"Sharpe Ratio", "-1.282"},
             {"Probabilistic Sharpe Ratio", "0.001%"},
             {"Loss Rate", "50%"},
             {"Win Rate", "50%"},
-            {"Profit-Loss Ratio", "0.51"},
-            {"Alpha", "-0.061"},
-            {"Beta", "0.002"},
+            {"Profit-Loss Ratio", "0.50"},
+            {"Alpha", "-0.062"},
+            {"Beta", "0.003"},
             {"Annual Standard Deviation", "0.048"},
             {"Annual Variance", "0.002"},
-            {"Information Ratio", "-0.221"},
+            {"Information Ratio", "-0.222"},
             {"Tracking Error", "0.376"},
-            {"Treynor Ratio", "-24.544"},
-            {"Total Fees", "$1.85"},
-            {"Estimated Strategy Capacity", "$330000000.00"},
-            {"Lowest Capacity Asset", "ES 31EL5FAOOQON8|ES XFH59UK0MYO1"},
+            {"Treynor Ratio", "-24.53"},
+            {"Total Fees", "$7.40"},
+            {"Estimated Strategy Capacity", "$220000000.00"},
+            {"Lowest Capacity Asset", "ES XFH59UK0MYO1"},
             {"Fitness Score", "0.008"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "-0.225"},
+            {"Sortino Ratio", "-0.224"},
             {"Return Over Maximum Drawdown", "-2.009"},
             {"Portfolio Turnover", "0.023"},
             {"Total Insights Generated", "0"},
@@ -244,7 +244,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "99f96f433bc76c31cb25bcd9117a6bf1"}
+            {"OrderListHash", "d3fa88c3acadb9345ceac76a2dd3b520"}
         };
     }
 }

Algorithm.CSharp/FutureOptionPutOTMExpiryRegressionAlgorithm.cs

@@ -178,11 +178,11 @@ namespace QuantConnect.Algorithm.CSharp
         {
             {"Total Trades", "2"},
             {"Average Win", "0%"},
-            {"Average Loss", "-5.11%"},
-            {"Compounding Annual Return", "-10.226%"},
+            {"Average Loss", "-5.12%"},
+            {"Compounding Annual Return", "-10.230%"},
             {"Drawdown", "5.100%"},
             {"Expectancy", "-1"},
-            {"Net Profit", "-5.114%"},
+            {"Net Profit", "-5.116%"},
             {"Sharpe Ratio", "-1.164"},
             {"Probabilistic Sharpe Ratio", "0.009%"},
             {"Loss Rate", "100%"},
@@ -195,7 +195,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Information Ratio", "-0.243"},
             {"Tracking Error", "0.378"},
             {"Treynor Ratio", "-23.284"},
-            {"Total Fees", "$1.85"},
+            {"Total Fees", "$3.70"},
             {"Estimated Strategy Capacity", "$360000000.00"},
             {"Lowest Capacity Asset", "ES 31EL5FBZBMXES|ES XFH59UK0MYO1"},
             {"Fitness Score", "0"},
@@ -217,7 +217,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "ec799886c15ac6c4b8fb3d873d7e6b14"}
+            {"OrderListHash", "a35054d03fd2caa0a96cbe12e427e928"}
         };
     }
 }

Algorithm.CSharp/FutureOptionShortCallITMExpiryRegressionAlgorithm.cs

@@ -189,31 +189,31 @@ namespace QuantConnect.Algorithm.CSharp
         {
             {"Total Trades", "3"},
             {"Average Win", "10.05%"},
-            {"Average Loss", "-5.60%"},
-            {"Compounding Annual Return", "8.148%"},
+            {"Average Loss", "-5.63%"},
+            {"Compounding Annual Return", "8.083%"},
             {"Drawdown", "0.500%"},
-            {"Expectancy", "0.397"},
-            {"Net Profit", "3.886%"},
-            {"Sharpe Ratio", "1.088"},
-            {"Probabilistic Sharpe Ratio", "53.421%"},
+            {"Expectancy", "0.393"},
+            {"Net Profit", "3.855%"},
+            {"Sharpe Ratio", "1.087"},
+            {"Probabilistic Sharpe Ratio", "53.360%"},
             {"Loss Rate", "50%"},
             {"Win Rate", "50%"},
             {"Profit-Loss Ratio", "1.79"},
             {"Alpha", "0.057"},
             {"Beta", "-0.002"},
-            {"Annual Standard Deviation", "0.053"},
+            {"Annual Standard Deviation", "0.052"},
             {"Annual Variance", "0.003"},
-            {"Information Ratio", "0.094"},
+            {"Information Ratio", "0.093"},
             {"Tracking Error", "0.379"},
-            {"Treynor Ratio", "-23.276"},
-            {"Total Fees", "$1.85"},
-            {"Estimated Strategy Capacity", "$300000000.00"},
-            {"Lowest Capacity Asset", "ES XFH59UP5K75W|ES XFH59UK0MYO1"},
+            {"Treynor Ratio", "-23.26"},
+            {"Total Fees", "$7.40"},
+            {"Estimated Strategy Capacity", "$200000000.00"},
+            {"Lowest Capacity Asset", "ES XFH59UK0MYO1"},
             {"Fitness Score", "0.02"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
             {"Sortino Ratio", "79228162514264337593543950335"},
-            {"Return Over Maximum Drawdown", "17.34"},
+            {"Return Over Maximum Drawdown", "17.201"},
             {"Portfolio Turnover", "0.02"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
@@ -228,7 +228,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "8a33f32b29bcc66d4dd779b36df9a010"}
+            {"OrderListHash", "8e380e4d5c5e3e145ba388f7853829bb"}
         };
     }
 }

Algorithm.CSharp/FutureOptionShortCallOTMExpiryRegressionAlgorithm.cs

@@ -173,12 +173,12 @@ namespace QuantConnect.Algorithm.CSharp
             {"Total Trades", "2"},
             {"Average Win", "1.81%"},
             {"Average Loss", "0%"},
-            {"Compounding Annual Return", "3.756%"},
+            {"Compounding Annual Return", "3.752%"},
             {"Drawdown", "0.000%"},
             {"Expectancy", "0"},
-            {"Net Profit", "1.811%"},
+            {"Net Profit", "1.809%"},
             {"Sharpe Ratio", "1.183"},
-            {"Probabilistic Sharpe Ratio", "60.811%"},
+            {"Probabilistic Sharpe Ratio", "60.809%"},
             {"Loss Rate", "0%"},
             {"Win Rate", "100%"},
             {"Profit-Loss Ratio", "0"},
@@ -188,15 +188,15 @@ namespace QuantConnect.Algorithm.CSharp
             {"Annual Variance", "0"},
             {"Information Ratio", "0.012"},
             {"Tracking Error", "0.375"},
-            {"Treynor Ratio", "-24.052"},
-            {"Total Fees", "$1.85"},
+            {"Treynor Ratio", "-24.051"},
+            {"Total Fees", "$3.70"},
             {"Estimated Strategy Capacity", "$78000000.00"},
             {"Lowest Capacity Asset", "ES XFH59UPNF7B8|ES XFH59UK0MYO1"},
             {"Fitness Score", "0"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
             {"Sortino Ratio", "79228162514264337593543950335"},
-            {"Return Over Maximum Drawdown", "95.594"},
+            {"Return Over Maximum Drawdown", "95.495"},
             {"Portfolio Turnover", "0"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
@@ -211,7 +211,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "8cb012d36057103bf26a897fe5fa54d6"}
+            {"OrderListHash", "35baadd70ec72c735eadbf55d702fe04"}
         };
     }
 }

Algorithm.CSharp/FutureOptionShortPutITMExpiryRegressionAlgorithm.cs

@@ -185,32 +185,32 @@ namespace QuantConnect.Algorithm.CSharp
         public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
             {"Total Trades", "3"},
-            {"Average Win", "10.19%"},
-            {"Average Loss", "-8.02%"},
-            {"Compounding Annual Return", "2.790%"},
+            {"Average Win", "10.18%"},
+            {"Average Loss", "-8.05%"},
+            {"Compounding Annual Return", "2.726%"},
             {"Drawdown", "0.500%"},
-            {"Expectancy", "0.135"},
-            {"Net Profit", "1.348%"},
-            {"Sharpe Ratio", "0.862"},
-            {"Probabilistic Sharpe Ratio", "42.945%"},
+            {"Expectancy", "0.133"},
+            {"Net Profit", "1.318%"},
+            {"Sharpe Ratio", "0.855"},
+            {"Probabilistic Sharpe Ratio", "42.696%"},
             {"Loss Rate", "50%"},
             {"Win Rate", "50%"},
             {"Profit-Loss Ratio", "1.27"},
-            {"Alpha", "0.02"},
+            {"Alpha", "0.019"},
             {"Beta", "-0.001"},
-            {"Annual Standard Deviation", "0.023"},
-            {"Annual Variance", "0.001"},
-            {"Information Ratio", "-0.006"},
+            {"Annual Standard Deviation", "0.022"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "-0.007"},
             {"Tracking Error", "0.375"},
-            {"Treynor Ratio", "-20.61"},
-            {"Total Fees", "$1.85"},
-            {"Estimated Strategy Capacity", "$200000000.00"},
-            {"Lowest Capacity Asset", "ES 31EL5FAOUP0P0|ES XFH59UK0MYO1"},
+            {"Treynor Ratio", "-20.534"},
+            {"Total Fees", "$7.40"},
+            {"Estimated Strategy Capacity", "$130000000.00"},
+            {"Lowest Capacity Asset", "ES XFH59UK0MYO1"},
             {"Fitness Score", "0.021"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
             {"Sortino Ratio", "79228162514264337593543950335"},
-            {"Return Over Maximum Drawdown", "5.859"},
+            {"Return Over Maximum Drawdown", "5.725"},
             {"Portfolio Turnover", "0.022"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
@@ -225,7 +225,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "eb37251ad1e32dd348af8a69e1888053"}
+            {"OrderListHash", "519cc7c39703b6e6913dbe98468da872"}
         };
     }
 }

Algorithm.CSharp/FutureOptionShortPutOTMExpiryRegressionAlgorithm.cs

@@ -170,12 +170,12 @@ namespace QuantConnect.Algorithm.CSharp
         public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
             {"Total Trades", "2"},
-            {"Average Win", "3.29%"},
+            {"Average Win", "3.28%"},
             {"Average Loss", "0%"},
-            {"Compounding Annual Return", "6.869%"},
+            {"Compounding Annual Return", "6.865%"},
             {"Drawdown", "0.000%"},
             {"Expectancy", "0"},
-            {"Net Profit", "3.286%"},
+            {"Net Profit", "3.284%"},
             {"Sharpe Ratio", "1.205"},
             {"Probabilistic Sharpe Ratio", "61.483%"},
             {"Loss Rate", "0%"},
@@ -188,14 +188,14 @@ namespace QuantConnect.Algorithm.CSharp
             {"Information Ratio", "0.07"},
             {"Tracking Error", "0.377"},
             {"Treynor Ratio", "-24.401"},
-            {"Total Fees", "$1.85"},
+            {"Total Fees", "$3.70"},
             {"Estimated Strategy Capacity", "$80000000.00"},
             {"Lowest Capacity Asset", "ES 31EL5FAJQ6SBO|ES XFH59UK0MYO1"},
             {"Fitness Score", "0"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
             {"Sortino Ratio", "79228162514264337593543950335"},
-            {"Return Over Maximum Drawdown", "150.849"},
+            {"Return Over Maximum Drawdown", "150.763"},
             {"Portfolio Turnover", "0"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
@@ -210,7 +210,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "76ed4eaa5f6ed50aa6134aecfbbe9e29"}
+            {"OrderListHash", "5571280f1efb15dd3899896fb72385ff"}
         };
     }
 }

Algorithm.CSharp/FuturesAndFuturesOptionsExpiryTimeAndLiquidationRegressionAlgorithm.cs

@@ -38,7 +38,7 @@ namespace QuantConnect.Algorithm.CSharp
 
         private readonly DateTime _expectedExpiryWarningTime = new DateTime(2020, 6, 19);
         private readonly DateTime _expectedExpiryDelistingTime = new DateTime(2020, 6, 20);
-        private readonly DateTime _expectedLiquidationTime = new DateTime(2020, 6, 20);
+        private readonly DateTime _expectedLiquidationTime = new DateTime(2020, 6, 19, 16, 0, 0);
 
         public override void Initialize()
         {
@@ -183,15 +183,15 @@ namespace QuantConnect.Algorithm.CSharp
             {"Total Trades", "3"},
             {"Average Win", "10.15%"},
             {"Average Loss", "-11.34%"},
-            {"Compounding Annual Return", "-2.573%"},
+            {"Compounding Annual Return", "-2.578%"},
             {"Drawdown", "2.300%"},
-            {"Expectancy", "-0.052"},
-            {"Net Profit", "-2.341%"},
+            {"Expectancy", "-0.053"},
+            {"Net Profit", "-2.345%"},
             {"Sharpe Ratio", "-0.867"},
             {"Probabilistic Sharpe Ratio", "0.001%"},
             {"Loss Rate", "50%"},
             {"Win Rate", "50%"},
-            {"Profit-Loss Ratio", "0.90"},
+            {"Profit-Loss Ratio", "0.89"},
             {"Alpha", "-0.014"},
             {"Beta", "0.001"},
             {"Annual Standard Deviation", "0.016"},
@@ -199,7 +199,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Information Ratio", "-0.603"},
             {"Tracking Error", "0.291"},
             {"Treynor Ratio", "-13.292"},
-            {"Total Fees", "$3.70"},
+            {"Total Fees", "$7.40"},
             {"Estimated Strategy Capacity", "$45000000.00"},
             {"Lowest Capacity Asset", "ES XFH59UK0MYO1"},
             {"Fitness Score", "0.005"},
@@ -221,7 +221,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "67d8ad460ff796937ee252c3e4340e62"}
+            {"OrderListHash", "0128b145984582f5eba7e95881d9b62d"}
         };
     }
 }

Algorithm.CSharp/IndexOptionCallITMExpiryRegressionAlgorithm.cs

@@ -195,12 +195,12 @@ namespace QuantConnect.Algorithm.CSharp
             {"Win Rate", "0%"},
             {"Profit-Loss Ratio", "0"},
             {"Alpha", "1.836"},
-            {"Beta", "-0.228"},
+            {"Beta", "-0.217"},
             {"Annual Standard Deviation", "0.345"},
             {"Annual Variance", "0.119"},
-            {"Information Ratio", "4.653"},
-            {"Tracking Error", "0.383"},
-            {"Treynor Ratio", "-8.001"},
+            {"Information Ratio", "4.66"},
+            {"Tracking Error", "0.382"},
+            {"Treynor Ratio", "-8.405"},
             {"Total Fees", "$0.00"},
             {"Estimated Strategy Capacity", "$0"},
             {"Lowest Capacity Asset", "SPX XL80P3GHDZXQ|SPX 31"},
@@ -223,7 +223,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "25cc4301125ffaa12dd8d8f4387adf06"}
+            {"OrderListHash", "6dda4f153b0be9fdf55026da439f90f6"}
         };
     }
 }

Algorithm.CSharp/IndexOptionCallITMGreeksExpiryRegressionAlgorithm.cs

@@ -173,13 +173,13 @@ namespace QuantConnect.Algorithm.CSharp
             {"Loss Rate", "100%"},
             {"Win Rate", "0%"},
             {"Profit-Loss Ratio", "0"},
-            {"Alpha", "2.169"},
-            {"Beta", "-0.238"},
+            {"Alpha", "2.168"},
+            {"Beta", "-0.226"},
             {"Annual Standard Deviation", "0.373"},
             {"Annual Variance", "0.139"},
-            {"Information Ratio", "5.17"},
-            {"Tracking Error", "0.409"},
-            {"Treynor Ratio", "-9.071"},
+            {"Information Ratio", "5.177"},
+            {"Tracking Error", "0.408"},
+            {"Treynor Ratio", "-9.544"},
             {"Total Fees", "$0.00"},
             {"Estimated Strategy Capacity", "$44000000.00"},
             {"Lowest Capacity Asset", "SPX XL80P3GHDZXQ|SPX 31"},
@@ -202,7 +202,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "bd96db56c80107572e8fc13c8794279b"}
+            {"OrderListHash", "3cccf8c2409ee8a9020ba79a6c45742a"}
         };
     }
 }

Algorithm.CSharp/IndexOptionCallOTMExpiryRegressionAlgorithm.cs

@@ -190,7 +190,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Annual Variance", "0"},
             {"Information Ratio", "-0.334"},
             {"Tracking Error", "0.138"},
-            {"Treynor Ratio", "-9.47"},
+            {"Treynor Ratio", "-3.273"},
             {"Total Fees", "$0.00"},
             {"Estimated Strategy Capacity", "$22000.00"},
             {"Lowest Capacity Asset", "SPX XL80P59H5E6M|SPX 31"},
@@ -213,7 +213,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "e971949fb0d1cc8d034ad7cba97d09cc"}
+            {"OrderListHash", "177477b78d9264384a09f1607e8c7d11"}
         };
     }
 }

Algorithm.CSharp/IndexOptionPutITMExpiryRegressionAlgorithm.cs

@@ -199,13 +199,13 @@ namespace QuantConnect.Algorithm.CSharp
             {"Loss Rate", "100%"},
             {"Win Rate", "0%"},
             {"Profit-Loss Ratio", "0"},
-            {"Alpha", "-0.683"},
-            {"Beta", "0.218"},
+            {"Alpha", "-0.682"},
+            {"Beta", "0.204"},
             {"Annual Standard Deviation", "0.356"},
             {"Annual Variance", "0.126"},
-            {"Information Ratio", "-1.94"},
-            {"Tracking Error", "0.37"},
-            {"Treynor Ratio", "-3.083"},
+            {"Information Ratio", "-1.936"},
+            {"Tracking Error", "0.371"},
+            {"Treynor Ratio", "-3.293"},
             {"Total Fees", "$0.00"},
             {"Estimated Strategy Capacity", "$0"},
             {"Lowest Capacity Asset", "SPX 31KC0UJHC75TA|SPX 31"},
@@ -228,7 +228,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "6df8b200489a2f217cd514592ee98663"}
+            {"OrderListHash", "f2d98f952cc0b54fb620795018682105"}
         };
     }
 }

Algorithm.CSharp/IndexOptionPutOTMExpiryRegressionAlgorithm.cs

@@ -189,7 +189,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Annual Variance", "0"},
             {"Information Ratio", "-0.595"},
             {"Tracking Error", "0.137"},
-            {"Treynor Ratio", "-5.349"},
+            {"Treynor Ratio", "-5.196"},
             {"Total Fees", "$0.00"},
             {"Estimated Strategy Capacity", "$0"},
             {"Lowest Capacity Asset", "SPX 31KC0UJFONTBI|SPX 31"},
@@ -212,7 +212,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "b0c50080f0229facd065721f1f5d715e"}
+            {"OrderListHash", "6faffe52c64c2148458af1d2deb68a6f"}
         };
     }
 }

Algorithm.CSharp/IndexOptionShortCallITMExpiryRegressionAlgorithm.cs

@@ -185,13 +185,13 @@ namespace QuantConnect.Algorithm.CSharp
             {"Loss Rate", "0%"},
             {"Win Rate", "100%"},
             {"Profit-Loss Ratio", "0"},
-            {"Alpha", "-0.671"},
-            {"Beta", "0.211"},
+            {"Alpha", "-0.67"},
+            {"Beta", "0.197"},
             {"Annual Standard Deviation", "0.344"},
             {"Annual Variance", "0.118"},
-            {"Information Ratio", "-1.963"},
-            {"Tracking Error", "0.36"},
-            {"Treynor Ratio", "-3.133"},
+            {"Information Ratio", "-1.96"},
+            {"Tracking Error", "0.361"},
+            {"Treynor Ratio", "-3.349"},
             {"Total Fees", "$0.00"},
             {"Estimated Strategy Capacity", "$0"},
             {"Lowest Capacity Asset", "SPX XL80P3GHDZXQ|SPX 31"},
@@ -214,7 +214,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "856448f4cbba4fc39af8dba369c054df"}
+            {"OrderListHash", "439042b39981ea246e50728cc57c31c7"}
         };
     }
 }

Algorithm.CSharp/IndexOptionShortCallOTMExpiryRegressionAlgorithm.cs

@@ -183,7 +183,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Annual Variance", "0"},
             {"Information Ratio", "-0.32"},
             {"Tracking Error", "0.138"},
-            {"Treynor Ratio", "-9.479"},
+            {"Treynor Ratio", "-3.277"},
             {"Total Fees", "$0.00"},
             {"Estimated Strategy Capacity", "$22000.00"},
             {"Lowest Capacity Asset", "SPX XL80P59H5E6M|SPX 31"},
@@ -206,7 +206,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "76ffdfc100ba7778009e35966bd92cfc"}
+            {"OrderListHash", "b55e2b2bd35bc3200e228b4e6e77dd90"}
         };
     }
 }

Algorithm.CSharp/IndexOptionShortPutITMExpiryRegressionAlgorithm.cs

@@ -190,12 +190,12 @@ namespace QuantConnect.Algorithm.CSharp
             {"Win Rate", "100%"},
             {"Profit-Loss Ratio", "0"},
             {"Alpha", "1.938"},
-            {"Beta", "-0.235"},
+            {"Beta", "-0.224"},
             {"Annual Standard Deviation", "0.356"},
             {"Annual Variance", "0.127"},
-            {"Information Ratio", "4.787"},
+            {"Information Ratio", "4.793"},
             {"Tracking Error", "0.393"},
-            {"Treynor Ratio", "-8.187"},
+            {"Treynor Ratio", "-8.593"},
             {"Total Fees", "$0.00"},
             {"Estimated Strategy Capacity", "$0"},
             {"Lowest Capacity Asset", "SPX 31KC0UJHC75TA|SPX 31"},
@@ -218,7 +218,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "77d9040316634cb6b9e8cd2e4e192fd5"}
+            {"OrderListHash", "f243341674cb1486d7cf009d74d4e6ff"}
         };
     }
 }

Algorithm.CSharp/IndexOptionShortPutOTMExpiryRegressionAlgorithm.cs

@@ -182,7 +182,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Annual Variance", "0"},
             {"Information Ratio", "-0.066"},
             {"Tracking Error", "0.139"},
-            {"Treynor Ratio", "-4.733"},
+            {"Treynor Ratio", "-4.611"},
             {"Total Fees", "$0.00"},
             {"Estimated Strategy Capacity", "$0"},
             {"Lowest Capacity Asset", "SPX 31KC0UJFONTBI|SPX 31"},
@@ -205,7 +205,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "12861ef440f68994997aeb24c8027748"}
+            {"OrderListHash", "636b79bb5bf3db20eeda02ccf1064d07"}
         };
     }
 }

Algorithm.CSharp/OptionAssignmentRegressionAlgorithm.cs

@@ -42,7 +42,7 @@ namespace QuantConnect.Algorithm.CSharp
         public override void Initialize()
         {
             SetStartDate(2015, 12, 23);
-            SetEndDate(2015, 12, 28);
+            SetEndDate(2015, 12, 24);
             SetCash(100000);
             Stock = AddEquity("GOOG", Resolution.Minute);
 
@@ -78,33 +78,33 @@ namespace QuantConnect.Algorithm.CSharp
         public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
             {"Total Trades", "24"},
-            {"Average Win", "9.60%"},
-            {"Average Loss", "-16.86%"},
-            {"Compounding Annual Return", "-75.533%"},
-            {"Drawdown", "2.300%"},
-            {"Expectancy", "0.046"},
-            {"Net Profit", "-2.162%"},
-            {"Sharpe Ratio", "-6.761"},
-            {"Probabilistic Sharpe Ratio", "1.125%"},
-            {"Loss Rate", "33%"},
-            {"Win Rate", "67%"},
-            {"Profit-Loss Ratio", "0.57"},
-            {"Alpha", "-0.01"},
-            {"Beta", "0.455"},
-            {"Annual Standard Deviation", "0.014"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "0%"},
+            {"Drawdown", "0%"},
+            {"Expectancy", "0"},
+            {"Net Profit", "0%"},
+            {"Sharpe Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "0%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
             {"Annual Variance", "0"},
-            {"Information Ratio", "6.047"},
-            {"Tracking Error", "0.015"},
-            {"Treynor Ratio", "-0.207"},
+            {"Information Ratio", "0"},
+            {"Tracking Error", "0"},
+            {"Treynor Ratio", "0"},
             {"Total Fees", "$12.00"},
-            {"Estimated Strategy Capacity", "$1100000.00"},
-            {"Lowest Capacity Asset", "GOOCV 305RBQ20WHPNQ|GOOCV VP83T1ZUHROL"},
-            {"Fitness Score", "0.057"},
+            {"Estimated Strategy Capacity", "$310000.00"},
+            {"Lowest Capacity Asset", "GOOCV VP83T1ZUHROL"},
+            {"Fitness Score", "0.5"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "-3.876"},
-            {"Return Over Maximum Drawdown", "-35.706"},
-            {"Portfolio Turnover", "3.258"},
+            {"Sortino Ratio", "79228162514264337593543950335"},
+            {"Return Over Maximum Drawdown", "-50.725"},
+            {"Portfolio Turnover", "8.14"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
             {"Total Insights Analysis Completed", "0"},
@@ -118,7 +118,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "a0e8eeee1c31968b773ebdf47bb996df"}
+            {"OrderListHash", "58557574cf0489dd38fb37768f509ca1"}
         };
     }
 }

Algorithm.CSharp/OptionChainConsistencyRegressionAlgorithm.cs

@@ -162,7 +162,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "b657be3f1b7f7f59cc7f260c99a89d00"}
+            {"OrderListHash", "6f6d53762c0bb64467ee6215b2aa57c9"}
         };
     }
 }

Algorithm.CSharp/OptionExerciseAssignRegressionAlgorithm.cs

@@ -124,16 +124,16 @@ namespace QuantConnect.Algorithm.CSharp
         {
             {"Total Trades", "4"},
             {"Average Win", "0.30%"},
-            {"Average Loss", "-0.32%"},
+            {"Average Loss", "-0.33%"},
             {"Compounding Annual Return", "-24.104%"},
             {"Drawdown", "0.400%"},
-            {"Expectancy", "-0.359"},
+            {"Expectancy", "-0.358"},
             {"Net Profit", "-0.352%"},
             {"Sharpe Ratio", "0"},
             {"Probabilistic Sharpe Ratio", "0%"},
             {"Loss Rate", "67%"},
             {"Win Rate", "33%"},
-            {"Profit-Loss Ratio", "0.92"},
+            {"Profit-Loss Ratio", "0.93"},
             {"Alpha", "0"},
             {"Beta", "0"},
             {"Annual Standard Deviation", "0"},
@@ -163,7 +163,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "c764f59736091ece264bd2f959eae97c"}
+            {"OrderListHash", "091c92055026a8323accb4508a68bf3f"}
         };
     }
 }

Algorithm.CSharp/OptionRenameDailyRegressionAlgorithm.cs

@@ -1,202 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
-*/
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using QuantConnect.Data;
-using QuantConnect.Data.Market;
-using QuantConnect.Interfaces;
-using QuantConnect.Orders;
-
-namespace QuantConnect.Algorithm.CSharp
-{
-    /// <summary>
-    /// This is an option split regression algorithm
-    /// </summary>
-    /// <meta name="tag" content="options" />
-    /// <meta name="tag" content="regression test" />
-    public class OptionRenameDailyRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
-    {
-        private Symbol _optionSymbol;
-        private Symbol _contractSymbol;
-        private Symbol _underlyingSymbol;
-
-        public override void Initialize()
-        {
-            // this test opens position in the first day of trading, lives through stock rename (NWSA->FOXA), dividends, and closes adjusted position on the third day
-            SetStartDate(2013, 06, 27);
-            SetEndDate(2013, 07, 02);
-            SetCash(1000000);
-
-            var option = AddOption("NWSA", Resolution.Daily);
-            _optionSymbol = option.Symbol;
-
-            // set our strike/expiry filter for this option chain
-            option.SetFilter(-1, +1, TimeSpan.Zero, TimeSpan.MaxValue);
-
-            // use the underlying equity as the benchmark
-            SetBenchmark("NWSA");
-        }
-
-        /// <summary>
-        /// Event - v3.0 DATA EVENT HANDLER: (Pattern) Basic template for user to override for receiving all subscription data in a single event
-        /// </summary>
-        /// <param name="slice">The current slice of data keyed by symbol string</param>
-        public override void OnData(Slice slice)
-        {
-            foreach (var dividend in slice.Dividends.Values)
-            {
-                if (dividend.ReferencePrice != 32.6m || dividend.Distribution != 3.82m)
-                {
-                    throw new Exception($"{Time} - Invalid dividend {dividend}");
-                }
-            }
-            if (!Portfolio.Invested)
-            {
-                OptionChain chain;
-                if (slice.OptionChains.TryGetValue(_optionSymbol, out chain))
-                {
-                    var contract =
-                        chain.OrderBy(x => x.Expiry)
-                        .Where(x => x.Right == OptionRight.Call && x.Strike == 33 && x.Expiry.Date == new DateTime(2013, 08, 17))
-                        .FirstOrDefault();
-
-                    if (contract != null)
-                    {
-                        // Buying option
-                        _contractSymbol = contract.Symbol;
-                        Buy(_contractSymbol, 1);
-
-                        // Buying the underlying stock
-                        _underlyingSymbol = contract.Symbol.Underlying;
-                        Buy(_underlyingSymbol, 100);
-
-                        // Check
-                        if (slice.Time != new DateTime(2013, 6, 28))
-                        {
-                            throw new Exception(@"Received first contract at {slice.Time}; Expected at 6/28/2013 12AM.");
-                        }
-
-                        if (contract.AskPrice != 1.15m)
-                        {
-                            throw new Exception("Current ask price was not loaded from NWSA backtest file and is not $1.1");
-                        }
-
-                        if (contract.UnderlyingSymbol.Value != "NWSA")
-                        {
-                            throw new Exception("Contract underlying symbol was not NWSA as expected");
-                        }
-                    }
-                }
-            }
-            else if (slice.Time.Day == 3) // Final day
-            {
-                // selling positions
-                Liquidate();
-
-                // checks
-                OptionChain chain;
-                if (slice.OptionChains.TryGetValue(_optionSymbol, out chain))
-                {
-                    if (chain.Underlying.Symbol.Value != "FOXA")
-                    {
-                        throw new Exception("Chain underlying symbol was not FOXA as expected");
-                    }
-
-                    var contract =
-                        chain.OrderBy(x => x.Expiry)
-                        .Where(x => x.Right == OptionRight.Call && x.Strike == 33 && x.Expiry.Date == new DateTime(2013, 08, 17))
-                        .FirstOrDefault();
-
-                    if (contract.BidPrice != 0.05m)
-                    {
-                        throw new Exception("Current bid price was not loaded from FOXA file and is not $0.05");
-                    }
-                }
-            }
-        }
-
-        /// <summary>
-        /// Order fill event handler. On an order fill update the resulting information is passed to this method.
-        /// </summary>
-        /// <param name="orderEvent">Order event details containing details of the evemts</param>
-        /// <remarks>This method can be called asynchronously and so should only be used by seasoned C# experts. Ensure you use proper locks on thread-unsafe objects</remarks>
-        public override void OnOrderEvent(OrderEvent orderEvent)
-        {
-            Log(orderEvent.ToString());
-        }
-
-        /// <summary>
-        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
-        /// </summary>
-        public bool CanRunLocally { get; } = true;
-
-        /// <summary>
-        /// This is used by the regression test system to indicate which languages this algorithm is written in.
-        /// </summary>
-        public Language[] Languages { get; } = { Language.CSharp };
-
-        /// <summary>
-        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
-        /// </summary>
-        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
-        {
-            {"Total Trades", "2"},
-            {"Average Win", "0%"},
-            {"Average Loss", "0%"},
-            {"Compounding Annual Return", "-0.273%"},
-            {"Drawdown", "0.000%"},
-            {"Expectancy", "0"},
-            {"Net Profit", "-0.004%"},
-            {"Sharpe Ratio", "-2.264"},
-            {"Probabilistic Sharpe Ratio", "32.662%"},
-            {"Loss Rate", "0%"},
-            {"Win Rate", "0%"},
-            {"Profit-Loss Ratio", "0"},
-            {"Alpha", "0"},
-            {"Beta", "0"},
-            {"Annual Standard Deviation", "0.001"},
-            {"Annual Variance", "0"},
-            {"Information Ratio", "-2.264"},
-            {"Tracking Error", "0.001"},
-            {"Treynor Ratio", "0"},
-            {"Total Fees", "$2.00"},
-            {"Estimated Strategy Capacity", "$0"},
-            {"Lowest Capacity Asset", "NWSA VJ5IKAXU7WBQ|NWSA T3MO1488O0H1"},
-            {"Fitness Score", "0"},
-            {"Kelly Criterion Estimate", "0"},
-            {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "-1.168"},
-            {"Return Over Maximum Drawdown", "-6.338"},
-            {"Portfolio Turnover", "0"},
-            {"Total Insights Generated", "0"},
-            {"Total Insights Closed", "0"},
-            {"Total Insights Analysis Completed", "0"},
-            {"Long Insight Count", "0"},
-            {"Short Insight Count", "0"},
-            {"Long/Short Ratio", "100%"},
-            {"Estimated Monthly Alpha Value", "$0"},
-            {"Total Accumulated Estimated Alpha Value", "$0"},
-            {"Mean Population Estimated Insight Value", "$0"},
-            {"Mean Population Direction", "0%"},
-            {"Mean Population Magnitude", "0%"},
-            {"Rolling Averaged Population Direction", "0%"},
-            {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "8b89135535d842f6df7b2849d6604fbd"}
-        };
-    }
-}

Algorithm.CSharp/RawDataRegressionAlgorithm.cs

@@ -32,7 +32,7 @@ namespace QuantConnect.Algorithm.CSharp
     public class RawDataRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
         private const string Ticker = "GOOGL";
-        private CorporateFactorProvider _factorFile;
+        private FactorFile _factorFile;
         private readonly IEnumerator<decimal> _expectedRawPrices = new List<decimal> { 1157.93m, 1158.72m,
             1131.97m, 1114.28m, 1120.15m, 1114.51m, 1134.89m, 567.55m, 571.50m, 545.25m, 540.63m }.GetEnumerator();
         private Symbol _googl;
@@ -56,7 +56,7 @@ namespace QuantConnect.Algorithm.CSharp
             mapFileProvider.Initialize(dataProvider);
             var factorFileProvider = new LocalDiskFactorFileProvider();
             factorFileProvider.Initialize(mapFileProvider, dataProvider);
-            _factorFile = factorFileProvider.Get(_googl) as CorporateFactorProvider;
+            _factorFile = factorFileProvider.Get(_googl);
 
             // Prime our expected values
             _expectedRawPrices.MoveNext();
@@ -81,7 +81,7 @@ namespace QuantConnect.Algorithm.CSharp
                 if (_expectedRawPrices.Current != googlData.Close)
                 {
                     // Our values don't match lets try and give a reason why
-                    var dayFactor = _factorFile.GetPriceFactor(googlData.Time, DataNormalizationMode.Adjusted);
+                    var dayFactor = _factorFile.GetPriceScaleFactor(googlData.Time);
                     var probableRawPrice = googlData.Close / dayFactor; // Undo adjustment
 
                     if (_expectedRawPrices.Current == probableRawPrice)

Algorithm.CSharp/RegressionTests/CustomData/CustomDataIconicTypesDefaultResolutionRegressionAlgorithm.cs

@@ -69,6 +69,16 @@ namespace QuantConnect.Algorithm.CSharp
                 throw new Exception("Was expecting resolution to be set to Hour");
             }
 
+            try
+            {
+                AddOption("AAPL", Resolution.Daily);
+                throw new Exception("Was expecting an ArgumentException to be thrown");
+            }
+            catch (ArgumentException)
+            {
+                // expected, options only support minute resolution
+            }
+
             var option = AddOption("AAPL");
             if (SubscriptionManager.SubscriptionDataConfigService.GetSubscriptionDataConfigs(option.Symbol)
                 .Any(config => config.Resolution != Resolution.Minute))

Algorithm.CSharp/UniverseSelectionRegressionAlgorithm.cs

@@ -173,32 +173,32 @@ namespace QuantConnect.Algorithm.CSharp
         public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
             {"Total Trades", "4"},
-            {"Average Win", "0.71%"},
+            {"Average Win", "0.64%"},
             {"Average Loss", "0%"},
-            {"Compounding Annual Return", "-55.953%"},
-            {"Drawdown", "3.700%"},
+            {"Compounding Annual Return", "-56.617%"},
+            {"Drawdown", "3.800%"},
             {"Expectancy", "0"},
-            {"Net Profit", "-3.747%"},
-            {"Sharpe Ratio", "-2.668"},
-            {"Probabilistic Sharpe Ratio", "13.421%"},
+            {"Net Profit", "-3.815%"},
+            {"Sharpe Ratio", "-2.708"},
+            {"Probabilistic Sharpe Ratio", "13.091%"},
             {"Loss Rate", "0%"},
             {"Win Rate", "100%"},
             {"Profit-Loss Ratio", "0"},
-            {"Alpha", "-0.268"},
-            {"Beta", "1.241"},
+            {"Alpha", "-0.275"},
+            {"Beta", "1.237"},
             {"Annual Standard Deviation", "0.167"},
             {"Annual Variance", "0.028"},
-            {"Information Ratio", "-2.443"},
+            {"Information Ratio", "-2.491"},
             {"Tracking Error", "0.124"},
-            {"Treynor Ratio", "-0.358"},
-            {"Total Fees", "$3.00"},
+            {"Treynor Ratio", "-0.365"},
+            {"Total Fees", "$4.00"},
             {"Estimated Strategy Capacity", "$870000.00"},
             {"Lowest Capacity Asset", "GOOAV VP83T1ZUHROL"},
             {"Fitness Score", "0.008"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "-3.793"},
-            {"Return Over Maximum Drawdown", "-14.933"},
+            {"Sortino Ratio", "-3.836"},
+            {"Return Over Maximum Drawdown", "-14.841"},
             {"Portfolio Turnover", "0.135"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
@@ -213,7 +213,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "afde32cdef5e61707f85d62f62eece17"}
+            {"OrderListHash", "bbc9f982183f74b34be4529acaa33fe8"}
         };
     }
 }

Algorithm.Framework/Alphas/AlphaStreamAlphaModule.cs

@@ -1,120 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System.Linq;
-using QuantConnect.Data;
-using System.Collections.Generic;
-using QuantConnect.Data.UniverseSelection;
-using QuantConnect.Data.Custom.AlphaStreams;
-
-namespace QuantConnect.Algorithm.Framework.Alphas
-{
-    /// <summary>
-    /// Alpha model that will handle adding and removing securities from the algorithm based on the current portfolio of the different alphas
-    /// </summary>
-    public sealed class AlphaStreamAlphaModule : AlphaModel
-    {
-        private Dictionary<Symbol, HashSet<Symbol>> _symbolsPerAlpha = new Dictionary<Symbol, HashSet<Symbol>>();
-
-        /// <summary>
-        /// Initialize new <see cref="AlphaStreamAlphaModule"/>
-        /// </summary>
-        public AlphaStreamAlphaModule(string name = null)
-        {
-            Name = name ?? "AlphaStreamAlphaModule";
-        }
-
-        /// <summary>
-        /// Updates this alpha model with the latest data from the algorithm.
-        /// This is called each time the algorithm receives data for subscribed securities
-        /// </summary>
-        /// <param name="algorithm">The algorithm instance</param>
-        /// <param name="data">The new data available</param>
-        /// <returns>The new insights generated</returns>
-        public override IEnumerable<Insight> Update(QCAlgorithm algorithm, Slice data)
-        {
-            foreach (var portfolioState in data.Get<AlphaStreamsPortfolioState>().Values)
-            {
-                ProcessPortfolioState(algorithm, portfolioState);
-            }
-
-            return Enumerable.Empty<Insight>();
-        }
-
-        /// <summary>
-        /// Event fired each time the we add/remove securities from the data feed
-        /// </summary>
-        /// <param name="algorithm">The algorithm instance that experienced the change in securities</param>
-        /// <param name="changes">The security additions and removals from the algorithm</param>
-        public override void OnSecuritiesChanged(QCAlgorithm algorithm, SecurityChanges changes)
-        {
-            changes.FilterCustomSecurities = false;
-            foreach (var addedSecurity in changes.AddedSecurities)
-            {
-                if (addedSecurity.Symbol.IsCustomDataType<AlphaStreamsPortfolioState>())
-                {
-                    if (!_symbolsPerAlpha.ContainsKey(addedSecurity.Symbol))
-                    {
-                        _symbolsPerAlpha[addedSecurity.Symbol] = new HashSet<Symbol>();
-                    }
-                    // warmup alpha state, adding target securities
-                    ProcessPortfolioState(algorithm, addedSecurity.Cache.GetData<AlphaStreamsPortfolioState>());
-                }
-            }
-
-            algorithm.Log($"OnSecuritiesChanged: {changes}");
-        }
-
-        /// <summary>
-        /// Will handle adding and removing securities from the algorithm based on the current portfolio of the different alphas
-        /// </summary>
-        private void ProcessPortfolioState(QCAlgorithm algorithm, AlphaStreamsPortfolioState portfolioState)
-        {
-            if (portfolioState == null)
-            {
-                return;
-            }
-
-            var alphaId = portfolioState.Symbol;
-            if (!_symbolsPerAlpha.TryGetValue(alphaId, out var currentSymbols))
-            {
-                _symbolsPerAlpha[alphaId] = currentSymbols = new HashSet<Symbol>();
-            }
-
-            var newSymbols = new HashSet<Symbol>(currentSymbols.Count);
-            foreach (var symbol in portfolioState.PositionGroups?.SelectMany(positionGroup => positionGroup.Positions).Select(state => state.Symbol) ?? Enumerable.Empty<Symbol>())
-            {
-                // only add it if it's not used by any alpha (already added check)
-                if (newSymbols.Add(symbol) && !UsedBySomeAlpha(symbol))
-                {
-                    algorithm.AddSecurity(symbol,
-                        resolution: algorithm.UniverseSettings.Resolution,
-                        extendedMarketHours: algorithm.UniverseSettings.ExtendedMarketHours);
-                }
-            }
-            _symbolsPerAlpha[alphaId] = newSymbols;
-
-            foreach (var symbol in currentSymbols.Where(symbol => !UsedBySomeAlpha(symbol)))
-            {
-                algorithm.RemoveSecurity(symbol);
-            }
-        }
-
-        private bool UsedBySomeAlpha(Symbol asset)
-        {
-            return _symbolsPerAlpha.Any(pair => pair.Value.Contains(asset));
-        }
-    }
-}

Algorithm.Framework/Alphas/MacdAlphaModel.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -35,7 +35,7 @@ namespace QuantConnect.Algorithm.Framework.Alphas
         private readonly MovingAverageType _movingAverageType;
         private readonly Resolution _resolution;
         private const decimal BounceThresholdPercent = 0.01m;
-        protected readonly Dictionary<Symbol, SymbolData> _symbolData;
+        private readonly Dictionary<Symbol, SymbolData> _symbolData;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="MacdAlphaModel"/> class
@@ -130,7 +130,7 @@ namespace QuantConnect.Algorithm.Framework.Alphas
             }
         }
 
-        public class SymbolData
+        class SymbolData
         {
             public InsightDirection? PreviousDirection { get; set; }
 
@@ -147,7 +147,6 @@ namespace QuantConnect.Algorithm.Framework.Alphas
                 MACD = new MovingAverageConvergenceDivergence(fastPeriod, slowPeriod, signalPeriod, movingAverageType);
 
                 algorithm.RegisterIndicator(security.Symbol, MACD, Consolidator);
-                algorithm.WarmUpIndicator(security.Symbol, MACD, resolution);
             }
         }
     }

Algorithm.Framework/Alphas/MacdAlphaModel.py

@@ -1,4 +1,4 @@
-# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
 # Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -98,6 +98,5 @@ class SymbolData:
 
         self.Consolidator = algorithm.ResolveConsolidator(security.Symbol, resolution)
         algorithm.RegisterIndicator(security.Symbol, self.MACD, self.Consolidator)
-        algorithm.WarmUpIndicator(security.Symbol, self.MACD, resolution)
 
         self.PreviousDirection = None

Algorithm.Framework/CustomWeightingAlphaStreamsPortfolioConstructionModel.cs

@@ -1,51 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System.Linq;
-using QuantConnect.Logging;
-using System.Collections.Generic;
-using QuantConnect.Algorithm.Framework.Portfolio;
-
-namespace QuantConnect.Algorithm.Framework
-{
-    /// <summary>
-    /// Custom weighting alpha streams portfolio construction model that will generate aggregated security targets taking into account all the alphas positions
-    /// and a custom weighting factor for each alpha, which is also factored by the relation of the alphas portfolio value and the current algorithms portfolio value
-    /// </summary>
-    public class CustomWeightingAlphaStreamsPortfolioConstructionModel : EqualWeightingAlphaStreamsPortfolioConstructionModel
-    {
-        private Dictionary<string, decimal> _alphaWeights;
-
-        /// <summary>
-        /// Specify a custom set of alpha portfolio weights to use
-        /// </summary>
-        /// <param name="alphaWeights">The alpha portfolio weights</param>
-        public void SetAlphaWeights(Dictionary<string, decimal> alphaWeights)
-        {
-            Log.Trace($"CustomWeightingAlphaStreamsPortfolioConstructionModel.SetAlphaWeights(): new weights: [{string.Join(",", alphaWeights.Select(pair => $"{pair.Key}:{pair.Value}"))}]");
-            _alphaWeights = alphaWeights;
-        }
-
-        /// <summary>
-        /// Get's the weight for an alpha
-        /// </summary>
-        /// <param name="alphaId">The algorithm instance that experienced the change in securities</param>
-        /// <returns>The alphas weight</returns>
-        public override decimal GetAlphaWeight(string alphaId)
-        {
-            return !_alphaWeights.TryGetValue(alphaId, out var alphaWeight) ? 0 : alphaWeight;
-        }
-    }
-}

Algorithm.Framework/Portfolio/AlphaStreamsPortfolioConstructionModel.cs

@@ -1,58 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System.Collections.Generic;
-using QuantConnect.Data.UniverseSelection;
-using QuantConnect.Algorithm.Framework.Alphas;
-
-namespace QuantConnect.Algorithm.Framework.Portfolio
-{
-    /// <summary>
-    /// Base alpha streams portfolio construction model
-    /// </summary>
-    public class AlphaStreamsPortfolioConstructionModel : IPortfolioConstructionModel
-    {
-        /// <summary>
-        /// Get's the weight for an alpha
-        /// </summary>
-        /// <param name="alphaId">The algorithm instance that experienced the change in securities</param>
-        /// <returns>The alphas weight</returns>
-        public virtual decimal GetAlphaWeight(string alphaId)
-        {
-            throw new System.NotImplementedException();
-        }
-
-        /// <summary>
-        /// Event fired each time the we add/remove securities from the data feed
-        /// </summary>
-        /// <param name="algorithm">The algorithm instance that experienced the change in securities</param>
-        /// <param name="changes">The security additions and removals from the algorithm</param>
-        public virtual void OnSecuritiesChanged(QCAlgorithm algorithm, SecurityChanges changes)
-        {
-            throw new System.NotImplementedException();
-        }
-
-        /// <summary>
-        /// Create portfolio targets from the specified insights
-        /// </summary>
-        /// <param name="algorithm">The algorithm instance</param>
-        /// <param name="insights">The insights to create portfolio targets from</param>
-        /// <returns>An enumerable of portfolio targets to be sent to the execution model</returns>
-        public virtual IEnumerable<IPortfolioTarget> CreateTargets(QCAlgorithm algorithm, Insight[] insights)
-        {
-            throw new System.NotImplementedException();
-        }
-    }
-}

Algorithm.Framework/Portfolio/EqualWeightingAlphaStreamsPortfolioConstructionModel.cs

@@ -29,7 +29,7 @@ namespace QuantConnect.Algorithm.Framework.Portfolio
     /// and an equal weighting factor for each alpha, which is also factored by the relation of the alphas portfolio value and the current algorithms portfolio value,
     /// overriding <see cref="GetAlphaWeight"/> allows custom weighting implementations
     /// </summary>
-    public class EqualWeightingAlphaStreamsPortfolioConstructionModel : AlphaStreamsPortfolioConstructionModel
+    public class EqualWeightingAlphaStreamsPortfolioConstructionModel : IPortfolioConstructionModel
     {
         private bool _rebalance;
         private Dictionary<Symbol, PortfolioTarget> _targetsPerSymbol;
@@ -48,7 +48,7 @@ namespace QuantConnect.Algorithm.Framework.Portfolio
         /// <param name="algorithm">The algorithm instance</param>
         /// <param name="insights">The insights to create portfolio targets from</param>
         /// <returns>An enumerable of portfolio targets to be sent to the execution model</returns>
-        public override IEnumerable<IPortfolioTarget> CreateTargets(QCAlgorithm algorithm, Insight[] insights)
+        public IEnumerable<IPortfolioTarget> CreateTargets(QCAlgorithm algorithm, Insight[] insights)
         {
             if (_targetsPerSymbol == null)
             {
@@ -104,22 +104,12 @@ namespace QuantConnect.Algorithm.Framework.Portfolio
             }
         }
 
-        /// <summary>
-        /// Get's the weight for an alpha
-        /// </summary>
-        /// <param name="alphaId">The algorithm instance that experienced the change in securities</param>
-        /// <returns>The alphas weight</returns>
-        public override decimal GetAlphaWeight(string alphaId)
-        {
-            return 1m / _targetsPerSymbolPerAlpha.Count;
-        }
-
         /// <summary>
         /// Event fired each time the we add/remove securities from the data feed
         /// </summary>
         /// <param name="algorithm">The algorithm instance that experienced the change in securities</param>
         /// <param name="changes">The security additions and removals from the algorithm</param>
-        public override void OnSecuritiesChanged(QCAlgorithm algorithm, SecurityChanges changes)
+        public void OnSecuritiesChanged(QCAlgorithm algorithm, SecurityChanges changes)
         {
             changes.FilterCustomSecurities = false;
 
@@ -163,10 +153,10 @@ namespace QuantConnect.Algorithm.Framework.Portfolio
         /// Determines the portfolio weight to give a specific alpha. Default implementation just returns equal weighting
         /// </summary>
         /// <param name="portfolioState">The alphas portfolio state to get the weight for</param>
-        /// <param name="totalUsablePortfolioValue">This algorithms usable total portfolio value, removing the free portfolio value</param>
+        /// <param name="totalUsablePortfolioValue">This algorithms usable total portfolio value</param>
         /// <param name="cashBook">This algorithms cash book</param>
         /// <returns>The weight to use on this alphas positions</returns>
-        private decimal GetAlphaWeight(AlphaStreamsPortfolioState portfolioState,
+        protected virtual decimal GetAlphaWeight(AlphaStreamsPortfolioState portfolioState,
             decimal totalUsablePortfolioValue,
             CashBook cashBook)
         {
@@ -178,7 +168,8 @@ namespace QuantConnect.Algorithm.Framework.Portfolio
                 return 0;
             }
 
-            return totalUsablePortfolioValue * GetAlphaWeight(portfolioState.AlphaId) / alphaPortfolioValueInOurAccountCurrency;
+            var equalWeightFactor = 1m / _targetsPerSymbolPerAlpha.Count;
+            return totalUsablePortfolioValue * equalWeightFactor / alphaPortfolioValueInOurAccountCurrency;
         }
 
         private bool ProcessPortfolioState(AlphaStreamsPortfolioState portfolioState, QCAlgorithm algorithm)

Algorithm.Python/BasicTemplateIndexAlgorithm.py

@@ -16,7 +16,7 @@ from AlgorithmImports import *
 class BasicTemplateIndexAlgorithm(QCAlgorithm):
     def Initialize(self) -> None:
         self.SetStartDate(2021, 1, 4)
-        self.SetEndDate(2021, 1, 18)
+        self.SetEndDate(2021, 1, 15)
         self.SetCash(1000000)
 
         # Use indicator for signal; but it cannot be traded

Algorithm.Python/BasicTemplateOptionsDailyAlgorithm.py

@@ -1,73 +0,0 @@
-# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
-# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from AlgorithmImports import *
-
-### <summary>
-### This example demonstrates how to add options for a given underlying equity security.
-### It also shows how you can prefilter contracts easily based on strikes and expirations, and how you
-### can inspect the option chain to pick a specific option contract to trade.
-### </summary>
-### <meta name="tag" content="using data" />
-### <meta name="tag" content="options" />
-### <meta name="tag" content="filter selection" />
-class BasicTemplateOptionsDailyAlgorithm(QCAlgorithm):
-    UnderlyingTicker = "GOOG"
-
-    def Initialize(self):
-        self.SetStartDate(2015, 12, 23)
-        self.SetEndDate(2016, 1, 20)
-        self.SetCash(100000)
-        self.optionExpired = False
-
-        equity = self.AddEquity(self.UnderlyingTicker, Resolution.Daily)
-        option = self.AddOption(self.UnderlyingTicker, Resolution.Daily)
-        self.option_symbol = option.Symbol
-
-        # set our strike/expiry filter for this option chain
-        option.SetFilter(lambda u: (u.Strikes(0, 1).Expiration(0, 30)))
-
-        # use the underlying equity as the benchmark
-        self.SetBenchmark(equity.Symbol)
-
-    def OnData(self,slice):
-        if self.Portfolio.Invested: return
-
-        chain = slice.OptionChains.GetValue(self.option_symbol)
-        if chain is None:
-            return
-
-        # Grab us the contract nearest expiry
-        contracts = sorted(chain, key = lambda x: x.Expiry)
-
-        # if found, trade it
-        if len(contracts) == 0: return
-        symbol = contracts[0].Symbol
-        self.MarketOrder(symbol, 1)
-
-    def OnOrderEvent(self, orderEvent):
-        self.Log(str(orderEvent))
-        
-        # Check for our expected OTM option expiry
-        if orderEvent.Message == "OTM":
-        
-            # Assert it is at midnight 1/16 (5AM UTC)
-            if orderEvent.UtcTime.month != 1 and orderEvent.UtcTime.day != 16 and orderEvent.UtcTime.hour != 5:
-                raise AssertionError(f"Expiry event was not at the correct time, {orderEvent.UtcTime}")
-
-            self.optionExpired = True
-    
-    def OnEndOfAlgorithm(self):
-        # Assert we had our option expire and fill a liquidation order
-        if not self.optionExpired:
-            raise AssertionError("Algorithm did not process the option expiration like expected")

Algorithm.Python/BasicTemplateOptionsFilterUniverseAlgorithm.py

@@ -26,7 +26,7 @@ class BasicTemplateOptionsFilterUniverseAlgorithm(QCAlgorithm):
 
     def Initialize(self):
         self.SetStartDate(2015, 12, 24)
-        self.SetEndDate(2015, 12, 28)
+        self.SetEndDate(2015, 12, 24)
         self.SetCash(100000)
 
         equity = self.AddEquity(self.UnderlyingTicker)

Algorithm.Python/BasicTemplateOptionsFrameworkAlgorithm.py

@@ -29,7 +29,7 @@ class BasicTemplateOptionsFrameworkAlgorithm(QCAlgorithm):
         self.UniverseSettings.Resolution = Resolution.Minute
 
         self.SetStartDate(2014, 6, 5)
-        self.SetEndDate(2014, 6, 9)
+        self.SetEndDate(2014, 6, 6)
         self.SetCash(100000)
 
         # set framework models

Algorithm.Python/BasicTemplateOptionsHourlyAlgorithm.py

@@ -1,66 +0,0 @@
-# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
-# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from AlgorithmImports import *
-
-### <summary>
-### This example demonstrates how to add options for a given underlying equity security.
-### It also shows how you can prefilter contracts easily based on strikes and expirations, and how you
-### can inspect the option chain to pick a specific option contract to trade.
-### </summary>
-### <meta name="tag" content="using data" />
-### <meta name="tag" content="options" />
-### <meta name="tag" content="filter selection" />
-class BasicTemplateOptionsHourlyAlgorithm(QCAlgorithm):
-    UnderlyingTicker = "AAPL"
-
-    def Initialize(self):
-        self.SetStartDate(2014, 6, 6)
-        self.SetEndDate(2014, 6, 9)
-        self.SetCash(100000)
-
-        equity = self.AddEquity(self.UnderlyingTicker, Resolution.Hour)
-        option = self.AddOption(self.UnderlyingTicker, Resolution.Hour)
-        self.option_symbol = option.Symbol
-
-        # set our strike/expiry filter for this option chain
-        option.SetFilter(lambda u: (u.Strikes(-2, +2)
-                                     # Expiration method accepts TimeSpan objects or integer for days.
-                                     # The following statements yield the same filtering criteria
-                                     .Expiration(0, 180)))
-                                     #.Expiration(TimeSpan.Zero, TimeSpan.FromDays(180))))
-
-        # use the underlying equity as the benchmark
-        self.SetBenchmark(equity.Symbol)
-
-    def OnData(self,slice):
-        if self.Portfolio.Invested or not self.IsMarketOpen(self.option_symbol): return
-
-        chain = slice.OptionChains.GetValue(self.option_symbol)
-        if chain is None:
-            return
-
-        # we sort the contracts to find at the money (ATM) contract with farthest expiration
-        contracts = sorted(sorted(sorted(chain, \
-            key = lambda x: abs(chain.Underlying.Price - x.Strike)), \
-            key = lambda x: x.Expiry, reverse=True), \
-            key = lambda x: x.Right, reverse=True)
-
-        # if found, trade it
-        if len(contracts) == 0: return
-        symbol = contracts[0].Symbol
-        self.MarketOrder(symbol, 1)
-        self.MarketOnCloseOrder(symbol, -1)
-
-    def OnOrderEvent(self, orderEvent):
-        self.Log(str(orderEvent))

Algorithm.Python/ContinuousFutureRegressionAlgorithm.py

@@ -29,7 +29,7 @@ class ContinuousFutureRegressionAlgorithm(QCAlgorithm):
         self._lastDateLog = -1
         self._continuousContract = self.AddFuture(Futures.Indices.SP500EMini,
                                                   dataNormalizationMode = DataNormalizationMode.BackwardsRatio,
-                                                  dataMappingMode = DataMappingMode.LastTradingDay,
+                                                  dataMappingMode = DataMappingMode.FirstDayMonth,
                                                   contractDepthOffset= 0)
 
     def OnData(self, data):

Algorithm.Python/FilterUniverseRegressionAlgorithm.py

@@ -25,7 +25,7 @@ class FilterUniverseRegressionAlgorithm(QCAlgorithm):
 
     def Initialize(self):
         self.SetStartDate(2015, 12, 24)
-        self.SetEndDate(2015, 12, 28)
+        self.SetEndDate(2015, 12, 24)
         self.SetCash(100000)
 
         equity = self.AddEquity(self.UnderlyingTicker)

Algorithm.Python/FutureOptionCallITMExpiryRegressionAlgorithm.py

@@ -84,7 +84,7 @@ class FutureOptionCallITMExpiryRegressionAlgorithm(QCAlgorithm):
         self.Log(f"{self.Time} -- {orderEvent.Symbol} :: Price: {self.Securities[orderEvent.Symbol].Holdings.Price} Qty: {self.Securities[orderEvent.Symbol].Holdings.Quantity} Direction: {orderEvent.Direction} Msg: {orderEvent.Message}")
 
     def AssertFutureOptionOrderExercise(self, orderEvent: OrderEvent, future: Security, optionContract: Security):
-        expectedLiquidationTimeUtc = datetime(2020, 6, 20, 4, 0, 0)
+        expectedLiquidationTimeUtc = datetime(2020, 6, 19, 20, 0, 0)
 
         if orderEvent.Direction == OrderDirection.Sell and future.Holdings.Quantity != 0:
             # We expect the contract to have been liquidated immediately

Algorithm.Python/FutureOptionPutITMExpiryRegressionAlgorithm.py

@@ -83,7 +83,7 @@ class FutureOptionPutITMExpiryRegressionAlgorithm(QCAlgorithm):
         self.Log(f"{self.Time} -- {orderEvent.Symbol} :: Price: {self.Securities[orderEvent.Symbol].Holdings.Price} Qty: {self.Securities[orderEvent.Symbol].Holdings.Quantity} Direction: {orderEvent.Direction} Msg: {orderEvent.Message}")
 
     def AssertFutureOptionOrderExercise(self, orderEvent: OrderEvent, future: Security, optionContract: Security):
-        expectedLiquidationTimeUtc = datetime(2020, 6, 20, 4, 0, 0)
+        expectedLiquidationTimeUtc = datetime(2020, 6, 19, 20, 0, 0)
 
         if orderEvent.Direction == OrderDirection.Buy and future.Holdings.Quantity != 0:
             # We expect the contract to have been liquidated immediately

Algorithm.Python/FuturesAndFuturesOptionsExpiryTimeAndLiquidationRegressionAlgorithm.py

@@ -24,7 +24,7 @@ class FuturesAndFuturesOptionsExpiryTimeAndLiquidationRegressionAlgorithm(QCAlgo
 
         self.expectedExpiryWarningTime = datetime(2020, 6, 19)
         self.expectedExpiryDelistingTime = datetime(2020, 6, 20)
-        self.expectedLiquidationTime = datetime(2020, 6, 20)
+        self.expectedLiquidationTime = datetime(2020, 6, 19, 16, 0, 0)
 
         self.SetStartDate(2020, 1, 5)
         self.SetEndDate(2020, 12, 1)

Algorithm/QCAlgorithm.Indicators.cs

@@ -352,31 +352,6 @@ namespace QuantConnect.Algorithm
             return bollingerBands;
         }
 
-        /// <summary>
-        /// Creates a BetaIndicator for the given target symbol in relation with the reference used. 
-        /// The indicator will be automatically updated on the given resolution.
-        /// </summary>
-        /// <param name="target">The target symbol whose Beta value we want</param>
-        /// <param name="reference">The reference symbol to compare with the target symbol</param>
-        /// <param name="period">The period of the BetaIndicator</param>
-        /// <param name="resolution">The resolution</param>
-        /// <returns>The BetaIndicator for the given parameters</returns>
-        public BetaIndicator B(Symbol target, Symbol reference, int period, Resolution? resolution = null)
-        {
-            var name = CreateIndicatorName(QuantConnect.Symbol.None, "B", resolution);
-            var betaIndicator = new BetaIndicator(name, period, target, reference);
-            RegisterIndicator(target, betaIndicator, resolution);
-            RegisterIndicator(reference, betaIndicator, resolution);
-
-            if (EnableAutomaticIndicatorWarmUp)
-            {
-                WarmUpIndicator(target, betaIndicator, resolution);
-                WarmUpIndicator(reference, betaIndicator, resolution);
-            }
-
-            return betaIndicator;
-        }
-
         /// <summary>
         /// Creates a new Balance Of Power indicator.
         /// The indicator will be automatically updated on the given resolution.
@@ -1609,28 +1584,6 @@ namespace QuantConnect.Algorithm
             return relativeDailyVolume;
         }
 
-        /// <summary>
-        /// Creates a new SuperTrend indicator.
-        /// </summary>
-        /// <param name="symbol">The symbol whose SuperTrend indicator we want.</param>
-        /// <param name="period">The smoothing period for average true range.</param>
-        /// <param name="multiplier">Multiplier to calculate basic upper and lower bands width.</param>
-        /// <param name="movingAverageType">Smoother type for average true range, defaults to Wilders.</param>
-        /// <param name="resolution">The resolution.</param>
-        public SuperTrend STR(Symbol symbol, int period, decimal multiplier, MovingAverageType movingAverageType = MovingAverageType.Wilders, Resolution? resolution = null)
-        {
-            var name = CreateIndicatorName(symbol, $"STR({period},{multiplier})", resolution);
-            var strend = new SuperTrend(name, period, multiplier, movingAverageType);
-            RegisterIndicator(symbol, strend, resolution);
-
-            if (EnableAutomaticIndicatorWarmUp)
-            {
-                WarmUpIndicator(symbol, strend, resolution);
-            }
-
-            return strend;
-        }
-
         /// <summary>
         /// Creates a new RollingSharpeRatio indicator.
         /// </summary>

Algorithm/QCAlgorithm.cs

@@ -1499,14 +1499,6 @@ namespace QuantConnect.Algorithm
         public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool fillDataForward = true, decimal leverage = Security.NullLeverage, bool extendedMarketHours = false,
             DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null, int contractDepthOffset = 0)
         {
-            // allow users to specify negative numbers, we get the abs of it
-            var contractOffset = (uint)Math.Abs(contractDepthOffset);
-            if (contractOffset > 2)
-            {
-                throw new ArgumentOutOfRangeException(nameof(contractDepthOffset), "'contractDepthOffset' current maximum value is 2." +
-                    " Front month (0) and only 2 back month contracts are currently supported.");
-            }
-
             var isCanonical = symbol.IsCanonical();
 
             // Short-circuit to AddOptionContract because it will add the underlying if required
@@ -1539,6 +1531,8 @@ namespace QuantConnect.Algorithm
                     }
                     else
                     {
+                        // allow users to specify negative numbers, we get the abs of it
+                        var contractOffset = (uint)Math.Abs(contractDepthOffset);
                         security.IsTradable = true;
 
                         var continuousConfigs = SubscriptionManager.SubscriptionDataConfigService.Add(symbol,
@@ -1548,7 +1542,7 @@ namespace QuantConnect.Algorithm
                             isFilteredSubscription: false,
                             subscriptionDataTypes: SubscriptionManager.LookupSubscriptionConfigDataTypes(SecurityType.Future,
                                 GetResolution(symbol, resolution), isCanonical:false),
-                            dataNormalizationMode: dataNormalizationMode ?? UniverseSettings.GetDefaultNormalizationMode(symbol.SecurityType),
+                            dataNormalizationMode: dataNormalizationMode ?? UniverseSettings.DataNormalizationMode,
                             dataMappingMode: dataMappingMode ?? UniverseSettings.DataMappingMode,
                             contractDepthOffset: contractOffset
                             );

Brokerages/Backtesting/BacktestingBrokerage.cs

@@ -17,8 +17,6 @@ using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
-using QuantConnect.Data.Market;
-using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Interfaces;
 using QuantConnect.Logging;
 using QuantConnect.Orders;
@@ -27,7 +25,6 @@ using QuantConnect.Orders.Fees;
 using QuantConnect.Securities;
 using QuantConnect.Securities.Option;
 using QuantConnect.Securities.Positions;
-using QuantConnect.Util;
 
 namespace QuantConnect.Brokerages.Backtesting
 {
@@ -151,7 +148,7 @@ namespace QuantConnect.Brokerages.Backtesting
                 var submitted = new OrderEvent(order,
                         Algorithm.UtcTime,
                         OrderFee.Zero)
-                { Status = OrderStatus.Submitted };
+                    { Status = OrderStatus.Submitted };
                 OnOrderEvent(submitted);
 
                 return true;
@@ -228,7 +225,7 @@ namespace QuantConnect.Brokerages.Backtesting
             var canceled = new OrderEvent(order,
                     Algorithm.UtcTime,
                     OrderFee.Zero)
-            { Status = OrderStatus.Canceled };
+                { Status = OrderStatus.Canceled };
             OnOrderEvent(canceled);
 
             return true;
@@ -345,7 +342,7 @@ namespace QuantConnect.Brokerages.Backtesting
                                 Algorithm.UtcTime,
                                 OrderFee.Zero,
                                 err.Message)
-                        { Status = OrderStatus.Invalid });
+                            { Status = OrderStatus.Invalid });
                         Order pending;
                         _pending.TryRemove(order.Id, out pending);
 
@@ -409,7 +406,7 @@ namespace QuantConnect.Brokerages.Backtesting
                                 Algorithm.UtcTime,
                                 OrderFee.Zero,
                                 message)
-                        { Status = OrderStatus.Invalid });
+                            { Status = OrderStatus.Invalid });
                         Order pending;
                         _pending.TryRemove(order.Id, out pending);
 
@@ -527,67 +524,5 @@ namespace QuantConnect.Brokerages.Backtesting
         {
             _pending[order.Id] = order;
         }
-
-        /// <summary>
-        /// Process delistings
-        /// </summary>
-        /// <param name="delistings">Delistings to process</param>
-        public void ProcessDelistings(Delistings delistings)
-        {
-            // Process our delistings, important to do options first because of possibility of having future options contracts
-            // and underlying future delisting at the same time.
-            foreach (var delisting in delistings?.Values.OrderBy(x => !x.Symbol.SecurityType.IsOption()))
-            {
-                Log.Trace($"BacktestingBrokerage.ProcessDelistings(): Delisting {delisting.Type}: {delisting.Symbol.Value}, UtcTime: {Algorithm.UtcTime}, DelistingTime: {delisting.Time}");
-                if (delisting.Type == DelistingType.Warning)
-                {
-                    // We do nothing with warnings
-                    continue;
-                }
-
-                var security = Algorithm.Securities[delisting.Symbol];
-
-                if (security.Symbol.SecurityType.IsOption())
-                {
-                    // Process the option delisting
-                    OnOptionNotification(new OptionNotificationEventArgs(delisting.Symbol, 0));
-                }
-                else
-                {
-                    // Any other type of delisting
-                    OnDelistingNotification(new DelistingNotificationEventArgs(delisting.Symbol));
-                }
-
-                // don't allow users to open a new position once we sent the liquidation order
-                security.IsTradable = false;
-                security.IsDelisted = true;
-
-                // the subscription are getting removed from the data feed because they end
-                // remove security from all universes
-                foreach (var ukvp in Algorithm.UniverseManager)
-                {
-                    var universe = ukvp.Value;
-                    if (universe.ContainsMember(security.Symbol))
-                    {
-                        var userUniverse = universe as UserDefinedUniverse;
-                        if (userUniverse != null)
-                        {
-                            userUniverse.Remove(security.Symbol);
-                        }
-                        else
-                        {
-                            universe.RemoveMember(Algorithm.UtcTime, security);
-                        }
-                    }
-                }
-
-                // Cancel any other orders
-                var cancelledOrders = Algorithm.Transactions.CancelOpenOrders(delisting.Symbol);
-                foreach (var cancelledOrder in cancelledOrders)
-                {
-                    Log.Trace("AlgorithmManager.Run(): " + cancelledOrder);
-                }
-            }
-        }
     }
 }

Brokerages/BaseWebsocketsBrokerage.cs

@@ -102,7 +102,7 @@ namespace QuantConnect.Brokerages
         /// </summary>
         /// <param name="sender"></param>
         /// <param name="e"></param>
-        protected abstract void OnMessage(object sender, WebSocketMessage e);
+        public abstract void OnMessage(object sender, WebSocketMessage e);
 
         /// <summary>
         /// Creates wss connection, monitors for disconnection and re-connects when necessary
@@ -121,7 +121,7 @@ namespace QuantConnect.Brokerages
         /// Handles the creation of websocket subscriptions
         /// </summary>
         /// <param name="symbols"></param>
-        protected abstract bool Subscribe(IEnumerable<Symbol> symbols);
+        public abstract void Subscribe(IEnumerable<Symbol> symbols);
 
         /// <summary>
         /// Gets a list of current subscriptions

Brokerages/Binance/BinanceBrokerage.cs

@@ -339,7 +339,7 @@ namespace QuantConnect.Brokerages.Binance
         /// </summary>
         /// <param name="sender"></param>
         /// <param name="e"></param>
-        protected override void OnMessage(object sender, WebSocketMessage e)
+        public override void OnMessage(object sender, WebSocketMessage e)
         {
             _messageHandler.HandleNewMessage(e);
         }
@@ -412,10 +412,9 @@ namespace QuantConnect.Brokerages.Binance
         /// <summary>
         /// Not used
         /// </summary>
-        protected override bool Subscribe(IEnumerable<Symbol> symbols)
+        public override void Subscribe(IEnumerable<Symbol> symbols)
         {
             // NOP
-            return true;
         }
 
         /// <summary>

Brokerages/Bitfinex/BitfinexBrokerage.Messaging.cs

@@ -131,7 +131,7 @@ namespace QuantConnect.Brokerages.Bitfinex
         /// </summary>
         /// <param name="sender"></param>
         /// <param name="e"></param>
-        protected override void OnMessage(object sender, WebSocketMessage e)
+        public override void OnMessage(object sender, WebSocketMessage e)
         {
             OnMessageImpl(e);
         }
@@ -164,10 +164,7 @@ namespace QuantConnect.Brokerages.Bitfinex
         /// Not used in master
         /// </summary>
         /// <param name="symbols"></param>
-        protected override bool Subscribe(IEnumerable<Symbol> symbols)
-        {
-            return true;
-        }
+        public override void Subscribe(IEnumerable<Symbol> symbols) { }
 
         private long GetNextClientOrderId()
         {

Brokerages/Brokerage.cs

@@ -54,11 +54,6 @@ namespace QuantConnect.Brokerages
         /// </summary>
         public event EventHandler<OptionNotificationEventArgs> OptionNotification;
 
-        /// <summary>
-        /// Event that fires each time a delisting occurs
-        /// </summary>
-        public event EventHandler<DelistingNotificationEventArgs> DelistingNotification;
-
         /// <summary>
         /// Event that fires each time a user's brokerage account is changed
         /// </summary>
@@ -179,24 +174,6 @@ namespace QuantConnect.Brokerages
             }
         }
 
-        /// <summary>
-        /// Event invocator for the DelistingNotification event
-        /// </summary>
-        /// <param name="e">The DelistingNotification event arguments</param>
-        protected virtual void OnDelistingNotification(DelistingNotificationEventArgs e)
-        {
-            try
-            {
-                Log.Debug("Brokerage.OnDelistingNotification(): " + e);
-
-                DelistingNotification?.Invoke(this, e);
-            }
-            catch (Exception err)
-            {
-                Log.Error(err);
-            }
-        }
-
         /// <summary>
         /// Event invocator for the AccountChanged event
         /// </summary>

Brokerages/GDAX/GDAXBrokerage.Messaging.cs

@@ -112,7 +112,7 @@ namespace QuantConnect.Brokerages.GDAX
         /// </summary>
         /// <param name="sender"></param>
         /// <param name="e"></param>
-        protected override void OnMessage(object sender, WebSocketMessage webSocketMessage)
+        public override void OnMessage(object sender, WebSocketMessage webSocketMessage)
         {
             var e = (WebSocketClientWrapper.TextMessage)webSocketMessage.Data;
 
@@ -392,7 +392,7 @@ namespace QuantConnect.Brokerages.GDAX
         /// <summary>
         /// Creates websocket message subscriptions for the supplied symbols
         /// </summary>
-        protected override bool Subscribe(IEnumerable<Symbol> symbols)
+        public override void Subscribe(IEnumerable<Symbol> symbols)
         {
             var fullList = GetSubscribed().Union(symbols);
             var pendingSymbols = new List<Symbol>();
@@ -427,7 +427,7 @@ namespace QuantConnect.Brokerages.GDAX
 
             if (payload.product_ids.Length == 0)
             {
-                return true;
+                return;
             }
 
             var token = GetAuthenticationToken(string.Empty, "GET", "/users/self/verify");
@@ -446,7 +446,6 @@ namespace QuantConnect.Brokerages.GDAX
             WebSocket.Send(json);
 
             Log.Trace("GDAXBrokerage.Subscribe: Sent subscribe.");
-            return true;
         }
 
         /// <summary>

Brokerages/InteractiveBrokers/InteractiveBrokersSymbolMapper.cs

@@ -284,7 +284,7 @@ namespace QuantConnect.Brokerages.InteractiveBrokers
             var ticker = symbol.ID.Symbol;
             if (symbol.ID.SecurityType == SecurityType.Equity)
             {
-                var mapFile = _mapFileProvider.Get(AuxiliaryDataKey.Create(symbol)).ResolveMapFile(symbol);
+                var mapFile = _mapFileProvider.Get(CorporateActionsKey.Create(symbol)).ResolveMapFile(symbol);
                 ticker = mapFile.GetMappedSymbol(DateTime.UtcNow, symbol.Value);
             }
 

Brokerages/Tradier/TradierBrokerage.DataQueueHandler.cs

@@ -18,6 +18,7 @@ using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using NodaTime;
@@ -36,7 +37,11 @@ namespace QuantConnect.Brokerages.Tradier
         #region IDataQueueHandler implementation
 
         private const string WebSocketUrl = "wss://ws.tradier.com/v1/markets/events";
+        private const int ConnectionTimeout = 30000;
 
+        private readonly WebSocketClientWrapper _webSocketClient = new WebSocketClientWrapper();
+
+        private bool _isDataQueueHandlerInitialized;
         private TradierStreamSession _streamSession;
 
         private readonly ConcurrentDictionary<string, Symbol> _subscribedTickers = new ConcurrentDictionary<string, Symbol>();
@@ -64,13 +69,36 @@ namespace QuantConnect.Brokerages.Tradier
                     "TradierBrokerage.DataQueueHandler.Subscribe(): The sandbox does not support data streaming.");
             }
 
+            // initialize data queue handler on-demand
+            if (!_isDataQueueHandlerInitialized)
+            {
+                _isDataQueueHandlerInitialized = true;
+
+                _streamSession = CreateStreamSession();
+
+                using (var resetEvent = new ManualResetEvent(false))
+                {
+                    EventHandler triggerEvent = (o, args) => resetEvent.Set();
+                    _webSocketClient.Open += triggerEvent;
+
+                    _webSocketClient.Connect();
+
+                    if (!resetEvent.WaitOne(ConnectionTimeout))
+                    {
+                        throw new TimeoutException("Websockets connection timeout.");
+                    }
+
+                    _webSocketClient.Open -= triggerEvent;
+                }
+            }
+
             if (!CanSubscribe(dataConfig.Symbol))
             {
                 return Enumerable.Empty<BaseData>().GetEnumerator();
             }
 
             var enumerator = _aggregator.Add(dataConfig, newDataAvailableHandler);
-            SubscriptionManager.Subscribe(dataConfig);
+            _subscriptionManager.Subscribe(dataConfig);
 
             return enumerator;
         }
@@ -87,11 +115,11 @@ namespace QuantConnect.Brokerages.Tradier
         /// <param name="dataConfig">Subscription config to be removed</param>
         public void Unsubscribe(SubscriptionDataConfig dataConfig)
         {
-            SubscriptionManager.Unsubscribe(dataConfig);
+            _subscriptionManager.Unsubscribe(dataConfig);
             _aggregator.Remove(dataConfig);
         }
 
-        protected override bool Subscribe(IEnumerable<Symbol> symbols)
+        private bool Subscribe(IEnumerable<Symbol> symbols, TickType tickType)
         {
             var symbolsAdded = false;
 
@@ -116,7 +144,7 @@ namespace QuantConnect.Brokerages.Tradier
             return true;
         }
 
-        private bool Unsubscribe(IEnumerable<Symbol> symbols)
+        private bool Unsubscribe(IEnumerable<Symbol> symbols, TickType tickType)
         {
             var symbolsRemoved = false;
 
@@ -151,7 +179,7 @@ namespace QuantConnect.Brokerages.Tradier
         {
             var obj = new
             {
-                sessionid = GetStreamSession().SessionId,
+                sessionid = _streamSession.SessionId,
                 symbols = tickers,
                 filter = new[] { "trade", "quote" },
                 linebreak = true
@@ -159,10 +187,10 @@ namespace QuantConnect.Brokerages.Tradier
 
             var json = JsonConvert.SerializeObject(obj);
 
-            WebSocket.Send(json);
+            _webSocketClient.Send(json);
         }
 
-        protected override void OnMessage(object sender, WebSocketMessage webSocketMessage)
+        private void OnMessage(object sender, WebSocketMessage webSocketMessage)
         {
             var e = (WebSocketClientWrapper.TextMessage)webSocketMessage.Data;
             var obj = JObject.Parse(e.Message);
@@ -227,17 +255,12 @@ namespace QuantConnect.Brokerages.Tradier
         }
 
         /// <summary>
-        /// Get the current Tradier stream session
+        /// Get the current market status
         /// </summary>
-        private TradierStreamSession GetStreamSession()
+        private TradierStreamSession CreateStreamSession()
         {
-            if (_streamSession == null)
-            {
-                var request = new RestRequest("markets/events/session", Method.POST);
-                _streamSession = Execute<TradierStreamSession>(request, TradierApiRequestType.Data, "stream");
-            }
-
-            return _streamSession;
+            var request = new RestRequest("markets/events/session", Method.POST);
+            return Execute<TradierStreamSession>(request, TradierApiRequestType.Data, "stream");
         }
 
         #endregion

Brokerages/Tradier/TradierBrokerage.cs

@@ -45,10 +45,11 @@ namespace QuantConnect.Brokerages.Tradier
     ///  - Getting user data.
     /// </summary>
     [BrokerageFactory(typeof(TradierBrokerageFactory))]
-    public partial class TradierBrokerage : BaseWebsocketsBrokerage, IDataQueueHandler, IDataQueueUniverseProvider, IHistoryProvider
+    public partial class TradierBrokerage : Brokerage, IDataQueueHandler, IDataQueueUniverseProvider, IHistoryProvider
     {
         private readonly bool _useSandbox;
         private readonly string _accountId;
+        private readonly string _accessToken;
 
         // we're reusing the equity exchange here to grab typical exchange hours
         private static readonly EquityExchange Exchange =
@@ -66,6 +67,9 @@ namespace QuantConnect.Brokerages.Tradier
         //Tradier Spec:
         private readonly Dictionary<TradierApiRequestType, RateGate> _rateLimitNextRequest;
 
+        //Endpoints:
+        private readonly string _requestEndpoint;
+
         private readonly IAlgorithm _algorithm;
         private readonly IOrderProvider _orderProvider;
         private readonly ISecurityProvider _securityProvider;
@@ -85,6 +89,7 @@ namespace QuantConnect.Brokerages.Tradier
         private readonly HashSet<long> _unknownTradierOrderIDs = new HashSet<long>();
         private readonly FixedSizeHashQueue<long> _verifiedUnknownTradierOrderIDs = new FixedSizeHashQueue<long>(1000);
         private readonly FixedSizeHashQueue<int> _cancelledQcOrderIDs = new FixedSizeHashQueue<int>(10000);
+        private readonly EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager;
 
         /// <summary>
         /// Returns the brokerage account's base currency
@@ -102,9 +107,7 @@ namespace QuantConnect.Brokerages.Tradier
             bool useSandbox,
             string accountId,
             string accessToken)
-            : base(WebSocketUrl, new WebSocketClientWrapper(),
-                new RestClient(useSandbox ? "https://sandbox.tradier.com/v1/" : "https://api.tradier.com/v1/"),
-                null, null, "Tradier Brokerage")
+            : base("Tradier Brokerage")
         {
             _algorithm = algorithm;
             _orderProvider = orderProvider;
@@ -112,14 +115,13 @@ namespace QuantConnect.Brokerages.Tradier
             _aggregator = aggregator;
             _useSandbox = useSandbox;
             _accountId = accountId;
+            _accessToken = accessToken;
 
-            RestClient.AddDefaultHeader("Accept", "application/json");
-            RestClient.AddDefaultHeader("Authorization", $"Bearer {accessToken}");
+            _requestEndpoint = useSandbox ? "https://sandbox.tradier.com/v1/" : "https://api.tradier.com/v1/";
 
-            var subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
-            subscriptionManager.SubscribeImpl += (symbols, _) => Subscribe(symbols);
-            subscriptionManager.UnsubscribeImpl += (symbols, _) => Unsubscribe(symbols);
-            SubscriptionManager = subscriptionManager;
+            _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager();
+            _subscriptionManager.SubscribeImpl += Subscribe;
+            _subscriptionManager.UnsubscribeImpl += Unsubscribe;
 
             _cachedOpenOrdersByTradierOrderID = new ConcurrentDictionary<long, TradierCachedOpenOrder>();
 
@@ -133,15 +135,9 @@ namespace QuantConnect.Brokerages.Tradier
             };
 
             _orderFillTimer = new Timer(state => CheckForFills(), null, interval, interval);
-            WebSocket.Error += (sender, error) =>
-            {
-                if (!WebSocket.IsOpen)
-                {
-                    // on error we clear our state, on Open we will re susbscribe
-                    _subscribedTickers.Clear();
-                    _streamSession = null;
-                }
-            };
+
+            _webSocketClient.Initialize(WebSocketUrl);
+            _webSocketClient.Message += OnMessage;
         }
 
         #region Tradier client implementation
@@ -163,11 +159,15 @@ namespace QuantConnect.Brokerages.Tradier
 
             lock (_lockAccessCredentials)
             {
+                var client = new RestClient(_requestEndpoint);
+                client.AddDefaultHeader("Accept", "application/json");
+                client.AddDefaultHeader("Authorization", "Bearer " + _accessToken);
+
                 //Wait for the API rate limiting
                 _rateLimitNextRequest[type].WaitToProceed();
 
                 //Send the request:
-                var raw = RestClient.Execute(request);
+                var raw = client.Execute(request);
                 _previousResponseRaw = raw.Content;
 
                 if (!raw.IsSuccessful)
@@ -611,7 +611,7 @@ namespace QuantConnect.Brokerages.Tradier
         /// <summary>
         /// Returns true if we're currently connected to the broker
         /// </summary>
-        public override bool IsConnected => WebSocket.IsOpen;
+        public override bool IsConnected => !_isDataQueueHandlerInitialized || _webSocketClient.IsOpen;
 
         /// <summary>
         /// Gets all open orders on the account.
@@ -906,14 +906,21 @@ namespace QuantConnect.Brokerages.Tradier
             return true;
         }
 
+        /// <summary>
+        /// Connects the client to the broker's remote servers
+        /// </summary>
+        public override void Connect()
+        {
+        }
+
         /// <summary>
         /// Disconnects the client from the broker's remote servers
         /// </summary>
         public override void Disconnect()
         {
-            if (WebSocket != null && WebSocket.IsOpen)
+            if (_webSocketClient != null && _webSocketClient.IsOpen)
             {
-                WebSocket.Close();
+                _webSocketClient.Close();
             }
 
             _orderFillTimer.Change(Timeout.Infinite, Timeout.Infinite);

Brokerages/WebSocketClientWrapper.cs

@@ -116,14 +116,7 @@ namespace QuantConnect.Brokerages
             {
                 try
                 {
-                    try
-                    {
-                        _client?.CloseOutputAsync(WebSocketCloseStatus.NormalClosure, "", _cts.Token).SynchronouslyAwaitTask();
-                    }
-                    catch
-                    {
-                        // ignored
-                    }
+                    _client?.CloseOutputAsync(WebSocketCloseStatus.NormalClosure, "", _cts.Token).SynchronouslyAwaitTask();
 
                     _cts?.Cancel();
 
@@ -211,7 +204,7 @@ namespace QuantConnect.Brokerages
         {
             var receiveBuffer = new byte[ReceiveBufferSize];
 
-            while (_cts is { IsCancellationRequested: false })
+            while (!_cts.IsCancellationRequested)
             {
                 Log.Trace($"WebSocketClientWrapper.HandleConnection({_url}): Connecting...");
 

Common/AlgorithmImports.py

@@ -29,8 +29,6 @@ for file in os.listdir(path):
         AddReference(file.replace(".dll", ""))
 
 from System import *
-from System.Drawing import *
-
 from QuantConnect import *
 from QuantConnect.Api import *
 from QuantConnect.Util import *

Common/Brokerages/DelistingNotificationEventArgs.cs

@@ -1,39 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using QuantConnect.Interfaces;
-
-namespace QuantConnect.Brokerages
-{
-    /// <summary>
-    /// Event arguments class for the <see cref="IBrokerage.DelistingNotification"/> event
-    /// </summary>
-    public class DelistingNotificationEventArgs
-    {
-        /// <summary>
-        /// Gets the option symbol which has received a notification
-        /// </summary>
-        public Symbol Symbol { get; }
-
-        /// <summary>
-        /// Initializes a new instance of the <see cref="DelistingNotificationEventArgs"/> class
-        /// </summary>
-        /// <param name="symbol">The symbol</param>
-        public DelistingNotificationEventArgs(Symbol symbol)
-        {
-            Symbol = symbol;
-        }
-    }
-}

Common/Data/Auxiliary/CorporateActionsKey.cs

@@ -17,14 +17,14 @@
 namespace QuantConnect.Data.Auxiliary
 {
     /// <summary>
-    /// Unique definition key for a collection of auxiliary data for a Market and SecurityType
+    /// Unique definition key for a collection of corporate actions for a Market and SecurityType
     /// </summary>
-    public class AuxiliaryDataKey
+    public class CorporateActionsKey
     {
         /// <summary>
         /// USA equities market corporate actions key definition
         /// </summary>
-        public static AuxiliaryDataKey EquityUsa { get; } = new (QuantConnect.Market.USA, SecurityType.Equity);
+        public static CorporateActionsKey EquityUsa { get; } = new (QuantConnect.Market.USA, SecurityType.Equity);
 
         /// <summary>
         /// The market associated with these corporate actions
@@ -39,7 +39,7 @@ namespace QuantConnect.Data.Auxiliary
         /// <summary>
         /// Creates a new instance
         /// </summary>
-        public AuxiliaryDataKey(string market, SecurityType securityType)
+        public CorporateActionsKey(string market, SecurityType securityType)
         {
             Market = market;
             SecurityType = securityType;
@@ -69,7 +69,7 @@ namespace QuantConnect.Data.Auxiliary
             if (ReferenceEquals(null, obj)) return false;
             if (obj.GetType() != GetType()) return false;
 
-            var other = (AuxiliaryDataKey)obj;
+            var other = (CorporateActionsKey)obj;
 
             return other.Market == Market
                 && other.SecurityType == SecurityType;
@@ -83,15 +83,15 @@ namespace QuantConnect.Data.Auxiliary
         /// <summary>
         /// Helper method to create a new instance from a Symbol
         /// </summary>
-        public static AuxiliaryDataKey Create(Symbol symbol) => Create(symbol.HasUnderlying ? symbol.Underlying.ID : symbol.ID);
+        public static CorporateActionsKey Create(Symbol symbol) => Create(symbol.HasUnderlying ? symbol.Underlying.ID : symbol.ID);
 
         /// <summary>
         /// Helper method to create a new instance from a SecurityIdentifier
         /// </summary>
-        public static AuxiliaryDataKey Create(SecurityIdentifier securityIdentifier)
+        public static CorporateActionsKey Create(SecurityIdentifier securityIdentifier)
         {
             securityIdentifier = securityIdentifier.HasUnderlying ? securityIdentifier.Underlying : securityIdentifier;
-            return new AuxiliaryDataKey(securityIdentifier.Market, securityIdentifier.SecurityType);
+            return new CorporateActionsKey(securityIdentifier.Market, securityIdentifier.SecurityType);
         }
     }
 }

Common/Data/Auxiliary/CorporateFactorProvider.cs

@@ -1,273 +0,0 @@
-/*
-* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
-* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*
-*/
-
-using System;
-using System.Linq;
-using QuantConnect.Util;
-using QuantConnect.Logging;
-using QuantConnect.Securities;
-using QuantConnect.Data.Market;
-using System.Collections.Generic;
-
-namespace QuantConnect.Data.Auxiliary
-{
-    /// <summary>
-    /// Corporate related factor provider. Factors based on splits and dividends
-    /// </summary>
-    public class CorporateFactorProvider : FactorFile<CorporateFactorRow>
-    {
-        /// <summary>
-        ///Creates a new instance
-        /// </summary>
-        public CorporateFactorProvider(string permtick, IEnumerable<CorporateFactorRow> data, DateTime? factorFileMinimumDate = null) : base(permtick, data, factorFileMinimumDate)
-        {
-        }
-
-        /// <summary>
-        /// Gets the price scale factor that includes dividend and split adjustments for the specified search date
-        /// </summary>
-        public override decimal GetPriceFactor(DateTime searchDate, DataNormalizationMode dataNormalizationMode, DataMappingMode? dataMappingMode = null, uint contractOffset = 0)
-        {
-            if (dataNormalizationMode == DataNormalizationMode.Raw)
-            {
-                return 0;
-            }
-
-            var factor = 1m;
-
-            for (var i = 0; i < _reversedFactorFileDates.Count; i++)
-            {
-                var factorDate = _reversedFactorFileDates[i];
-                if (factorDate.Date < searchDate.Date)
-                {
-                    break;
-                }
-
-                var factorFileRow = SortedFactorFileData[factorDate];
-                switch (dataNormalizationMode)
-                {
-                    case DataNormalizationMode.TotalReturn:
-                    case DataNormalizationMode.SplitAdjusted:
-                        factor = factorFileRow.First().SplitFactor;
-                        break;
-                    case DataNormalizationMode.Adjusted:
-                        factor = factorFileRow.First().PriceScaleFactor;
-                        break;
-                    default:
-                        throw new ArgumentOutOfRangeException();
-                }
-            }
-
-            return factor;
-        }
-
-        /// <summary>
-        /// Gets price and split factors to be applied at the specified date
-        /// </summary>
-        public CorporateFactorRow GetScalingFactors(DateTime searchDate)
-        {
-            var factors = new CorporateFactorRow(searchDate, 1m, 1m, 0m);
-
-            // Iterate backwards to find the most recent factors
-            foreach (var splitDate in _reversedFactorFileDates)
-            {
-                if (splitDate.Date < searchDate.Date) break;
-                factors = SortedFactorFileData[splitDate][0];
-            }
-
-            return factors;
-        }
-
-        /// <summary>
-        /// Returns true if the specified date is the last trading day before a dividend event
-        /// is to be fired
-        /// </summary>
-        /// <remarks>
-        /// NOTE: The dividend event in the algorithm should be fired at the end or AFTER
-        /// this date. This is the date in the file that a factor is applied, so for example,
-        /// MSFT has a 31 cent dividend on 2015.02.17, but in the factor file the factor is applied
-        /// to 2015.02.13, which is the first trading day BEFORE the actual effective date.
-        /// </remarks>
-        /// <param name="date">The date to check the factor file for a dividend event</param>
-        /// <param name="priceFactorRatio">When this function returns true, this value will be populated
-        /// with the price factor ratio required to scale the closing value (pf_i/pf_i+1)</param>
-        /// <param name="referencePrice">When this function returns true, this value will be populated
-        /// with the reference raw price, which is the close of the provided date</param>
-        public bool HasDividendEventOnNextTradingDay(DateTime date, out decimal priceFactorRatio, out decimal referencePrice)
-        {
-            priceFactorRatio = 0;
-            referencePrice = 0;
-            var index = SortedFactorFileData.IndexOfKey(date);
-            if (index > -1 && index < SortedFactorFileData.Count - 1)
-            {
-                // grab the next key to ensure it's a dividend event
-                var thisRow = SortedFactorFileData.Values[index].First();
-                var nextRow = SortedFactorFileData.Values[index + 1].First();
-
-                // if the price factors have changed then it's a dividend event
-                if (thisRow.PriceFactor != nextRow.PriceFactor)
-                {
-                    priceFactorRatio = thisRow.PriceFactor / nextRow.PriceFactor;
-                    referencePrice = thisRow.ReferencePrice;
-                    return true;
-                }
-            }
-            return false;
-        }
-
-        /// <summary>
-        /// Returns true if the specified date is the last trading day before a split event
-        /// is to be fired
-        /// </summary>
-        /// <remarks>
-        /// NOTE: The split event in the algorithm should be fired at the end or AFTER this
-        /// date. This is the date in the file that a factor is applied, so for example MSFT
-        /// has a split on 1999.03.29, but in the factor file the split factor is applied on
-        /// 1999.03.26, which is the first trading day BEFORE the actual split date.
-        /// </remarks>
-        /// <param name="date">The date to check the factor file for a split event</param>
-        /// <param name="splitFactor">When this function returns true, this value will be populated
-        /// with the split factor ratio required to scale the closing value</param>
-        /// <param name="referencePrice">When this function returns true, this value will be populated
-        /// with the reference raw price, which is the close of the provided date</param>
-        public bool HasSplitEventOnNextTradingDay(DateTime date, out decimal splitFactor, out decimal referencePrice)
-        {
-            splitFactor = 1;
-            referencePrice = 0;
-            var index = SortedFactorFileData.IndexOfKey(date);
-            if (index > -1 && index < SortedFactorFileData.Count - 1)
-            {
-                // grab the next key to ensure it's a split event
-                var thisRow = SortedFactorFileData.Values[index].First();
-                var nextRow = SortedFactorFileData.Values[index + 1].First();
-
-                // if the split factors have changed then it's a split event
-                if (thisRow.SplitFactor != nextRow.SplitFactor)
-                {
-                    splitFactor = thisRow.SplitFactor / nextRow.SplitFactor;
-                    referencePrice = thisRow.ReferencePrice;
-                    return true;
-                }
-            }
-            return false;
-        }
-
-        /// <summary>
-        /// Gets all of the splits and dividends represented by this factor file
-        /// </summary>
-        /// <param name="symbol">The symbol to ues for the dividend and split objects</param>
-        /// <param name="exchangeHours">Exchange hours used for resolving the previous trading day</param>
-        /// <param name="decimalPlaces">The number of decimal places to round the dividend's distribution to, defaulting to 2</param>
-        /// <returns>All splits and dividends represented by this factor file in chronological order</returns>
-        public List<BaseData> GetSplitsAndDividends(Symbol symbol, SecurityExchangeHours exchangeHours, int decimalPlaces = 2)
-        {
-            var dividendsAndSplits = new List<BaseData>();
-            if (SortedFactorFileData.Count == 0)
-            {
-                Log.Trace($"{symbol} has no factors!");
-                return dividendsAndSplits;
-            }
-
-            var futureFactorFileRow = SortedFactorFileData.Last().Value.First();
-            for (var i = SortedFactorFileData.Count - 2; i >= 0; i--)
-            {
-                var row = SortedFactorFileData.Values[i].First();
-                var dividend = row.GetDividend(futureFactorFileRow, symbol, exchangeHours, decimalPlaces);
-                if (dividend.Distribution != 0m)
-                {
-                    dividendsAndSplits.Add(dividend);
-                }
-
-                var split = row.GetSplit(futureFactorFileRow, symbol, exchangeHours);
-                if (split.SplitFactor != 1m)
-                {
-                    dividendsAndSplits.Add(split);
-                }
-
-                futureFactorFileRow = row;
-            }
-
-            return dividendsAndSplits.OrderBy(d => d.Time.Date).ToList();
-        }
-
-        /// <summary>
-        /// Creates a new factor file with the specified data applied.
-        /// Only <see cref="Dividend"/> and <see cref="Split"/> data types
-        /// will be used.
-        /// </summary>
-        /// <param name="data">The data to apply</param>
-        /// <param name="exchangeHours">Exchange hours used for resolving the previous trading day</param>
-        /// <returns>A new factor file that incorporates the specified dividend</returns>
-        public CorporateFactorProvider Apply(List<BaseData> data, SecurityExchangeHours exchangeHours)
-        {
-            if (data.Count == 0)
-            {
-                return this;
-            }
-
-            var factorFileRows = new List<CorporateFactorRow>();
-            var firstEntry = SortedFactorFileData.First().Value.First();
-            var lastEntry = SortedFactorFileData.Last().Value.First();
-            factorFileRows.Add(lastEntry);
-
-            var splitsAndDividends = GetSplitsAndDividends(data[0].Symbol, exchangeHours);
-
-            var combinedData = splitsAndDividends.Concat(data)
-                .DistinctBy(e => $"{e.GetType().Name}{e.Time.ToStringInvariant(DateFormat.EightCharacter)}")
-                .OrderByDescending(d => d.Time.Date);
-
-            foreach (var datum in combinedData)
-            {
-                CorporateFactorRow nextEntry = null;
-                var split = datum as Split;
-                var dividend = datum as Dividend;
-                if (dividend != null)
-                {
-                    nextEntry = lastEntry.Apply(dividend, exchangeHours);
-                    lastEntry = nextEntry;
-                }
-                else if (split != null)
-                {
-                    nextEntry = lastEntry.Apply(split, exchangeHours);
-                    lastEntry = nextEntry;
-                }
-
-                if (nextEntry != null)
-                {
-                    // overwrite the latest entry -- this handles splits/dividends on the same date
-                    if (nextEntry.Date == factorFileRows.Last().Date)
-                    {
-                        factorFileRows[factorFileRows.Count - 1] = nextEntry;
-                    }
-                    else
-                    {
-                        factorFileRows.Add(nextEntry);
-                    }
-                }
-            }
-
-            var firstFactorFileRow = new CorporateFactorRow(firstEntry.Date, factorFileRows.Last().PriceFactor, factorFileRows.Last().SplitFactor, firstEntry.ReferencePrice == 0 ? 0 : firstEntry.ReferencePrice);
-            var existing = factorFileRows.FindIndex(row => row.Date == firstFactorFileRow.Date);
-            if (existing == -1)
-            {
-                // only add it if not present
-                factorFileRows.Add(firstFactorFileRow);
-            }
-
-            return new CorporateFactorProvider(Permtick, factorFileRows, FactorFileMinimumDate);
-        }
-    }
-}

Common/Data/Auxiliary/FactorFile.cs

@@ -15,29 +15,32 @@
 */
 
 using System;
-using System.IO;
-using System.Linq;
-using QuantConnect.Util;
 using System.Collections;
 using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using QuantConnect.Data.Market;
+using QuantConnect.Logging;
+using QuantConnect.Securities;
+using QuantConnect.Util;
+using static QuantConnect.StringExtensions;
 
 namespace QuantConnect.Data.Auxiliary
 {
     /// <summary>
     /// Represents an entire factor file for a specified symbol
     /// </summary>
-    public abstract class FactorFile<T> : IFactorProvider
-        where T : IFactorRow
+    public class FactorFile : IEnumerable<FactorFileRow>
     {
         /// <summary>
         /// Keeping a reversed version is more performant that reversing it each time we need it
         /// </summary>
-        protected readonly List<DateTime> _reversedFactorFileDates;
+        private readonly List<DateTime> _reversedFactorFileDates;
 
         /// <summary>
         /// The factor file data rows sorted by date
         /// </summary>
-        public SortedList<DateTime, List<T>> SortedFactorFileData { get; set; }
+        public SortedList<DateTime, FactorFileRow> SortedFactorFileData { get; set; }
 
         /// <summary>
         /// The minimum tradeable date for the symbol
@@ -65,22 +68,24 @@ namespace QuantConnect.Data.Auxiliary
         /// <summary>
         /// Initializes a new instance of the <see cref="FactorFile"/> class.
         /// </summary>
-        protected FactorFile(string permtick, IEnumerable<T> data, DateTime? factorFileMinimumDate = null)
+        public FactorFile(string permtick, IEnumerable<FactorFileRow> data, DateTime? factorFileMinimumDate = null)
         {
             Permtick = permtick.LazyToUpper();
 
-            SortedFactorFileData = new SortedList<DateTime, List<T>>();
+            var dictionary = new Dictionary<DateTime, FactorFileRow>();
             foreach (var row in data)
             {
-                if (!SortedFactorFileData.TryGetValue(row.Date, out var factorFileRows))
+                if (dictionary.ContainsKey(row.Date))
                 {
-                    SortedFactorFileData[row.Date] = factorFileRows = new List<T>();
+                    Log.Trace(Invariant($"Skipping duplicate factor file row for symbol: {permtick}, date: {row.Date:yyyyMMdd}"));
+                    continue;
                 }
 
-                factorFileRows.Add(row);
+                dictionary.Add(row.Date, row);
             }
+            SortedFactorFileData = new SortedList<DateTime, FactorFileRow>(dictionary);
 
-            _reversedFactorFileDates = new List<DateTime>(SortedFactorFileData.Count);
+            _reversedFactorFileDates = new List<DateTime>();
             foreach (var time in SortedFactorFileData.Keys.Reverse())
             {
                 _reversedFactorFileDates.Add(time);
@@ -90,45 +95,288 @@ namespace QuantConnect.Data.Auxiliary
         }
 
         /// <summary>
-        /// Gets the price scale factor for the specified search date
+        /// Reads a FactorFile in from the <see cref="Globals.DataFolder"/>.
         /// </summary>
-        public abstract decimal GetPriceFactor(
-            DateTime searchDate,
-            DataNormalizationMode dataNormalizationMode,
-            DataMappingMode? dataMappingMode = null,
-            uint contractOffset = 0
-            );
+        public static FactorFile Read(string permtick, Stream file)
+        {
+            return new FactorFile(permtick, FactorFileRow.Read(file, out DateTime? factorFileMinimumDate), factorFileMinimumDate);
+        }
+
+        /// <summary>
+        /// Parses the specified lines as a factor file
+        /// </summary>
+        public static FactorFile Parse(string permtick, IEnumerable<string> lines)
+        {
+            DateTime? factorFileMinimumDate;
+            return new FactorFile(permtick, FactorFileRow.Parse(lines, out factorFileMinimumDate), factorFileMinimumDate);
+        }
+
+        /// <summary>
+        /// Gets the price scale factor that includes dividend and split adjustments for the specified search date
+        /// </summary>
+        public decimal GetPriceScaleFactor(DateTime searchDate)
+        {
+            decimal factor = 1;
+            //Iterate backwards to find the most recent factor:
+            foreach (var splitDate in _reversedFactorFileDates)
+            {
+                if (splitDate.Date < searchDate.Date) break;
+                factor = SortedFactorFileData[splitDate].PriceScaleFactor;
+            }
+            return factor;
+        }
+
+        /// <summary>
+        /// Gets the split factor to be applied at the specified date
+        /// </summary>
+        public decimal GetSplitFactor(DateTime searchDate)
+        {
+            decimal factor = 1;
+            //Iterate backwards to find the most recent factor:
+            foreach (var splitDate in _reversedFactorFileDates)
+            {
+                if (splitDate.Date < searchDate.Date) break;
+                factor = SortedFactorFileData[splitDate].SplitFactor;
+            }
+            return factor;
+        }
+
+        /// <summary>
+        /// Gets price and split factors to be applied at the specified date
+        /// </summary>
+        public FactorFileRow GetScalingFactors(DateTime searchDate)
+        {
+            var factors = new FactorFileRow(searchDate, 1m, 1m, 0m);
+
+            // Iterate backwards to find the most recent factors
+            foreach (var splitDate in _reversedFactorFileDates)
+            {
+                if (splitDate.Date < searchDate.Date) break;
+                factors = SortedFactorFileData[splitDate];
+            }
+
+            return factors;
+        }
+
+        /// <summary>
+        /// Checks whether or not a symbol has scaling factors
+        /// </summary>
+        public static bool HasScalingFactors(string permtick, string market)
+        {
+            // check for factor files
+            var path = Path.Combine(Globals.CacheDataFolder, "equity", market, "factor_files", permtick.ToLowerInvariant() + ".csv");
+            if (File.Exists(path))
+            {
+                return true;
+            }
+            Log.Trace($"FactorFile.HasScalingFactors(): Factor file not found: {permtick}");
+            return false;
+        }
+
+        /// <summary>
+        /// Returns true if the specified date is the last trading day before a dividend event
+        /// is to be fired
+        /// </summary>
+        /// <remarks>
+        /// NOTE: The dividend event in the algorithm should be fired at the end or AFTER
+        /// this date. This is the date in the file that a factor is applied, so for example,
+        /// MSFT has a 31 cent dividend on 2015.02.17, but in the factor file the factor is applied
+        /// to 2015.02.13, which is the first trading day BEFORE the actual effective date.
+        /// </remarks>
+        /// <param name="date">The date to check the factor file for a dividend event</param>
+        /// <param name="priceFactorRatio">When this function returns true, this value will be populated
+        /// with the price factor ratio required to scale the closing value (pf_i/pf_i+1)</param>
+        /// <param name="referencePrice">When this function returns true, this value will be populated
+        /// with the reference raw price, which is the close of the provided date</param>
+        public bool HasDividendEventOnNextTradingDay(DateTime date, out decimal priceFactorRatio, out decimal referencePrice)
+        {
+            priceFactorRatio = 0;
+            referencePrice = 0;
+            var index = SortedFactorFileData.IndexOfKey(date);
+            if (index > -1 && index < SortedFactorFileData.Count - 1)
+            {
+                // grab the next key to ensure it's a dividend event
+                var thisRow = SortedFactorFileData.Values[index];
+                var nextRow = SortedFactorFileData.Values[index + 1];
+
+                // if the price factors have changed then it's a dividend event
+                if (thisRow.PriceFactor != nextRow.PriceFactor)
+                {
+                    priceFactorRatio = thisRow.PriceFactor / nextRow.PriceFactor;
+                    referencePrice = thisRow.ReferencePrice;
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        /// <summary>
+        /// Returns true if the specified date is the last trading day before a split event
+        /// is to be fired
+        /// </summary>
+        /// <remarks>
+        /// NOTE: The split event in the algorithm should be fired at the end or AFTER this
+        /// date. This is the date in the file that a factor is applied, so for example MSFT
+        /// has a split on 1999.03.29, but in the factor file the split factor is applied on
+        /// 1999.03.26, which is the first trading day BEFORE the actual split date.
+        /// </remarks>
+        /// <param name="date">The date to check the factor file for a split event</param>
+        /// <param name="splitFactor">When this function returns true, this value will be populated
+        /// with the split factor ratio required to scale the closing value</param>
+        /// <param name="referencePrice">When this function returns true, this value will be populated
+        /// with the reference raw price, which is the close of the provided date</param>
+        public bool HasSplitEventOnNextTradingDay(DateTime date, out decimal splitFactor, out decimal referencePrice)
+        {
+            splitFactor = 1;
+            referencePrice = 0;
+            var index = SortedFactorFileData.IndexOfKey(date);
+            if (index > -1 && index < SortedFactorFileData.Count - 1)
+            {
+                // grab the next key to ensure it's a split event
+                var thisRow = SortedFactorFileData.Values[index];
+                var nextRow = SortedFactorFileData.Values[index + 1];
+
+                // if the split factors have changed then it's a split event
+                if (thisRow.SplitFactor != nextRow.SplitFactor)
+                {
+                    splitFactor = thisRow.SplitFactor / nextRow.SplitFactor;
+                    referencePrice = thisRow.ReferencePrice;
+                    return true;
+                }
+            }
+            return false;
+        }
 
         /// <summary>
         /// Writes this factor file data to an enumerable of csv lines
         /// </summary>
         /// <returns>An enumerable of lines representing this factor file</returns>
-        public IEnumerable<string> GetFileFormat()
+        public IEnumerable<string> ToCsvLines()
         {
-            return SortedFactorFileData.SelectMany(kvp => kvp.Value.Select(row => row.GetFileFormat()));
+            foreach (var kvp in SortedFactorFileData)
+            {
+                yield return kvp.Value.ToCsv();
+            }
         }
 
         /// <summary>
         /// Write the factor file to the correct place in the default Data folder
         /// </summary>
         /// <param name="symbol">The symbol this factor file represents</param>
-        public void WriteToFile(Symbol symbol)
+        public void WriteToCsv(Symbol symbol)
         {
             var filePath = LeanData.GenerateRelativeFactorFilePath(symbol);
-            File.WriteAllLines(filePath, GetFileFormat());
+            File.WriteAllLines(filePath, ToCsvLines());
+        }
+
+        /// <summary>
+        /// Gets all of the splits and dividends represented by this factor file
+        /// </summary>
+        /// <param name="symbol">The symbol to ues for the dividend and split objects</param>
+        /// <param name="exchangeHours">Exchange hours used for resolving the previous trading day</param>
+        /// <param name="decimalPlaces">The number of decimal places to round the dividend's distribution to, defaulting to 2</param>
+        /// <returns>All splits and diviends represented by this factor file in chronological order</returns>
+        public List<BaseData> GetSplitsAndDividends(Symbol symbol, SecurityExchangeHours exchangeHours, int decimalPlaces = 2)
+        {
+            var dividendsAndSplits = new List<BaseData>();
+            if (SortedFactorFileData.Count == 0)
+            {
+                Log.Trace($"{symbol} has no factors!");
+                return dividendsAndSplits;
+            }
+
+            var futureFactorFileRow = SortedFactorFileData.Last().Value;
+            for (var i = SortedFactorFileData.Count - 2; i >= 0; i--)
+            {
+                var row = SortedFactorFileData.Values[i];
+                var dividend = row.GetDividend(futureFactorFileRow, symbol, exchangeHours, decimalPlaces);
+                if (dividend.Distribution != 0m)
+                {
+                    dividendsAndSplits.Add(dividend);
+                }
+
+                var split = row.GetSplit(futureFactorFileRow, symbol, exchangeHours);
+                if (split.SplitFactor != 1m)
+                {
+                    dividendsAndSplits.Add(split);
+                }
+
+                futureFactorFileRow = row;
+            }
+
+            return dividendsAndSplits.OrderBy(d => d.Time.Date).ToList();
+        }
+
+        /// <summary>
+        /// Creates a new factor file with the specified data applied.
+        /// Only <see cref="Dividend"/> and <see cref="Split"/> data types
+        /// will be used.
+        /// </summary>
+        /// <param name="data">The data to apply</param>
+        /// <param name="exchangeHours">Exchange hours used for resolving the previous trading day</param>
+        /// <returns>A new factor file that incorporates the specified dividend</returns>
+        public FactorFile Apply(List<BaseData> data, SecurityExchangeHours exchangeHours)
+        {
+            if (data.Count == 0)
+            {
+                return this;
+            }
+
+            var factorFileRows = new List<FactorFileRow>();
+            var firstEntry = SortedFactorFileData.First().Value;
+            var lastEntry = SortedFactorFileData.Last().Value;
+            factorFileRows.Add(lastEntry);
+
+            var splitsAndDividends = GetSplitsAndDividends(data[0].Symbol, exchangeHours);
+
+            var combinedData = splitsAndDividends.Concat(data)
+                .DistinctBy(e => $"{e.GetType().Name}{e.Time.ToStringInvariant(DateFormat.EightCharacter)}")
+                .OrderByDescending(d => d.Time.Date);
+
+            foreach (var datum in combinedData)
+            {
+                FactorFileRow nextEntry = null;
+                var split = datum as Split;
+                var dividend = datum as Dividend;
+                if (dividend != null)
+                {
+                    nextEntry = lastEntry.Apply(dividend, exchangeHours);
+                    lastEntry = nextEntry;
+                }
+                else if (split != null)
+                {
+                    nextEntry = lastEntry.Apply(split, exchangeHours);
+                    lastEntry = nextEntry;
+                }
+
+                if (nextEntry != null)
+                {
+                    // overwrite the latest entry -- this handles splits/dividends on the same date
+                    if (nextEntry.Date == factorFileRows.Last().Date)
+                    {
+                        factorFileRows[factorFileRows.Count - 1] = nextEntry;
+                    }
+                    else
+                    {
+                        factorFileRows.Add(nextEntry);
+                    }
+                }
+            }
+
+            var firstFactorFileRow = new FactorFileRow(firstEntry.Date, factorFileRows.Last().PriceFactor, factorFileRows.Last().SplitFactor, firstEntry.ReferencePrice == 0 ? 0 : firstEntry.ReferencePrice);
+            factorFileRows.Add(firstFactorFileRow);
+
+            return new FactorFile(Permtick, factorFileRows, FactorFileMinimumDate);
         }
 
         /// <summary>Returns an enumerator that iterates through the collection.</summary>
         /// <returns>A <see cref="T:System.Collections.Generic.IEnumerator`1" /> that can be used to iterate through the collection.</returns>
         /// <filterpriority>1</filterpriority>
-        public IEnumerator<IFactorRow> GetEnumerator()
+        public IEnumerator<FactorFileRow> GetEnumerator()
         {
             foreach (var kvp in SortedFactorFileData)
             {
-                foreach (var factorRow in kvp.Value)
-                {
-                    yield return factorRow;
-                }
+                yield return kvp.Value;
             }
         }
 

Common/Data/Auxiliary/FactorFileRow.cs

@@ -15,11 +15,13 @@
 */
 
 using System;
-using System.Linq;
-using System.Globalization;
-using QuantConnect.Securities;
-using QuantConnect.Data.Market;
 using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Text;
+using QuantConnect.Data.Market;
+using QuantConnect.Securities;
 using static QuantConnect.StringExtensions;
 
 namespace QuantConnect.Data.Auxiliary
@@ -27,7 +29,7 @@ namespace QuantConnect.Data.Auxiliary
     /// <summary>
     /// Defines a single row in a factor_factor file. This is a csv file ordered as {date, price factor, split factor, reference price}
     /// </summary>
-    public class CorporateFactorRow : IFactorRow
+    public class FactorFileRow
     {
         private decimal _splitFactor;
         private decimal _priceFactor;
@@ -81,9 +83,9 @@ namespace QuantConnect.Data.Auxiliary
         public decimal ReferencePrice { get; private set; }
 
         /// <summary>
-        /// Initializes a new instance of the <see cref="CorporateFactorRow"/> class
+        /// Initializes a new instance of the <see cref="FactorFileRow"/> class
         /// </summary>
-        public CorporateFactorRow(DateTime date, decimal priceFactor, decimal splitFactor, decimal referencePrice = 0)
+        public FactorFileRow(DateTime date, decimal priceFactor, decimal splitFactor, decimal referencePrice = 0)
         {
             Date = date;
             ReferencePrice = referencePrice;
@@ -91,17 +93,42 @@ namespace QuantConnect.Data.Auxiliary
             SplitFactor = splitFactor;
         }
 
+        /// <summary>
+        /// Reads in the factor file for the specified equity symbol
+        /// </summary>
+        public static IEnumerable<FactorFileRow> Read(Stream file, out DateTime? factorFileMinimumDate)
+        {
+            factorFileMinimumDate = null;
+
+            var streamReader = new StreamReader(file, Encoding.UTF8);
+
+            string line;
+            var lines = new List<string>();
+            while (!streamReader.EndOfStream)
+            {
+                line = streamReader.ReadLine();
+                if (!string.IsNullOrWhiteSpace(line))
+                {
+                    lines.Add(line);
+                }
+            }
+
+            streamReader.Dispose();
+
+            return Parse(lines, out factorFileMinimumDate);
+        }
+
         /// <summary>
         /// Parses the lines as factor files rows while properly handling inf entries
         /// </summary>
         /// <param name="lines">The lines from the factor file to be parsed</param>
         /// <param name="factorFileMinimumDate">The minimum date from the factor file</param>
         /// <returns>An enumerable of factor file rows</returns>
-        public static List<CorporateFactorRow> Parse(IEnumerable<string> lines, out DateTime? factorFileMinimumDate)
+        public static List<FactorFileRow> Parse(IEnumerable<string> lines, out DateTime? factorFileMinimumDate)
         {
             factorFileMinimumDate = null;
 
-            var rows = new List<CorporateFactorRow>();
+            var rows = new List<FactorFileRow>();
 
             // parse factor file lines
             foreach (var line in lines)
@@ -123,7 +150,7 @@ namespace QuantConnect.Data.Auxiliary
                 }
             }
 
-            if (rows.Count > 0)
+            if (factorFileMinimumDate == null && rows.Count > 0)
             {
                 factorFileMinimumDate = rows.Min(ffr => ffr.Date).AddDays(-1);
             }
@@ -139,7 +166,7 @@ namespace QuantConnect.Data.Auxiliary
         /// <param name="dividend">The dividend to apply with reference price and distribution specified</param>
         /// <param name="exchangeHours">Exchange hours used for resolving the previous trading day</param>
         /// <returns>A new factor file row that applies the dividend to this row's factors</returns>
-        public CorporateFactorRow Apply(Dividend dividend, SecurityExchangeHours exchangeHours)
+        public FactorFileRow Apply(Dividend dividend, SecurityExchangeHours exchangeHours)
         {
             if (dividend.ReferencePrice == 0m)
             {
@@ -161,7 +188,7 @@ namespace QuantConnect.Data.Auxiliary
             // pfi = pf(i+1) * (C-D)/C
             var priceFactor = PriceFactor * (dividend.ReferencePrice - dividend.Distribution) / dividend.ReferencePrice;
 
-            return new CorporateFactorRow(
+            return new FactorFileRow(
                 previousTradingDay,
                 priceFactor,
                 SplitFactor,
@@ -177,7 +204,7 @@ namespace QuantConnect.Data.Auxiliary
         /// <param name="split">The split to apply with reference price and split factor specified</param>
         /// <param name="exchangeHours">Exchange hours used for resolving the previous trading day</param>
         /// <returns>A new factor file row that applies the split to this row's factors</returns>
-        public CorporateFactorRow Apply(Split split, SecurityExchangeHours exchangeHours)
+        public FactorFileRow Apply(Split split, SecurityExchangeHours exchangeHours)
         {
             if (split.Type == SplitType.Warning)
             {
@@ -200,7 +227,7 @@ namespace QuantConnect.Data.Auxiliary
                 ));
             }
 
-            return new CorporateFactorRow(
+            return new FactorFileRow(
                 previousTradingDay,
                 PriceFactor,
                 SplitFactor * split.SplitFactor,
@@ -212,14 +239,14 @@ namespace QuantConnect.Data.Auxiliary
         /// Creates a new dividend from this factor file row and the one chronologically in front of it
         /// This dividend may have a distribution of zero if this row doesn't represent a dividend
         /// </summary>
-        /// <param name="nextCorporateFactorRow">The next factor file row in time</param>
+        /// <param name="futureFactorFileRow">The next factor file row in time</param>
         /// <param name="symbol">The symbol to use for the dividend</param>
         /// <param name="exchangeHours">Exchange hours used for resolving the previous trading day</param>
         /// <param name="decimalPlaces">The number of decimal places to round the dividend's distribution to, defaulting to 2</param>
         /// <returns>A new dividend instance</returns>
-        public Dividend GetDividend(CorporateFactorRow nextCorporateFactorRow, Symbol symbol, SecurityExchangeHours exchangeHours, int decimalPlaces=2)
+        public Dividend GetDividend(FactorFileRow futureFactorFileRow, Symbol symbol, SecurityExchangeHours exchangeHours, int decimalPlaces=2)
         {
-            if (nextCorporateFactorRow.PriceFactor == 0m)
+            if (futureFactorFileRow.PriceFactor == 0m)
             {
                 throw new InvalidOperationException(Invariant(
                     $"Unable to resolve dividend for '{symbol.ID}' at {Date:yyyy-MM-dd}. Price factor is zero."
@@ -233,7 +260,7 @@ namespace QuantConnect.Data.Auxiliary
                 symbol,
                 previousTradingDay,
                 ReferencePrice,
-                PriceFactor / nextCorporateFactorRow.PriceFactor,
+                PriceFactor / futureFactorFileRow.PriceFactor,
                 decimalPlaces
             );
         }
@@ -242,13 +269,13 @@ namespace QuantConnect.Data.Auxiliary
         /// Creates a new split from this factor file row and the one chronologically in front of it
         /// This split may have a split factor of one if this row doesn't represent a split
         /// </summary>
-        /// <param name="nextCorporateFactorRow">The next factor file row in time</param>
+        /// <param name="futureFactorFileRow">The next factor file row in time</param>
         /// <param name="symbol">The symbol to use for the split</param>
         /// <param name="exchangeHours">Exchange hours used for resolving the previous trading day</param>
         /// <returns>A new split instance</returns>
-        public Split GetSplit(CorporateFactorRow nextCorporateFactorRow, Symbol symbol, SecurityExchangeHours exchangeHours)
+        public Split GetSplit(FactorFileRow futureFactorFileRow, Symbol symbol, SecurityExchangeHours exchangeHours)
         {
-            if (nextCorporateFactorRow.SplitFactor == 0m)
+            if (futureFactorFileRow.SplitFactor == 0m)
             {
                 throw new InvalidOperationException(Invariant(
                     $"Unable to resolve split for '{symbol.ID}' at {Date:yyyy-MM-dd}. Split factor is zero."
@@ -262,7 +289,7 @@ namespace QuantConnect.Data.Auxiliary
                 symbol,
                 previousTradingDay,
                 ReferencePrice,
-                SplitFactor / nextCorporateFactorRow.SplitFactor,
+                SplitFactor / futureFactorFileRow.SplitFactor,
                 SplitType.SplitOccurred
             );
         }
@@ -270,10 +297,10 @@ namespace QuantConnect.Data.Auxiliary
         /// <summary>
         /// Parses the specified line as a factor file row
         /// </summary>
-        private static CorporateFactorRow Parse(string line)
+        private static FactorFileRow Parse(string line)
         {
             var csv = line.Split(',');
-            return new CorporateFactorRow(
+            return new FactorFileRow(
                 QuantConnect.Parse.DateTimeExact(csv[0], DateFormat.EightCharacter, DateTimeStyles.None),
                 QuantConnect.Parse.Decimal(csv[1]),
                 QuantConnect.Parse.Decimal(csv[2]),
@@ -282,10 +309,9 @@ namespace QuantConnect.Data.Auxiliary
         }
 
         /// <summary>
-        /// Writes factor file row into it's file format
+        /// Writes this row to csv format
         /// </summary>
-        /// <remarks>CSV formatted</remarks>
-        public string GetFileFormat(string source = null)
+        public string ToCsv(string source = null)
         {
             source = source == null ? "" : $",{source}";
             return $"{Date.ToStringInvariant(DateFormat.EightCharacter)}," +

Common/Data/Auxiliary/FactorFileZipHelper.cs

@@ -36,26 +36,45 @@ namespace QuantConnect.Data.Auxiliary
         /// <summary>
         /// Reads the zip bytes as text and parses as FactorFileRows to create FactorFiles
         /// </summary>
-        public static IEnumerable<KeyValuePair<Symbol, IFactorProvider>> ReadFactorFileZip(Stream file, MapFileResolver mapFileResolver, string market, SecurityType securityType)
+        public static IEnumerable<KeyValuePair<Symbol, FactorFile>> ReadFactorFileZip(Stream file, MapFileResolver mapFileResolver, string market, SecurityType securityType)
         {
             if (file == null || file.Length == 0)
             {
-                return new Dictionary<Symbol, IFactorProvider>();
+                return new Dictionary<Symbol, FactorFile>();
             }
 
             var keyValuePairs = (
                     from kvp in Compression.Unzip(file)
                     let filename = kvp.Key
                     let lines = kvp.Value
-                    let factorFile = PriceScalingExtensions.SafeRead(Path.GetFileNameWithoutExtension(filename), lines, securityType)
+                    let factorFile = SafeRead(filename, lines)
                     let mapFile = mapFileResolver.GetByPermtick(factorFile.Permtick)
                     where mapFile != null
-                    select new KeyValuePair<Symbol, IFactorProvider>(GetSymbol(mapFile, market, securityType), factorFile)
+                    select new KeyValuePair<Symbol, FactorFile>(GetSymbol(mapFile, market, securityType), factorFile)
                 );
 
             return keyValuePairs;
         }
 
+        /// <summary>
+        /// Parses the contents as a FactorFile, if error returns a new empty factor file
+        /// </summary>
+        public static FactorFile SafeRead(string filename, IEnumerable<string> contents)
+        {
+            var permtick = Path.GetFileNameWithoutExtension(filename);
+            try
+            {
+                DateTime? minimumDate;
+                // FactorFileRow.Parse handles entries with 'inf' and exponential notation and provides the associated minimum tradeable date for these cases
+                // previously these cases were not handled causing an exception and returning an empty factor file
+                return new FactorFile(permtick, FactorFileRow.Parse(contents, out minimumDate), minimumDate);
+            }
+            catch
+            {
+                return new FactorFile(permtick, Enumerable.Empty<FactorFileRow>());
+            }
+        }
+
         private static Symbol GetSymbol(MapFile mapFile, string market, SecurityType securityType)
         {
             SecurityIdentifier sid;

Common/Data/Auxiliary/IFactorProvider.cs

@@ -1,49 +0,0 @@
-/*
-* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
-* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*
-*/
-
-using System;
-using System.Collections.Generic;
-
-namespace QuantConnect.Data.Auxiliary
-{
-    /// <summary>
-    /// Providers price scaling factors for a permanent tick
-    /// </summary>
-    public interface IFactorProvider : IEnumerable<IFactorRow>
-    {
-        /// <summary>
-        /// Gets the symbol this factor file represents
-        /// </summary>
-        public string Permtick { get; }
-
-        /// <summary>
-        /// The minimum tradeable date for the symbol
-        /// </summary>
-        /// <remarks>
-        /// Some factor files have INF split values, indicating that the stock has so many splits
-        /// that prices can't be calculated with correct numerical precision.
-        /// To allow backtesting these symbols, we need to move the starting date
-        /// forward when reading the data.
-        /// Known symbols: GBSN, JUNI, NEWL
-        /// </remarks>
-        public DateTime? FactorFileMinimumDate { get; set; }
-
-        /// <summary>
-        /// Gets the price factor for the specified search date
-        /// </summary>
-        decimal GetPriceFactor(DateTime searchDate, DataNormalizationMode dataNormalizationMode, DataMappingMode? dataMappingMode = null, uint contractOffset = 0);
-    }
-}

Common/Data/Auxiliary/IFactorRow.cs

@@ -1,36 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
-*/
-
-using System;
-
-namespace QuantConnect.Data.Auxiliary
-{
-    /// <summary>
-    /// Factor row abstraction. <see cref="IFactorProvider"/>
-    /// </summary>
-    public interface IFactorRow
-    {
-        /// <summary>
-        /// Gets the date associated with this data
-        /// </summary>
-        DateTime Date { get; }
-
-        /// <summary>
-        /// Writes factor file row into it's file format
-        /// </summary>
-        string GetFileFormat(string source = null);
-    }
-}

Common/Data/Auxiliary/LocalDiskFactorFileProvider.cs

@@ -28,14 +28,14 @@ namespace QuantConnect.Data.Auxiliary
     {
         private IMapFileProvider _mapFileProvider;
         private IDataProvider _dataProvider;
-        private readonly ConcurrentDictionary<Symbol, IFactorProvider> _cache;
+        private readonly ConcurrentDictionary<Symbol, FactorFile> _cache;
 
         /// <summary>
         /// Creates a new instance of the <see cref="LocalDiskFactorFileProvider"/>
         /// </summary>
         public LocalDiskFactorFileProvider()
         {
-            _cache = new ConcurrentDictionary<Symbol, IFactorProvider>();
+            _cache = new ConcurrentDictionary<Symbol, FactorFile>();
         }
 
         /// <summary>
@@ -55,17 +55,16 @@ namespace QuantConnect.Data.Auxiliary
         /// </summary>
         /// <param name="symbol">The security's symbol whose factor file we seek</param>
         /// <returns>The resolved factor file, or null if not found</returns>
-        public IFactorProvider Get(Symbol symbol)
+        public FactorFile Get(Symbol symbol)
         {
-            symbol = symbol.GetFactorFileSymbol();
-            IFactorProvider factorFile;
+            FactorFile factorFile;
             if (_cache.TryGetValue(symbol, out factorFile))
             {
                 return factorFile;
             }
 
             // we first need to resolve the map file to get a permtick, that's how the factor files are stored
-            var mapFileResolver = _mapFileProvider.Get(AuxiliaryDataKey.Create(symbol));
+            var mapFileResolver = _mapFileProvider.Get(CorporateActionsKey.Create(symbol));
             if (mapFileResolver == null)
             {
                 return GetFactorFile(symbol, symbol.Value);
@@ -83,12 +82,25 @@ namespace QuantConnect.Data.Auxiliary
         /// <summary>
         /// Checks that the factor file exists on disk, and if it does, loads it into memory
         /// </summary>
-        private IFactorProvider GetFactorFile(Symbol symbol, string permtick)
+        private FactorFile GetFactorFile(Symbol symbol, string permtick)
         {
+            FactorFile factorFile = null;
+
             var path = Path.Combine(Globals.CacheDataFolder, symbol.SecurityType.SecurityTypeToLower(), symbol.ID.Market, "factor_files", permtick.ToLowerInvariant() + ".csv");
 
-            var factorFile = PriceScalingExtensions.SafeRead(permtick, _dataProvider.ReadLines(path), symbol.SecurityType);
-            _cache.AddOrUpdate(symbol, factorFile, (s, c) => factorFile);
+            var factorFileStream = _dataProvider.Fetch(path);
+            if (factorFileStream != null)
+            {
+                factorFile = FactorFile.Read(permtick, factorFileStream);
+                factorFileStream.DisposeSafely();
+                _cache.AddOrUpdate(symbol, factorFile, (s, c) => factorFile);
+            }
+            else
+            {
+                // add null value to the cache, we don't want to check the disk multiple times
+                // but keep existing value if it exists, just in case
+                _cache.AddOrUpdate(symbol, factorFile, (s, oldValue) => oldValue);
+            }
             return factorFile;
         }
     }

Common/Data/Auxiliary/LocalDiskMapFileProvider.cs

@@ -29,7 +29,7 @@ namespace QuantConnect.Data.Auxiliary
     public class LocalDiskMapFileProvider : IMapFileProvider
     {
         private static int _wroteTraceStatement;
-        private readonly ConcurrentDictionary<AuxiliaryDataKey, MapFileResolver> _cache;
+        private readonly ConcurrentDictionary<CorporateActionsKey, MapFileResolver> _cache;
         private IDataProvider _dataProvider;
 
         /// <summary>
@@ -37,7 +37,7 @@ namespace QuantConnect.Data.Auxiliary
         /// </summary>
         public LocalDiskMapFileProvider()
         {
-            _cache = new ConcurrentDictionary<AuxiliaryDataKey, MapFileResolver>();
+            _cache = new ConcurrentDictionary<CorporateActionsKey, MapFileResolver>();
         }
 
         /// <summary>
@@ -53,14 +53,14 @@ namespace QuantConnect.Data.Auxiliary
         /// Gets a <see cref="MapFileResolver"/> representing all the map
         /// files for the specified market
         /// </summary>
-        /// <param name="auxiliaryDataKey">Key used to fetch a map file resolver. Specifying market and security type</param>
+        /// <param name="corporateActionsKey">Key used to fetch a map file resolver. Specifying market and security type</param>
         /// <returns>A <see cref="MapFileRow"/> containing all map files for the specified market</returns>
-        public MapFileResolver Get(AuxiliaryDataKey auxiliaryDataKey)
+        public MapFileResolver Get(CorporateActionsKey corporateActionsKey)
         {
-            return _cache.GetOrAdd(auxiliaryDataKey, GetMapFileResolver);
+            return _cache.GetOrAdd(corporateActionsKey, GetMapFileResolver);
         }
 
-        private MapFileResolver GetMapFileResolver(AuxiliaryDataKey key)
+        private MapFileResolver GetMapFileResolver(CorporateActionsKey key)
         {
             var securityType = key.SecurityType;
             var market = key.Market;

Common/Data/Auxiliary/LocalZipFactorFileProvider.cs

@@ -30,8 +30,8 @@ namespace QuantConnect.Data.Auxiliary
         private readonly object _lock;
         private IDataProvider _dataProvider;
         private IMapFileProvider _mapFileProvider;
-        private Dictionary<AuxiliaryDataKey, bool> _seededMarket;
-        private readonly Dictionary<Symbol, IFactorProvider> _factorFiles;
+        private Dictionary<CorporateActionsKey, bool> _seededMarket;
+        private readonly Dictionary<Symbol, FactorFile> _factorFiles;
 
         /// <summary>
         /// The cached refresh period for the factor files
@@ -44,8 +44,8 @@ namespace QuantConnect.Data.Auxiliary
         /// </summary>
         public LocalZipFactorFileProvider()
         {
-            _factorFiles = new Dictionary<Symbol, IFactorProvider>();
-            _seededMarket = new Dictionary<AuxiliaryDataKey, bool>();
+            _factorFiles = new Dictionary<Symbol, FactorFile>();
+            _seededMarket = new Dictionary<CorporateActionsKey, bool>();
             _lock = new object();
         }
 
@@ -72,10 +72,9 @@ namespace QuantConnect.Data.Auxiliary
         /// </summary>
         /// <param name="symbol">The security's symbol whose factor file we seek</param>
         /// <returns>The resolved factor file, or null if not found</returns>
-        public IFactorProvider Get(Symbol symbol)
+        public FactorFile Get(Symbol symbol)
         {
-            symbol = symbol.GetFactorFileSymbol();
-            var key = AuxiliaryDataKey.Create(symbol);
+            var key = CorporateActionsKey.Create(symbol);
             lock (_lock)
             {
                 if (!_seededMarket.ContainsKey(key))
@@ -84,15 +83,16 @@ namespace QuantConnect.Data.Auxiliary
                     _seededMarket[key] = true;
                 }
 
-                IFactorProvider factorFile;
-                if (!_factorFiles.TryGetValue(symbol, out factorFile))
+                FactorFile factorFile;
+                if (_factorFiles.TryGetValue(symbol, out factorFile))
                 {
-                    // Could not find factor file for symbol
-                    Log.Error($"LocalZipFactorFileProvider.Get({symbol}): No factor file found.");
-                    _factorFiles[symbol] = factorFile = symbol.GetEmptyFactorFile();
+                    return factorFile;
                 }
-                return factorFile;
             }
+
+            // Could not find factor file for symbol
+            Log.Error($"LocalZipFactorFileProvider.Get({symbol}): No factor file found.");
+            return null;
         }
 
         /// <summary>
@@ -103,13 +103,13 @@ namespace QuantConnect.Data.Auxiliary
             lock (_lock)
             {
                 // we clear the seeded markets so they are reloaded
-                _seededMarket = new Dictionary<AuxiliaryDataKey, bool>();
+                _seededMarket = new Dictionary<CorporateActionsKey, bool>();
             }
             _ = Task.Delay(CacheRefreshPeriod).ContinueWith(_ => StartExpirationTask());
         }
 
         /// Hydrate the <see cref="_factorFiles"/> from the latest zipped factor file on disk
-        private void HydrateFactorFileFromLatestZip(AuxiliaryDataKey key)
+        private void HydrateFactorFileFromLatestZip(CorporateActionsKey key)
         {
             var market = key.Market;
             // start the search with yesterday, today's file will be available tomorrow

Common/Data/Auxiliary/LocalZipMapFileProvider.cs

@@ -27,7 +27,7 @@ namespace QuantConnect.Data.Auxiliary
     /// </summary>
     public class LocalZipMapFileProvider : IMapFileProvider
     {
-        private Dictionary<AuxiliaryDataKey, MapFileResolver> _cache;
+        private Dictionary<CorporateActionsKey, MapFileResolver> _cache;
         private IDataProvider _dataProvider;
         private object _lock;
 
@@ -43,7 +43,7 @@ namespace QuantConnect.Data.Auxiliary
         public LocalZipMapFileProvider()
         {
             _lock = new object();
-            _cache = new Dictionary<AuxiliaryDataKey, MapFileResolver>();
+            _cache = new Dictionary<CorporateActionsKey, MapFileResolver>();
         }
 
         /// <summary>
@@ -64,18 +64,18 @@ namespace QuantConnect.Data.Auxiliary
         /// <summary>
         /// Gets a <see cref="MapFileResolver"/> representing all the map files for the specified market
         /// </summary>
-        /// <param name="auxiliaryDataKey">Key used to fetch a map file resolver. Specifying market and security type</param>
+        /// <param name="corporateActionsKey">Key used to fetch a map file resolver. Specifying market and security type</param>
         /// <returns>A <see cref="MapFileResolver"/> containing all map files for the specified market</returns>
-        public MapFileResolver Get(AuxiliaryDataKey auxiliaryDataKey)
+        public MapFileResolver Get(CorporateActionsKey corporateActionsKey)
         {
             MapFileResolver result;
             // we use a lock so that only 1 thread loads the map file resolver while the rest wait
             // else we could have multiple threads loading the map file resolver at the same time!
             lock (_lock)
             {
-                if (!_cache.TryGetValue(auxiliaryDataKey, out result))
+                if (!_cache.TryGetValue(corporateActionsKey, out result))
                 {
-                    _cache[auxiliaryDataKey] = result = GetMapFileResolver(auxiliaryDataKey);
+                    _cache[corporateActionsKey] = result = GetMapFileResolver(corporateActionsKey);
                 }
             }
             return result;
@@ -89,14 +89,14 @@ namespace QuantConnect.Data.Auxiliary
             lock (_lock)
             {
                 // we clear the seeded markets so they are reloaded
-                _cache = new Dictionary<AuxiliaryDataKey, MapFileResolver>();
+                _cache = new Dictionary<CorporateActionsKey, MapFileResolver>();
             }
             _ = Task.Delay(CacheRefreshPeriod).ContinueWith(_ => StartExpirationTask());
         }
 
-        private MapFileResolver GetMapFileResolver(AuxiliaryDataKey auxiliaryDataKey)
+        private MapFileResolver GetMapFileResolver(CorporateActionsKey corporateActionsKey)
         {
-            var market = auxiliaryDataKey.Market;
+            var market = corporateActionsKey.Market;
             var timestamp = DateTime.UtcNow.ConvertFromUtc(TimeZones.NewYork);
             var todayNewYork = timestamp.Date;
             var yesterdayNewYork = todayNewYork.AddDays(-1);
@@ -106,7 +106,7 @@ namespace QuantConnect.Data.Auxiliary
             var date = yesterdayNewYork;
             do
             {
-                var zipFileName = MapFileZipHelper.GetMapFileZipFileName(market, date, auxiliaryDataKey.SecurityType);
+                var zipFileName = MapFileZipHelper.GetMapFileZipFileName(market, date, corporateActionsKey.SecurityType);
 
                 // Fetch a stream for our zip from our data provider
                 var stream = _dataProvider.Fetch(zipFileName);
@@ -115,7 +115,7 @@ namespace QuantConnect.Data.Auxiliary
                 if (stream != null)
                 {
                     Log.Trace("LocalZipMapFileProvider.Get({0}): Fetched map files for: {1} NY", market, date.ToShortDateString());
-                    var result =  new MapFileResolver(MapFileZipHelper.ReadMapFileZip(stream, market, auxiliaryDataKey.SecurityType));
+                    var result =  new MapFileResolver(MapFileZipHelper.ReadMapFileZip(stream, market, corporateActionsKey.SecurityType));
                     stream.DisposeSafely();
                     return result;
                 }

Common/Data/Auxiliary/MapFilePrimaryExchangeProvider.cs

@@ -48,7 +48,7 @@ namespace QuantConnect.Data.Auxiliary
             Exchange primaryExchange;
             if (!_primaryExchangeBySid.TryGetValue(securityIdentifier, out primaryExchange))
             {
-                var mapFile = _mapFileProvider.Get(AuxiliaryDataKey.Create(securityIdentifier))
+                var mapFile = _mapFileProvider.Get(CorporateActionsKey.Create(securityIdentifier))
                     .ResolveMapFile(securityIdentifier.Symbol, securityIdentifier.Date);
                 if (mapFile != null && mapFile.Any())
                 {

Common/Data/Auxiliary/MapFileResolver.cs

@@ -139,7 +139,7 @@ namespace QuantConnect.Data.Auxiliary
             // secondary search for exact mapping, find path than ends with symbol.csv
             MapFile mapFile;
             if (!_mapFilesByPermtick.TryGetValue(symbol, out mapFile)
-                || mapFile.FirstDate > date && date != SecurityIdentifier.DefaultDate)
+                || mapFile.FirstDate > date && date != Time.BeginningOfTime)
             {
                 return new MapFile(symbol, Enumerable.Empty<MapFileRow>());
             }

Common/Data/Auxiliary/MapFileRow.cs

@@ -184,7 +184,7 @@ namespace QuantConnect.Data.Auxiliary
             {
                 encodedExchange = $",{PrimaryExchange.Code}";
             }
-            var mappingMode = DataMappingMode != null ? $",{(int)DataMappingMode}" : string.Empty;
+            var mappingMode = DataMappingMode != null ? $",{DataMappingMode}" : string.Empty;
             return $"{Date.ToStringInvariant(DateFormat.EightCharacter)},{MappedSymbol.ToLowerInvariant()}{encodedExchange}{mappingMode}";
         }
 

Common/Data/Auxiliary/MappingContractFactorProvider.cs

@@ -1,99 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
-*/
-
-using System;
-using System.Linq;
-using System.Collections.Generic;
-
-namespace QuantConnect.Data.Auxiliary
-{
-    /// <summary>
-    /// Mapping related factor provider. Factors based on price differences on mapping dates
-    /// </summary>
-    public class MappingContractFactorProvider : FactorFile<MappingContractFactorRow>
-    {
-        /// <summary>
-        ///Creates a new instance
-        /// </summary>
-        public MappingContractFactorProvider(string permtick, IEnumerable<MappingContractFactorRow> data, DateTime? factorFileMinimumDate = null)
-            : base(permtick, data, factorFileMinimumDate)
-        {
-        }
-
-        /// <summary>
-        /// Gets the price scale factor for the specified search date
-        /// </summary>
-        public override decimal GetPriceFactor(DateTime searchDate, DataNormalizationMode dataNormalizationMode, DataMappingMode? dataMappingMode = null, uint contractOffset = 0)
-        {
-            if (dataNormalizationMode == DataNormalizationMode.Raw)
-            {
-                return 0;
-            }
-
-            var factor = 1m;
-            if (dataNormalizationMode is DataNormalizationMode.BackwardsPanamaCanal or DataNormalizationMode.ForwardPanamaCanal)
-            {
-                // default value depends on the data mode
-                factor = 0;
-            }
-
-            for (var i = 0; i < _reversedFactorFileDates.Count; i++)
-            {
-                var factorDate = _reversedFactorFileDates[i];
-                if (factorDate.Date < searchDate.Date)
-                {
-                    break;
-                }
-
-                var factorFileRow = SortedFactorFileData[factorDate];
-                switch (dataNormalizationMode)
-                {
-                    case DataNormalizationMode.BackwardsRatio:
-                    {
-                        var row = factorFileRow.FirstOrDefault(row => row.DataMappingMode == dataMappingMode);
-                        if (row != null && row.BackwardsRatioScale.Count > contractOffset)
-                        {
-                            factor = row.BackwardsRatioScale[(int)contractOffset];
-                        }
-                        break;
-                    }
-                    case DataNormalizationMode.BackwardsPanamaCanal:
-                    {
-                        var row = factorFileRow.FirstOrDefault(row => row.DataMappingMode == dataMappingMode);
-                        if (row != null && row.BackwardsPanamaCanalScale.Count > contractOffset)
-                        {
-                            factor = row.BackwardsPanamaCanalScale[(int)contractOffset];
-                        }
-                        break;
-                    }
-                    case DataNormalizationMode.ForwardPanamaCanal:
-                    {
-                        var row = factorFileRow.FirstOrDefault(row => row.DataMappingMode == dataMappingMode);
-                        if (row != null && row.ForwardPanamaCanalScale.Count > contractOffset)
-                        {
-                            factor = row.ForwardPanamaCanalScale[(int)contractOffset];
-                        }
-                        break;
-                    }
-                    default:
-                        throw new ArgumentOutOfRangeException();
-                }
-            }
-
-            return factor;
-        }
-    }
-}

Common/Data/Auxiliary/MappingContractFactorRow.cs

@@ -1,99 +0,0 @@
-/*
-* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
-* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*
-*/
-
-using System;
-using System.Linq;
-using Newtonsoft.Json;
-using System.Collections.Generic;
-
-namespace QuantConnect.Data.Auxiliary
-{
-    /// <summary>
-    /// Collection of factors for continuous contracts and their back months contracts for a specific mapping mode <see cref="DataMappingMode"/> and date
-    /// </summary>
-    public class MappingContractFactorRow : IFactorRow
-    {
-        /// <summary>
-        /// Gets the date associated with this data
-        /// </summary>
-        public DateTime Date { get; set; }
-
-        /// <summary>
-        /// Backwards ratio price scaling factors for the front month [index 0] and it's 'i' back months [index 0 + i]
-        /// <see cref="DataNormalizationMode.BackwardsRatio"/>
-        /// </summary>
-        public IReadOnlyList<decimal> BackwardsRatioScale { get; set;  } = new List<decimal>();
-
-        /// <summary>
-        /// Backwards Panama Canal price scaling factors for the front month [index 0] and it's 'i' back months [index 0 + i]
-        /// <see cref="DataNormalizationMode.BackwardsPanamaCanal"/>
-        /// </summary>
-        public IReadOnlyList<decimal> BackwardsPanamaCanalScale { get; set;  } = new List<decimal>();
-
-        /// <summary>
-        /// Forward Panama Canal price scaling factors for the front month [index 0] and it's 'i' back months [index 0 + i]
-        /// <see cref="DataNormalizationMode.ForwardPanamaCanal"/>
-        /// </summary>
-        public IReadOnlyList<decimal> ForwardPanamaCanalScale { get; set;  } = new List<decimal>();
-
-        /// <summary>
-        /// Allows the consumer to specify a desired mapping mode
-        /// </summary>
-        public DataMappingMode? DataMappingMode { get; set; }
-
-        /// <summary>
-        /// Empty constructor for json converter
-        /// </summary>
-        public MappingContractFactorRow()
-        {
-        }
-
-        /// <summary>
-        /// Writes factor file row into it's file format
-        /// </summary>
-        /// <remarks>Json formatted</remarks>
-        public string GetFileFormat(string source = null)
-        {
-            return JsonConvert.SerializeObject(this);
-        }
-
-        /// <summary>
-        /// Parses the lines as factor files rows while properly handling inf entries
-        /// </summary>
-        /// <param name="lines">The lines from the factor file to be parsed</param>
-        /// <param name="factorFileMinimumDate">The minimum date from the factor file</param>
-        /// <returns>An enumerable of factor file rows</returns>
-        public static List<MappingContractFactorRow> Parse(IEnumerable<string> lines, out DateTime? factorFileMinimumDate)
-        {
-            factorFileMinimumDate = null;
-
-            var rows = new List<MappingContractFactorRow>();
-
-            // parse factor file lines
-            foreach (var line in lines)
-            {
-                rows.Add(JsonConvert.DeserializeObject<MappingContractFactorRow>(line));
-            }
-
-            if (rows.Count > 0)
-            {
-                factorFileMinimumDate = rows.Min(ffr => ffr.Date).AddDays(-1);
-            }
-
-            return rows;
-        }
-    }
-}

Common/Data/Auxiliary/MappingExtensions.cs

@@ -36,7 +36,7 @@ namespace QuantConnect.Data.Auxiliary
             var resolver = MapFileResolver.Empty;
             if(dataConfig.TickerShouldBeMapped())
             {
-                resolver = mapFileProvider.Get(AuxiliaryDataKey.Create(dataConfig.Symbol));
+                resolver = mapFileProvider.Get(CorporateActionsKey.Create(dataConfig.Symbol));
             }
             return resolver.ResolveMapFile(dataConfig.Symbol , dataConfig.Type.Name, dataConfig.DataMappingMode);
         }

Common/Data/Auxiliary/PriceScalingExtensions.cs

@@ -1,107 +0,0 @@
-/*
-* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
-* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*
-*/
-
-using System;
-using System.Linq;
-using System.Collections.Generic;
-
-namespace QuantConnect.Data.Auxiliary
-{
-    /// <summary>
-    /// Set of helper methods for factor files and price scaling operations
-    /// </summary>
-    public static class PriceScalingExtensions
-    {
-        /// <summary>
-        /// Resolves the price scale for a date given a factor file and required settings
-        /// </summary>
-        /// <param name="factorFile">The factor file to use</param>
-        /// <param name="dateTime">The date for the price scale lookup</param>
-        /// <param name="normalizationMode">The price normalization mode requested</param>
-        /// <param name="contractOffset">The contract offset, useful for continuous contracts</param>
-        /// <param name="dataMappingMode">The data mapping mode used, useful for continuous contracts</param>
-        /// <returns>The price scale to use</returns>
-        public static decimal GetPriceScale(
-            this IFactorProvider factorFile,
-            DateTime dateTime,
-            DataNormalizationMode normalizationMode,
-            uint contractOffset = 0,
-            DataMappingMode? dataMappingMode = null
-            )
-        {
-            if (factorFile == null)
-            {
-                if (normalizationMode is DataNormalizationMode.BackwardsPanamaCanal or DataNormalizationMode.ForwardPanamaCanal)
-                {
-                    return 0;
-                }
-                return 1;
-            }
-
-            return factorFile.GetPriceFactor(dateTime, normalizationMode, dataMappingMode, contractOffset);
-        }
-
-        /// <summary>
-        /// Determines the symbol to use to fetch it's factor file
-        /// </summary>
-        /// <remarks>This is useful for futures where the symbol to use is the canonical</remarks>
-        public static Symbol GetFactorFileSymbol(this Symbol symbol)
-        {
-            return symbol.SecurityType == SecurityType.Future ? symbol.Canonical : symbol;
-        }
-
-        /// <summary>
-        /// Helper method to return an empty factor file
-        /// </summary>
-        public static IFactorProvider GetEmptyFactorFile(this Symbol symbol)
-        {
-            if (symbol.SecurityType == SecurityType.Future)
-            {
-                return new MappingContractFactorProvider(symbol.ID.Symbol, Enumerable.Empty<MappingContractFactorRow>());
-            }
-            return new CorporateFactorProvider(symbol.ID.Symbol, Enumerable.Empty<CorporateFactorRow>());
-        }
-
-        /// <summary>
-        /// Parses the contents as a FactorFile, if error returns a new empty factor file
-        /// </summary>
-        public static IFactorProvider SafeRead(string permtick, IEnumerable<string> contents, SecurityType securityType)
-        {
-            try
-            {
-                DateTime? minimumDate;
-
-                contents = contents.Distinct();
-
-                if (securityType == SecurityType.Future)
-                {
-                    return new MappingContractFactorProvider(permtick, MappingContractFactorRow.Parse(contents, out minimumDate), minimumDate);
-                }
-                // FactorFileRow.Parse handles entries with 'inf' and exponential notation and provides the associated minimum tradeable date for these cases
-                // previously these cases were not handled causing an exception and returning an empty factor file
-                return new CorporateFactorProvider(permtick, CorporateFactorRow.Parse(contents, out minimumDate), minimumDate);
-            }
-            catch (Exception e)
-            {
-                if (securityType == SecurityType.Future)
-                {
-                    return new MappingContractFactorProvider(permtick, Enumerable.Empty<MappingContractFactorRow>());
-                }
-                return new CorporateFactorProvider(permtick, Enumerable.Empty<CorporateFactorRow>());
-            }
-        }
-    }
-}

Common/Data/BaseData.cs

@@ -63,11 +63,6 @@ namespace QuantConnect.Data
         /// </summary>
         protected static readonly List<Resolution> HighResolution = new List<Resolution> { Resolution.Minute, Resolution.Second, Resolution.Tick };
 
-        /// <summary>
-        /// A list of resolutions support by Options
-        /// </summary>
-        protected static readonly List<Resolution> OptionResolutions = new List<Resolution> { Resolution.Daily, Resolution.Hour, Resolution.Minute };
-
         /// <summary>
         /// Market Data Type of this data - does it come in individual price packets or is it grouped into OHLC.
         /// </summary>
@@ -237,7 +232,7 @@ namespace QuantConnect.Data
         /// custom data types can override it</remarks>
         public virtual List<Resolution> SupportedResolutions()
         {
-            if (Symbol.SecurityType == SecurityType.Index)
+            if (Symbol.SecurityType.IsOption() || Symbol.SecurityType == SecurityType.Index)
             {
                 return MinuteResolution;
             }
@@ -247,11 +242,6 @@ namespace QuantConnect.Data
                 return HighResolution;
             }
 
-            if (Symbol.SecurityType.IsOption())
-            {
-                return OptionResolutions;
-            }
-
             return AllResolutions;
         }
 

Common/Data/DiskDataCacheProvider.cs

@@ -1,104 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
-*/
-
-using System.IO;
-using Ionic.Zip;
-using QuantConnect.Interfaces;
-using QuantConnect.Logging;
-using QuantConnect.Util;
-
-namespace QuantConnect.Data
-{
-    /// <summary>
-    /// Simple data cache provider, writes and reads directly from disk
-    /// Used as default for <see cref="LeanDataWriter"/>
-    /// </summary>
-    public class DiskDataCacheProvider : IDataCacheProvider
-    {
-        /// <summary>
-        /// Property indicating the data is temporary in nature and should not be cached.
-        /// </summary>
-        public bool IsDataEphemeral => false;
-
-        /// <summary>
-        /// Fetch data from the cache
-        /// </summary>
-        /// <param name="key">A string representing the key of the cached data</param>
-        /// <returns>An <see cref="Stream"/> of the cached data</returns>
-        public Stream Fetch(string key)
-        {
-            LeanData.ParseKey(key, out var filePath, out var entryName);
-
-            if (!File.Exists(filePath))
-            {
-                return null;
-            }
-
-            try
-            {
-                using (var zip = ZipFile.Read(filePath))
-                {
-                    ZipEntry entry;
-                    if (entryName.IsNullOrEmpty())
-                    {
-                        // Return the first entry
-                        entry = zip[0];
-                    }
-                    else
-                    {
-                        // Attempt to find our specific entry
-                        if (!zip.ContainsEntry(entryName))
-                        {
-                            return null;
-                        }
-
-                        entry = zip[entryName];
-                    }
-
-                    // Extract our entry and return it
-                    var stream = new MemoryStream();
-                    entry.Extract(stream);
-                    stream.Position = 0;
-                    return stream;
-                }
-            }
-            catch (ZipException exception)
-            {
-                Log.Error("DiskDataCacheProvider.Fetch(): Corrupt file: " + key + " Error: " + exception);
-                return null;
-            }
-        }
-
-        /// <summary>
-        /// Store the data in the cache. Not implemented in this instance of the IDataCacheProvider
-        /// </summary>
-        /// <param name="key">The source of the data, used as a key to retrieve data in the cache</param>
-        /// <param name="data">The data as a byte array</param>
-        public void Store(string key, byte[] data)
-        {
-            LeanData.ParseKey(key, out var filePath, out var entryName);
-            Compression.ZipCreateAppendData(filePath, entryName, data, true);
-        }
-
-        /// <summary>
-        /// Dispose for this class
-        /// </summary>
-        public void Dispose()
-        {
-            //NOP
-        }
-    }
-}

Common/Data/LeanDataWriter.cs

@@ -23,9 +23,6 @@ using QuantConnect.Logging;
 using QuantConnect.Interfaces;
 using QuantConnect.Securities;
 using System.Collections.Generic;
-using System.Collections.Immutable;
-using System.Globalization;
-using System.Threading.Tasks;
 
 namespace QuantConnect.Data
 {
@@ -37,9 +34,9 @@ namespace QuantConnect.Data
         private readonly Symbol _symbol;
         private readonly string _dataDirectory;
         private readonly TickType _tickType;
+        private readonly bool _appendToZips;
         private readonly Resolution _resolution;
         private readonly SecurityType _securityType;
-        private readonly IDataCacheProvider _dataCacheProvider;
 
         /// <summary>
         /// Create a new lean data writer to this base data directory.
@@ -48,12 +45,11 @@ namespace QuantConnect.Data
         /// <param name="dataDirectory">Base data directory</param>
         /// <param name="resolution">Resolution of the desired output data</param>
         /// <param name="tickType">The tick type</param>
-        public LeanDataWriter(Resolution resolution, Symbol symbol, string dataDirectory, TickType tickType = TickType.Trade, IDataCacheProvider dataCacheProvider = null) : this(
+        public LeanDataWriter(Resolution resolution, Symbol symbol, string dataDirectory, TickType tickType = TickType.Trade) : this(
             dataDirectory,
             resolution,
             symbol.ID.SecurityType,
-            tickType,
-            dataCacheProvider
+            tickType
         )
         {
             _symbol = symbol;
@@ -76,13 +72,13 @@ namespace QuantConnect.Data
         /// <param name="resolution">Resolution of the desired output data</param>
         /// <param name="securityType">The security type</param>
         /// <param name="tickType">The tick type</param>
-        public LeanDataWriter(string dataDirectory, Resolution resolution, SecurityType securityType, TickType tickType, IDataCacheProvider dataCacheProvider = null)
+        public LeanDataWriter(string dataDirectory, Resolution resolution, SecurityType securityType, TickType tickType)
         {
             _dataDirectory = dataDirectory;
             _resolution = resolution;
             _securityType = securityType;
             _tickType = tickType;
-            _dataCacheProvider = dataCacheProvider ?? new DiskDataCacheProvider();
+            _appendToZips = securityType == SecurityType.Future || securityType.IsOption();
         }
 
         /// <summary>
@@ -91,63 +87,18 @@ namespace QuantConnect.Data
         /// <param name="source">IEnumerable source of the data: sorted from oldest to newest.</param>
         public void Write(IEnumerable<BaseData> source)
         {
-            var lastTime = DateTime.MinValue;
-            var outputFile = string.Empty;
-            var currentFileData = new List<(DateTime, string)>();
-            var writeTasks = new Queue<Task>();
-
-            foreach (var data in source)
+            switch (_resolution)
             {
-                // Ensure the data is sorted as a safety check
-                if (data.Time < lastTime) throw new Exception("The data must be pre-sorted from oldest to newest");
+                case Resolution.Daily:
+                case Resolution.Hour:
+                    WriteDailyOrHour(source);
+                    break;
 
-                // Update our output file
-                // Only do this on date change, because we know we don't have a any data zips smaller than a day, saves time
-                if (data.Time.Date != lastTime.Date)
-                {
-                    // Get the latest file name, if it has changed, we have entered a new file, write our current data to file
-                    var latestOutputFile = GetZipOutputFileName(_dataDirectory, data.Time);
-                    if (outputFile.IsNullOrEmpty() || outputFile != latestOutputFile)
-                    {
-                        if (!currentFileData.IsNullOrEmpty())
-                        {
-                            // Launch a write task for the current file and data set
-                            var file = outputFile;
-                            var fileData = currentFileData;
-                            writeTasks.Enqueue(Task.Run(() =>
-                            {
-                                WriteFile(file, fileData, data.Time);
-                            }));
-                        }
-
-                        // Reset our dictionary and store new output file
-                        currentFileData = new List<(DateTime, string)>();
-                        outputFile = latestOutputFile;
-                    }
-                }
-
-                // Add data to our current dictionary
-                var line = LeanData.GenerateLine(data, _securityType, _resolution);
-                currentFileData.Add((data.Time, line));
-
-                // Update our time
-                lastTime = data.Time;
-            }
-
-            // Finish off my processing the last file as well
-            if (!currentFileData.IsNullOrEmpty())
-            {
-                writeTasks.Enqueue(Task.Run(() =>
-                {
-                    WriteFile(outputFile, currentFileData, lastTime);
-                }));
-            }
-
-            // Wait for all our write tasks to finish
-            while (writeTasks.Count > 0)
-            {
-                var task = writeTasks.Dequeue();
-                task.Wait();
+                case Resolution.Minute:
+                case Resolution.Second:
+                case Resolution.Tick:
+                    WriteMinuteOrSecondOrTick(source);
+                    break;
             }
         }
 
@@ -170,6 +121,11 @@ namespace QuantConnect.Data
                 throw new ArgumentException("DownloadAndSave(): The tick type must be Trade or Quote.");
             }
 
+            if (_securityType != SecurityType.Future && _securityType != SecurityType.Option && _securityType != SecurityType.FutureOption)
+            {
+                throw new ArgumentException($"DownloadAndSave(): The security type must be {SecurityType.Future} or {SecurityType.Option}.");
+            }
+
             if (symbols.Any(x => x.SecurityType != _securityType))
             {
                 throw new ArgumentException($"DownloadAndSave(): All symbols must have {_securityType} security type.");
@@ -192,6 +148,9 @@ namespace QuantConnect.Data
             var exchangeHours = marketHoursDatabase.GetExchangeHours(canonicalSymbol.ID.Market, canonicalSymbol, _securityType);
             var dataTimeZone = marketHoursDatabase.GetDataTimeZone(canonicalSymbol.ID.Market, canonicalSymbol, _securityType);
 
+            var historyBySymbol = new Dictionary<Symbol, List<IGrouping<DateTime, BaseData>>>();
+            var historyBySymbolDailyOrHour = new Dictionary<Symbol, List<BaseData>>();
+
             foreach (var symbol in symbols)
             {
                 var historyRequest = new HistoryRequest(
@@ -213,95 +172,328 @@ namespace QuantConnect.Data
                     .Select(
                         x =>
                         {
-                            // Convert to date timezone before we write it
                             x.Time = x.Time.ConvertTo(exchangeHours.TimeZone, dataTimeZone);
                             return x;
                         })
                     .ToList();
 
-                // Generate a writer for this data and write it
-                var writer = new LeanDataWriter(_resolution, symbol, _dataDirectory, _tickType);
-                writer.Write(history);
+                if (_resolution == Resolution.Daily || _resolution == Resolution.Hour)
+                {
+                    historyBySymbolDailyOrHour.Add(symbol, history);
+                }
+                else
+                {
+                    // group by date in DataTimeZone
+                    var historyByDate = history.GroupBy(x => x.Time.Date).ToList();
+                    historyBySymbol.Add(symbol, historyByDate);
+                }
+            }
+
+            if (_resolution == Resolution.Daily || _resolution == Resolution.Hour)
+            {
+                SaveDailyOrHour(symbols, canonicalSymbol, historyBySymbolDailyOrHour);
+            }
+            else
+            {
+                SaveMinuteOrSecondOrTick(symbols, startTimeUtc, endTimeUtc, canonicalSymbol, historyBySymbol);
+            }
+        }
+
+        private void SaveDailyOrHour(
+            List<Symbol> symbols,
+            Symbol canonicalSymbol,
+            IReadOnlyDictionary<Symbol, List<BaseData>> historyBySymbol)
+        {
+            var zipFileName = Path.Combine(
+                _dataDirectory,
+                LeanData.GenerateRelativeZipFilePath(canonicalSymbol, DateTime.MinValue, _resolution, _tickType));
+
+            var folder = Path.GetDirectoryName(zipFileName);
+            if (!Directory.Exists(folder))
+            {
+                Directory.CreateDirectory(folder);
+            }
+
+            using (var zip = new ZipFile(zipFileName))
+            {
+                foreach (var symbol in symbols)
+                {
+                    // Load new data rows into a SortedDictionary for easy merge/update
+                    var newRows = new SortedDictionary<DateTime, string>(historyBySymbol[symbol]
+                        .ToDictionary(x => x.Time, x => LeanData.GenerateLine(x, _securityType, _resolution)));
+
+                    var rows = new SortedDictionary<DateTime, string>();
+
+                    var zipEntryName = LeanData.GenerateZipEntryName(symbol, DateTime.MinValue, _resolution, _tickType);
+
+                    if (zip.ContainsEntry(zipEntryName))
+                    {
+                        // If file exists, we load existing data and perform merge
+                        using (var stream = new MemoryStream())
+                        {
+                            zip[zipEntryName].Extract(stream);
+                            stream.Seek(0, SeekOrigin.Begin);
+
+                            using (var reader = new StreamReader(stream))
+                            {
+                                string line;
+                                while ((line = reader.ReadLine()) != null)
+                                {
+                                    var time = Parse.DateTimeExact(line.Substring(0, DateFormat.TwelveCharacter.Length), DateFormat.TwelveCharacter);
+                                    rows[time] = line;
+                                }
+                            }
+                        }
+
+                        foreach (var kvp in newRows)
+                        {
+                            rows[kvp.Key] = kvp.Value;
+                        }
+                    }
+                    else
+                    {
+                        // No existing file, just use the new data
+                        rows = newRows;
+                    }
+
+                    // Loop through the SortedDictionary and write to zip entry
+                    var sb = new StringBuilder();
+                    foreach (var kvp in rows)
+                    {
+                        // Build the line and append it to the file
+                        sb.AppendLine(kvp.Value);
+                    }
+
+                    // Write the zip entry
+                    if (sb.Length > 0)
+                    {
+                        if (zip.ContainsEntry(zipEntryName))
+                        {
+                            zip.RemoveEntry(zipEntryName);
+                        }
+
+                        zip.AddEntry(zipEntryName, sb.ToString());
+                    }
+                }
+
+                if (zip.Count > 0)
+                {
+                    zip.Save();
+                }
+            }
+        }
+
+        private void SaveMinuteOrSecondOrTick(
+            List<Symbol> symbols,
+            DateTime startTimeUtc,
+            DateTime endTimeUtc,
+            Symbol canonicalSymbol,
+            IReadOnlyDictionary<Symbol, List<IGrouping<DateTime, BaseData>>> historyBySymbol)
+        {
+            var date = startTimeUtc;
+            while (date <= endTimeUtc)
+            {
+                var zipFileName = Path.Combine(
+                    _dataDirectory,
+                    LeanData.GenerateRelativeZipFilePath(canonicalSymbol, date, _resolution, _tickType));
+
+                var folder = Path.GetDirectoryName(zipFileName);
+                if (!Directory.Exists(folder))
+                {
+                    Directory.CreateDirectory(folder);
+                }
+
+                if (File.Exists(zipFileName) && !_appendToZips)
+                {
+                    File.Delete(zipFileName);
+                }
+
+                using (var zip = new ZipFile(zipFileName))
+                {
+                    foreach (var symbol in symbols)
+                    {
+                        var zipEntryName = LeanData.GenerateZipEntryName(symbol, date, _resolution, _tickType);
+
+                        foreach (var group in historyBySymbol[symbol])
+                        {
+                            if (group.Key == date.Date)
+                            {
+                                var sb = new StringBuilder();
+                                foreach (var row in group)
+                                {
+                                    var line = LeanData.GenerateLine(row, _securityType, _resolution);
+                                    sb.AppendLine(line);
+                                }
+
+                                if (_appendToZips && zip.ContainsEntry(zipEntryName))
+                                {
+                                    zip.RemoveEntry(zipEntryName);
+                                }
+
+                                zip.AddEntry(zipEntryName, sb.ToString());
+                                break;
+                            }
+                        }
+                    }
+
+                    if (zip.Count > 0)
+                    {
+                        zip.Save();
+                    }
+                }
+
+                date = date.AddDays(1);
+            }
+        }
+
+        /// <summary>
+        /// Write out the data in LEAN format (minute, second or tick resolutions)
+        /// </summary>
+        /// <param name="source">IEnumerable source of the data: sorted from oldest to newest.</param>
+        /// <remarks>This function overwrites existing data files</remarks>
+        private void WriteMinuteOrSecondOrTick(IEnumerable<BaseData> source)
+        {
+            var lines = new List<string>();
+            var lastTime = new DateTime();
+
+            // Loop through all the data and write to file as we go
+            foreach (var data in source)
+            {
+                // Ensure the data is sorted
+                if (data.Time < lastTime) throw new Exception("The data must be pre-sorted from oldest to newest");
+
+                // Based on the security type and resolution, write the data to the zip file
+                if (lastTime != DateTime.MinValue && data.Time.Date > lastTime.Date)
+                {
+                    // Write and clear the file contents
+                    var outputFile = GetZipOutputFileName(_dataDirectory, lastTime);
+                    WriteFile(outputFile, lines, lastTime);
+                    lines.Clear();
+                }
+
+                lastTime = data.Time;
+
+                // Build the line and append it to the file
+                lines.Add(LeanData.GenerateLine(data, _securityType, _resolution));
+            }
+
+            // Write the last file
+            if (lines.Count > 0)
+            {
+                var outputFile = GetZipOutputFileName(_dataDirectory, lastTime);
+                WriteFile(outputFile, lines, lastTime);
+            }
+        }
+
+        /// <summary>
+        /// Write out the data in LEAN format (daily or hour resolutions)
+        /// </summary>
+        /// <param name="source">IEnumerable source of the data: sorted from oldest to newest.</param>
+        /// <remarks>This function performs a merge (insert/append/overwrite) with the existing Lean zip file</remarks>
+        private void WriteDailyOrHour(IEnumerable<BaseData> source)
+        {
+            var lines = new List<string>();
+            var lastTime = new DateTime();
+
+            // Determine file path
+            var outputFile = GetZipOutputFileName(_dataDirectory, lastTime);
+
+            // Load new data rows into a SortedDictionary for easy merge/update
+            var newRows = new SortedDictionary<DateTime, string>(source.ToDictionary(x => x.Time, x => LeanData.GenerateLine(x, _securityType, _resolution)));
+            SortedDictionary<DateTime, string> rows;
+
+            if (File.Exists(outputFile))
+            {
+                // If file exists, we load existing data and perform merge
+                rows = LoadHourlyOrDailyFile(outputFile);
+                foreach (var kvp in newRows)
+                {
+                    rows[kvp.Key] = kvp.Value;
+                }
+            }
+            else
+            {
+                // No existing file, just use the new data
+                rows = newRows;
+            }
+
+            // Loop through the SortedDictionary and write to file contents
+            foreach (var kvp in rows)
+            {
+                // Build the line and append it to the file
+                lines.Add(kvp.Value);
+            }
+
+            // Write the file contents
+            if (lines.Count > 0)
+            {
+                WriteFile(outputFile, lines, lastTime);
             }
         }
 
         /// <summary>
         /// Loads an existing hourly or daily Lean zip file into a SortedDictionary
         /// </summary>
-        protected virtual bool TryLoadFile(string fileName, string entryName, out SortedDictionary<DateTime, string> rows)
+        private static SortedDictionary<DateTime, string> LoadHourlyOrDailyFile(string fileName)
         {
-            rows = new SortedDictionary<DateTime, string>();
+            var rows = new SortedDictionary<DateTime, string>();
 
-            using (var stream = _dataCacheProvider.Fetch($"{fileName}#{entryName}"))
+            using (var zip = ZipFile.Read(fileName))
             {
-                if (stream == null)
+                using (var stream = new MemoryStream())
                 {
-                    return false;
-                }
+                    zip[0].Extract(stream);
+                    stream.Seek(0, SeekOrigin.Begin);
 
-                using (var reader = new StreamReader(stream))
-                {
-                    string line;
-                    while ((line = reader.ReadLine()) != null)
+                    using (var reader = new StreamReader(stream))
                     {
-                        var time = DateTime.ParseExact(line.AsSpan(0, DateFormat.TwelveCharacter.Length), DateFormat.TwelveCharacter, CultureInfo.InvariantCulture);
-                        rows[time] = line;
+                        string line;
+                        while ((line = reader.ReadLine()) != null)
+                        {
+                            var time = Parse.DateTimeExact(line.Substring(0, DateFormat.TwelveCharacter.Length), DateFormat.TwelveCharacter);
+                            rows[time] = line;
+                        }
                     }
                 }
-
-                return true;
             }
+
+            return rows;
         }
 
         /// <summary>
-        /// Write this file to disk with the given data.
+        /// Write this file to disk.
         /// </summary>
         /// <param name="filePath">The full path to the new file</param>
-        /// <param name="data">The data to write as a list of dates and strings</param>
+        /// <param name="data">The data to write as a string</param>
         /// <param name="date">The date the data represents</param>
-        /// <remarks>The reason we have the data as IEnumerable(DateTime, string) is to support
-        /// a generic write that works for all resolutions. In order to merge in hour/daily case I need the
-        /// date of the data to correctly merge the two. In order to support writing ticks I need to allow
-        /// two data points to have the same time. Thus I cannot use a single list of just strings nor
-        /// a sorted dictionary of DateTimes and strings. </remarks>
-        private void WriteFile(string filePath, IEnumerable<(DateTime, string)> data, DateTime date)
+        private void WriteFile(string filePath, IEnumerable<string> data, DateTime date)
         {
-            // Generate this csv entry name
-            var entryName = LeanData.GenerateZipEntryName(_symbol, date, _resolution, _tickType);
-            
-            // Check disk once for this file ahead of time, reuse where possible
-            var fileExists = File.Exists(filePath);
+            var tempFilePath = filePath + ".tmp";
 
-            // Handle merging of files
-            // Only merge on files with hour/daily resolution, that exist, and can be loaded
-            string finalData;
-            if (_resolution >= Resolution.Hour && fileExists && TryLoadFile(filePath, entryName, out var rows))
+            if (File.Exists(filePath) && !_appendToZips)
             {
-                // Preform merge on loaded rows
-                foreach (var (time, line) in data)
-                {
-                    rows[time] = line;
-                }
+                File.Delete(filePath);
+                Log.Trace("LeanDataWriter.Write(): Existing deleted: " + filePath);
+            }
 
-                // Final merged data product
-                finalData = string.Join("\n", rows.Values);
+            // Create the directory if it doesnt exist
+            Directory.CreateDirectory(Path.GetDirectoryName(filePath));
+
+            if (_appendToZips)
+            {
+                var entryName = LeanData.GenerateZipEntryName(_symbol, date, _resolution, _tickType);
+                Compression.ZipCreateAppendData(filePath, entryName, string.Join(Environment.NewLine, data), true);
+                Log.Trace("LeanDataWriter.Write(): Appended: " + filePath);
             }
             else
             {
-                // Otherwise just extract the data from the given list.
-                finalData = string.Join("\n", data.Select(x => x.Item2));
+                // Write out this data string to a zip file
+                Compression.ZipData(tempFilePath, LeanData.GenerateZipEntryName(_symbol, date, _resolution, _tickType), data);
+
+                // Move temp file to the final destination with the appropriate name
+                File.Move(tempFilePath, filePath);
+                Log.Trace("LeanDataWriter.Write(): Created: " + filePath);
             }
-
-            // If our file doesn't exist its possible the directory doesn't exist, make sure at least the directory exists
-            if (!fileExists)
-            {
-                Directory.CreateDirectory(Path.GetDirectoryName(filePath));
-            }
-
-            var bytes = Encoding.UTF8.GetBytes(finalData);
-            _dataCacheProvider.Store($"{filePath}#{entryName}", bytes);
-
-            Log.Debug($"LeanDataWriter.Write(): Appended: {filePath} @ {entryName}");
         }
 
         /// <summary>

Common/Data/Market/SymbolChangedEvent.cs

@@ -70,9 +70,6 @@ namespace QuantConnect.Data.Market
             return new SymbolChangedEvent(Symbol, Time, OldSymbol, NewSymbol);
         }
 
-        /// <summary>
-        /// Friendly string representation of this symbol changed event
-        /// </summary>
         public override string ToString()
         {
             return $"{Time} {OldSymbol}->{NewSymbol}";

Common/Data/SubscriptionDataConfig.cs

@@ -136,14 +136,10 @@ namespace QuantConnect.Data
             }
             set
             {
-                var oldMappedValue = MappedSymbol;
                 var oldSymbol = Symbol;
                 Symbol = Symbol.UpdateMappedSymbol(value, ContractDepthOffset);
 
-                if (MappedSymbol != oldMappedValue)
-                {
-                    NewSymbol?.Invoke(this, new NewSymbolEventArgs(Symbol, oldSymbol));
-                }
+                NewSymbol?.Invoke(this, new NewSymbolEventArgs(Symbol, oldSymbol));
             }
         }
 

Common/Data/SubscriptionDataConfigExtensions.cs

@@ -15,7 +15,6 @@
 
 using System.Collections.Generic;
 using System.Linq;
-using QuantConnect.Util;
 
 namespace QuantConnect.Data
 {
@@ -129,32 +128,6 @@ namespace QuantConnect.Data
         /// check and void code duplication and related issues</remarks>
         /// <returns>True if ticker prices should be scaled</returns>
         public static bool PricesShouldBeScaled(this SubscriptionDataConfig config)
-        {
-            if (config.IsCustomData || config.Symbol.Value.Contains("UNIVERSE"))
-            {
-                return false;
-            }
-
-            if(config.SecurityType == SecurityType.Equity)
-            {
-                return true;
-            }
-            if (config.SecurityType == SecurityType.Future && config.Symbol.IsCanonical())
-            {
-                return LeanData.IsCommonLeanDataType(config.Type);
-            }
-
-            return false;
-        }
-
-        /// <summary>
-        /// Will determine if splits and dividends should be used for this subscription configuration
-        /// </summary>
-        /// <param name="config">The subscription data configuration we are processing</param>
-        /// <remarks>Different than <see cref="PricesShouldBeScaled"/> because prices could be scale and no split and dividends
-        /// really exist, like in the continuous futures case</remarks>
-        /// <returns>True if this configuration requires split and divided handling</returns>
-        public static bool EmitSplitsAndDividends(this SubscriptionDataConfig config)
         {
             return !config.IsCustomData && !config.Symbol.Value.Contains("UNIVERSE") && config.SecurityType == SecurityType.Equity;
         }

Common/Data/UniverseSelection/Universe.cs

@@ -252,8 +252,7 @@ namespace QuantConnect.Data.UniverseSelection
                 UniverseSettings.ExtendedMarketHours,
                 dataNormalizationMode: UniverseSettings.DataNormalizationMode,
                 subscriptionDataTypes: UniverseSettings.SubscriptionDataTypes,
-                dataMappingMode: UniverseSettings.DataMappingMode,
-                contractDepthOffset: (uint)Math.Abs(UniverseSettings.ContractDepthOffset));
+                dataMappingMode: UniverseSettings.DataMappingMode);
             return result.Select(config => new SubscriptionRequest(isUniverseSubscription: false,
                 universe: this,
                 security: security,