Refactor MarginRequirementEntry. Update margins (#6210)

* Fix master

* Refactor MarginRequirementEntry to it's own file

* Update margin files

* Remove duplicate dates cases

* Update margins after fix

.github/workflows/gh-actions.yml

@@ -28,3 +28,5 @@ jobs:
           ./ci_build_stubs.sh -t -g -p
         env:
           PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
+          ADDITIONAL_STUBS_REPOS: ${{ secrets.ADDITIONAL_STUBS_REPOS }}
+          QC_GIT_TOKEN: ${{ secrets.QC_GIT_TOKEN }}

.vscode/launch.json

@@ -8,8 +8,8 @@
         marketplace.
 
         Attach to Python:
-        Will attempt to attach to LEAN running locally using PTVSD. Requires that the process is
-        actively running and config is set: "debugging": true, "debugging-method": "PTVSD",
+        Will attempt to attach to LEAN running locally using DebugPy. Requires that the process is
+        actively running and config is set: "debugging": true, "debugging-method": "DebugPy",
         Requires Python extension from the marketplace. Currently only works with algorithms in
         Algorithm.Python directory. This is because we map that directory to our build directory
         that contains the py file at runtime. If using another location change "localRoot" value 

.vscode/readme.md

@@ -95,7 +95,7 @@ Python algorithms require a little extra work in order to be able to debug them.
 First in order to debug a Python algorithm in VS Code we must make the following change to our configuration (Launcher\config.json) under the comment debugging configuration:
 
     "debugging": true,
-    "debugging-method": "PTVSD",
+    "debugging-method": "DebugPy,
 
 In setting this we are telling Lean to expect a debugger connection using ‘Python Tools for Visual Studio Debugger’. Once this is set Lean will stop upon initialization and await a connection to the debugger via port 5678.
 

Algorithm.CSharp/AddRemoveOptionUniverseRegressionAlgorithm.cs

@@ -40,8 +40,8 @@ namespace QuantConnect.Algorithm.CSharp
         private int _expectedContractIndex;
         private readonly List<Symbol> _expectedContracts = new List<Symbol>
         {
-            SymbolRepresentation.ParseOptionTickerOSI("GOOG  151224P00750000"),
             SymbolRepresentation.ParseOptionTickerOSI("GOOG  151224P00747500"),
+            SymbolRepresentation.ParseOptionTickerOSI("GOOG  151224P00750000"),
             SymbolRepresentation.ParseOptionTickerOSI("GOOG  151224P00752500")
         };
 
@@ -109,6 +109,11 @@ namespace QuantConnect.Algorithm.CSharp
                 var googOptionChain = AddOption(UnderlyingTicker);
                 googOptionChain.SetFilter(u =>
                 {
+                    // we added the universe at 10, the universe selection data should not be from before
+                    if (u.Underlying.EndTime.Hour < 10)
+                    {
+                        throw new Exception($"Unexpected underlying data point {u.Underlying.EndTime} {u.Underlying}");
+                    }
                     // find first put above market price
                     return u.IncludeWeeklys()
                         .Strikes(+1, +1)
@@ -231,7 +236,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Treynor Ratio", "0"},
             {"Total Fees", "$6.00"},
             {"Estimated Strategy Capacity", "$2000.00"},
-            {"Lowest Capacity Asset", "GOOCV 305RBQ2BZBZT2|GOOCV VP83T1ZUHROL"},
+            {"Lowest Capacity Asset", "GOOCV 305RBR0BSWIX2|GOOCV VP83T1ZUHROL"},
             {"Fitness Score", "0"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
@@ -251,7 +256,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "1e7b3e90918777b9dbf46353a96f3329"}
+            {"OrderListHash", "550a99c482106defd8ba15f48183768e"}
         };
     }
 }

Algorithm.CSharp/AddTwoAndRemoveOneOptionContractRegressionAlgorithm.cs

@@ -0,0 +1,150 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Regression algorithm reproducing issue where underlying option contract would be removed with the first call
+    /// too RemoveOptionContract
+    /// </summary>
+    public class AddTwoAndRemoveOneOptionContractRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        private Symbol _contract1;
+        private Symbol _contract2;
+        private bool _hasRemoved;
+
+        public override void Initialize()
+        {
+            SetStartDate(2014, 06, 06);
+            SetEndDate(2014, 06, 06);
+
+            UniverseSettings.DataNormalizationMode = DataNormalizationMode.Raw;
+            UniverseSettings.MinimumTimeInUniverse = TimeSpan.Zero;
+
+            var aapl = QuantConnect.Symbol.Create("AAPL", SecurityType.Equity, Market.USA);
+
+            var contracts = OptionChainProvider.GetOptionContractList(aapl, Time)
+                .OrderBy(symbol => symbol.ID.Symbol)
+                .Where(optionContract => optionContract.ID.OptionRight == OptionRight.Call
+                    && optionContract.ID.OptionStyle == OptionStyle.American)
+                .Take(2)
+                .ToList();
+
+            _contract1 = contracts[0];
+            _contract2 = contracts[1];
+            AddOptionContract(_contract1);
+            AddOptionContract(_contract2);
+        }
+
+        public override void OnData(Slice slice)
+        {
+            if (slice.HasData)
+            {
+                if (!_hasRemoved)
+                {
+                    RemoveOptionContract(_contract1);
+                    _hasRemoved = true;
+                }
+                else
+                {
+                    var subscriptions =
+                        SubscriptionManager.SubscriptionDataConfigService.GetSubscriptionDataConfigs("AAPL");
+                    if (subscriptions.Count == 0)
+                    {
+                        throw new Exception("No configuration for underlying was found!");
+                    }
+
+                    if (!Portfolio.Invested)
+                    {
+                        Buy(_contract2, 1);
+                    }
+                }
+            }
+        }
+
+        public override void OnEndOfAlgorithm()
+        {
+            if (!_hasRemoved)
+            {
+                throw new Exception("Expect a single call to OnData where we removed the option and underlying");
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public Language[] Languages { get; } = { Language.CSharp };
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Trades", "2"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "0%"},
+            {"Drawdown", "0%"},
+            {"Expectancy", "0"},
+            {"Net Profit", "0%"},
+            {"Sharpe Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "0%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "0"},
+            {"Tracking Error", "0"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "$2.00"},
+            {"Estimated Strategy Capacity", "$230000.00"},
+            {"Lowest Capacity Asset", "AAPL VXBK4QQIRLZA|AAPL R735QTJ8XC9X"},
+            {"Fitness Score", "0"},
+            {"Kelly Criterion Estimate", "0"},
+            {"Kelly Criterion Probability Value", "0"},
+            {"Sortino Ratio", "0"},
+            {"Return Over Maximum Drawdown", "0"},
+            {"Portfolio Turnover", "0"},
+            {"Total Insights Generated", "0"},
+            {"Total Insights Closed", "0"},
+            {"Total Insights Analysis Completed", "0"},
+            {"Long Insight Count", "0"},
+            {"Short Insight Count", "0"},
+            {"Long/Short Ratio", "100%"},
+            {"Estimated Monthly Alpha Value", "$0"},
+            {"Total Accumulated Estimated Alpha Value", "$0"},
+            {"Mean Population Estimated Insight Value", "$0"},
+            {"Mean Population Direction", "0%"},
+            {"Mean Population Magnitude", "0%"},
+            {"Rolling Averaged Population Direction", "0%"},
+            {"Rolling Averaged Population Magnitude", "0%"},
+            {"OrderListHash", "228194dcc6fd8689a67f383577ee2d85"}
+        };
+    }
+}

Algorithm.CSharp/Alphas/MortgageRateVolatilityAlpha.cs

@@ -1,162 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-using System;
-using System.Linq;
-using QuantConnect.Data;
-using QuantConnect.Indicators;
-using QuantConnect.Orders.Fees;
-using QuantConnect.Data.Custom;
-using System.Collections.Generic;
-using QuantConnect.Algorithm.Framework.Alphas;
-using QuantConnect.Algorithm.Framework.Execution;
-using QuantConnect.Algorithm.Framework.Portfolio;
-using QuantConnect.Algorithm.Framework.Risk;
-using QuantConnect.Algorithm.Framework.Selection;
-
-namespace QuantConnect.Algorithm.CSharp.Alphas
-{
-    ///<summary>
-    /// This Alpha Model uses Wells Fargo 30-year Fixed Rate Mortgage data from Quandl to
-    /// generate Insights about the movement of Real Estate ETFs. Mortgage rates can provide information
-    /// regarding the general price trend of real estate, and ETFs provide good continuous-time instruments
-    /// to measure the impact against. Volatility in mortgage rates tends to put downward pressure on real
-    /// estate prices, whereas stable mortgage rates, regardless of true rate, lead to stable or higher real
-    /// estate prices. This Alpha model seeks to take advantage of this correlation by emitting insights
-    /// based on volatility and rate deviation from its historic mean.
-    ///
-    /// This alpha is part of the Benchmark Alpha Series created by QuantConnect which are open
-    /// sourced so the community and client funds can see an example of an alpha.
-    ///</summary>
-    public class MortgageRateVolatilityAlgorithm : QCAlgorithm
-    {
-        public override void Initialize()
-        {
-            SetStartDate(2017, 1, 1);  //Set Start Date
-            SetCash(100000);             //Set Strategy Cash
-
-            UniverseSettings.Resolution = Resolution.Daily;
-            SetSecurityInitializer(security => security.FeeModel = new ConstantFeeModel(0));
-
-            // Basket of 6 liquid real estate ETFs
-            Func<string, Symbol> toSymbol = x => QuantConnect.Symbol.Create(x, SecurityType.Equity, Market.USA);
-            var realEstateETFs = new[] { "VNQ", "REET", "TAO", "FREL", "SRET", "HIPS" }.Select(toSymbol).ToArray();
-            SetUniverseSelection(new ManualUniverseSelectionModel(realEstateETFs));
-
-            SetAlpha(new MortgageRateVolatilityAlphaModel(this));
-
-            SetPortfolioConstruction(new EqualWeightingPortfolioConstructionModel());
-
-            SetExecution(new ImmediateExecutionModel());
-
-            SetRiskManagement(new NullRiskManagementModel());
-
-        }
-
-        private class MortgageRateVolatilityAlphaModel : AlphaModel
-        {
-            private readonly int _indicatorPeriod;
-            private readonly Resolution _resolution;
-            private readonly TimeSpan _insightDuration;
-            private readonly int _deviations;
-            private readonly double _insightMagnitude;
-            private readonly Symbol _mortgageRate;
-            private readonly SimpleMovingAverage _mortgageRateSma;
-            private readonly StandardDeviation _mortgageRateStd;
-
-            public MortgageRateVolatilityAlphaModel(
-                QCAlgorithm algorithm,
-                int indicatorPeriod = 15,
-                double insightMagnitude = 0.0005,
-                int deviations = 2,
-                Resolution resolution = Resolution.Daily
-                )
-            {
-            	// Add Quandl data for a Well's Fargo 30-year Fixed Rate mortgage
-                _mortgageRate = algorithm.AddData<QuandlMortgagePriceColumns>("WFC/PR_GOV_30YFIXEDVA_APR").Symbol;
-                _indicatorPeriod = indicatorPeriod;
-                _resolution = resolution;
-                _insightDuration = resolution.ToTimeSpan().Multiply(indicatorPeriod);
-                _insightMagnitude = insightMagnitude;
-                _deviations = deviations;
-                
-                // Add indicators for the mortgage rate -- Standard Deviation and Simple Moving Average
-                _mortgageRateStd = algorithm.STD(_mortgageRate, _indicatorPeriod, resolution);
-                _mortgageRateSma = algorithm.SMA(_mortgageRate, _indicatorPeriod, resolution);
-                
-                // Use a history call to warm-up the indicators
-                WarmUpIndicators(algorithm);
-            }
-            
-            public override IEnumerable<Insight> Update(QCAlgorithm algorithm, Slice data)
-            {
-                var insights = new List<Insight>();
-                
-                // Return empty list if data slice doesn't contain monrtgage rate data
-                if (!data.Keys.Contains(_mortgageRate))
-                {
-                    return insights;
-                }
-				// Extract current mortgage rate, the current STD indicator value, and current SMA value
-                var rate = data[_mortgageRate].Value;
-                var deviation = _deviations * _mortgageRateStd;
-                var sma = _mortgageRateSma;
-
-				// Loop through all Active Securities to emit insights
-                foreach (var security in algorithm.ActiveSecurities.Keys)
-                {
-                	// Mortgage rate Symbol will be in the collection, so skip it
-                    if (security == _mortgageRate)
-                    {
-                        return insights;
-                    }
-
-					// If volatility in mortgage rates is high, then we emit an Insight to sell
-                    if ((rate < sma - deviation) || (rate > sma + deviation))
-                    {
-                        insights.Add(Insight.Price(security, _insightDuration, InsightDirection.Down, _insightMagnitude));
-                    }
-                    
-                    // If volatility in mortgage rates is low, then we emit an Insight to buy
-                    if ((rate < sma - (decimal)deviation/2) || (rate > sma + (decimal)deviation/2))
-                    {
-                        insights.Add(Insight.Price(security, _insightDuration, InsightDirection.Up, _insightMagnitude));
-                    }
-                }
-
-                return insights;
-            }
-
-            private void WarmUpIndicators(QCAlgorithm algorithm)
-            {
-            	// Make a history call and update the indicators
-                algorithm.History(new[] { _mortgageRate }, _indicatorPeriod, _resolution).PushThrough(bar =>
-                {
-                    _mortgageRateSma.Update(bar.EndTime, bar.Value);
-                    _mortgageRateStd.Update(bar.EndTime, bar.Value);
-                });
-            }
-        }
-        public class QuandlMortgagePriceColumns : Quandl
-        {
-            public QuandlMortgagePriceColumns()
-            
-            // Rename the Quandl object column to the data we want, which is the 'Value' column
-	        // of the CSV that our API call returns
-                : base(valueColumnName: "Value")
-            {
-            }
-        }
-    }
-}

Algorithm.CSharp/BasicTemplateContinuousFutureAlgorithm.cs

@@ -13,6 +13,7 @@
  * limitations under the License.
 */
 
+using System;
 using QuantConnect.Data;
 using QuantConnect.Orders;
 using QuantConnect.Interfaces;
@@ -62,7 +63,11 @@ namespace QuantConnect.Algorithm.CSharp
         {
             foreach (var changedEvent in data.SymbolChangedEvents.Values)
             {
-                Log($"{Time} - SymbolChanged event: {changedEvent}");
+                Debug($"{Time} - SymbolChanged event: {changedEvent}");
+                if (Time.TimeOfDay != TimeSpan.Zero)
+                {
+                    throw new Exception($"{Time} unexpected symbol changed event {changedEvent}!");
+                }
             }
 
             if (!Portfolio.Invested)

Algorithm.CSharp/BasicTemplateFuturesAlgorithm.cs

@@ -18,6 +18,7 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using QuantConnect.Data;
+using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Interfaces;
 using QuantConnect.Securities;
 using QuantConnect.Securities.Future;
@@ -64,6 +65,9 @@ namespace QuantConnect.Algorithm.CSharp
 
             var benchmark = AddEquity("SPY");
             SetBenchmark(benchmark.Symbol);
+
+            var seeder = new FuncSecuritySeeder(GetLastKnownPrices);
+            SetSecurityInitializer(security => seeder.SeedSecurity(security));
         }
 
         /// <summary>
@@ -72,6 +76,15 @@ namespace QuantConnect.Algorithm.CSharp
         /// <param name="slice">The current slice of data keyed by symbol string</param>
         public override void OnData(Slice slice)
         {
+            foreach (var changedEvent in slice.SymbolChangedEvents.Values)
+            {
+                Debug($"{Time} - SymbolChanged event: {changedEvent}");
+                if (Time.TimeOfDay != TimeSpan.Zero)
+                {
+                    throw new Exception($"{Time} unexpected symbol changed event {changedEvent}!");
+                }
+            }
+
             if (!Portfolio.Invested)
             {
                 foreach(var chain in slice.FutureChains)
@@ -112,6 +125,19 @@ namespace QuantConnect.Algorithm.CSharp
             var maintenanceIntraday = futureMarginModel.MaintenanceIntradayMarginRequirement;
         }
 
+        public override void OnSecuritiesChanged(SecurityChanges changes)
+        {
+            foreach (var addedSecurity in changes.AddedSecurities)
+            {
+                if (addedSecurity.Symbol.SecurityType == SecurityType.Future
+                    && !addedSecurity.Symbol.IsCanonical()
+                    && !addedSecurity.HasData)
+                {
+                    throw new Exception($"Future contracts did not work up as expected: {addedSecurity.Symbol}");
+                }
+            }
+        }
+
         /// <summary>
         /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
         /// </summary>

Algorithm.CSharp/BasicTemplateFuturesDailyAlgorithm.cs

@@ -46,7 +46,7 @@ namespace QuantConnect.Algorithm.CSharp
         public override void Initialize()
         {
             SetStartDate(2013, 10, 08);
-            SetEndDate(2013, 10, 10);
+            SetEndDate(2014, 10, 10);
             SetCash(1000000);
 
             var futureSP500 = AddFuture(RootSP500, Resolution);
@@ -77,7 +77,7 @@ namespace QuantConnect.Algorithm.CSharp
                     ).FirstOrDefault();
 
                     // if found, trade it
-                    if (contract != null)
+                    if (contract != null && IsMarketOpen(contract.Symbol))
                     {
                         _contractSymbol = contract.Symbol;
                         MarketOrder(_contractSymbol, 1);
@@ -88,6 +88,14 @@ namespace QuantConnect.Algorithm.CSharp
             {
                 Liquidate();
             }
+
+            foreach (var changedEvent in slice.SymbolChangedEvents.Values)
+            {
+                if (Time.TimeOfDay != TimeSpan.Zero)
+                {
+                    throw new Exception($"{Time} unexpected symbol changed event {changedEvent}!");
+                }
+            }
         }
 
         /// <summary>
@@ -105,34 +113,34 @@ namespace QuantConnect.Algorithm.CSharp
         /// </summary>
         public virtual Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
-            {"Total Trades", "6"},
-            {"Average Win", "0%"},
-            {"Average Loss", "-0.10%"},
-            {"Compounding Annual Return", "-23.119%"},
-            {"Drawdown", "0.300%"},
-            {"Expectancy", "-1"},
-            {"Net Profit", "-0.276%"},
-            {"Sharpe Ratio", "-13.736"},
+            {"Total Trades", "92"},
+            {"Average Win", "0.08%"},
+            {"Average Loss", "-0.01%"},
+            {"Compounding Annual Return", "-0.450%"},
+            {"Drawdown", "0.500%"},
+            {"Expectancy", "-0.824"},
+            {"Net Profit", "-0.453%"},
+            {"Sharpe Ratio", "-1.803"},
             {"Probabilistic Sharpe Ratio", "0%"},
-            {"Loss Rate", "100%"},
-            {"Win Rate", "0%"},
-            {"Profit-Loss Ratio", "0"},
-            {"Alpha", "-0.526"},
-            {"Beta", "0.057"},
-            {"Annual Standard Deviation", "0.015"},
+            {"Loss Rate", "98%"},
+            {"Win Rate", "2%"},
+            {"Profit-Loss Ratio", "7.09"},
+            {"Alpha", "-0.003"},
+            {"Beta", "-0.001"},
+            {"Annual Standard Deviation", "0.002"},
             {"Annual Variance", "0"},
-            {"Information Ratio", "-31.088"},
-            {"Tracking Error", "0.189"},
-            {"Treynor Ratio", "-3.51"},
-            {"Total Fees", "$11.10"},
-            {"Estimated Strategy Capacity", "$200000000.00"},
-            {"Lowest Capacity Asset", "GC VOFJUCDY9XNH"},
-            {"Fitness Score", "0"},
+            {"Information Ratio", "-1.394"},
+            {"Tracking Error", "0.089"},
+            {"Treynor Ratio", "4.298"},
+            {"Total Fees", "$170.20"},
+            {"Estimated Strategy Capacity", "$36000.00"},
+            {"Lowest Capacity Asset", "ES VP274HSU1AF5"},
+            {"Fitness Score", "0.009"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "-17.118"},
-            {"Return Over Maximum Drawdown", "-83.844"},
-            {"Portfolio Turnover", "0.16"},
+            {"Sortino Ratio", "-0.8"},
+            {"Return Over Maximum Drawdown", "-0.992"},
+            {"Portfolio Turnover", "0.025"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
             {"Total Insights Analysis Completed", "0"},
@@ -146,7 +154,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "512f55519e5221c7e82e1d9f5ddd1b9f"}
+            {"OrderListHash", "09b2f274fa2385597a803e58b784f675"}
         };
     }
 }

Algorithm.CSharp/BasicTemplateFuturesHourlyAlgorithm.cs

@@ -49,34 +49,34 @@ namespace QuantConnect.Algorithm.CSharp
         /// </summary>
         public override Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
-            {"Total Trades", "140"},
+            {"Total Trades", "1988"},
             {"Average Win", "0.01%"},
-            {"Average Loss", "-0.02%"},
-            {"Compounding Annual Return", "-38.171%"},
-            {"Drawdown", "0.400%"},
-            {"Expectancy", "-0.369"},
-            {"Net Profit", "-0.394%"},
-            {"Sharpe Ratio", "-24.82"},
+            {"Average Loss", "0.00%"},
+            {"Compounding Annual Return", "-4.120%"},
+            {"Drawdown", "4.200%"},
+            {"Expectancy", "-0.870"},
+            {"Net Profit", "-4.150%"},
+            {"Sharpe Ratio", "-6.061"},
             {"Probabilistic Sharpe Ratio", "0%"},
-            {"Loss Rate", "66%"},
-            {"Win Rate", "34%"},
-            {"Profit-Loss Ratio", "0.84"},
-            {"Alpha", "0.42"},
-            {"Beta", "-0.041"},
-            {"Annual Standard Deviation", "0.01"},
+            {"Loss Rate", "97%"},
+            {"Win Rate", "3%"},
+            {"Profit-Loss Ratio", "2.92"},
+            {"Alpha", "-0.027"},
+            {"Beta", "-0.006"},
+            {"Annual Standard Deviation", "0.005"},
             {"Annual Variance", "0"},
-            {"Information Ratio", "-65.112"},
-            {"Tracking Error", "0.253"},
-            {"Treynor Ratio", "6.024"},
-            {"Total Fees", "$259.00"},
-            {"Estimated Strategy Capacity", "$130000.00"},
-            {"Lowest Capacity Asset", "GC VOFJUCDY9XNH"},
-            {"Fitness Score", "0"},
+            {"Information Ratio", "-1.66"},
+            {"Tracking Error", "0.089"},
+            {"Treynor Ratio", "4.919"},
+            {"Total Fees", "$3677.80"},
+            {"Estimated Strategy Capacity", "$2000.00"},
+            {"Lowest Capacity Asset", "ES VP274HSU1AF5"},
+            {"Fitness Score", "0.128"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "-43.422"},
-            {"Return Over Maximum Drawdown", "-100.459"},
-            {"Portfolio Turnover", "4.716"},
+            {"Sortino Ratio", "-6.856"},
+            {"Return Over Maximum Drawdown", "-0.995"},
+            {"Portfolio Turnover", "0.648"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
             {"Total Insights Analysis Completed", "0"},
@@ -90,7 +90,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "320067074c8dd771f69602ab07001f1e"}
+            {"OrderListHash", "87d2b127c9859cad9d2c65ac9d76deb5"}
         };
     }
 }

Algorithm.CSharp/BasicTemplateIndiaAlgorithm.cs

@@ -14,8 +14,9 @@
 */
 
 using QuantConnect.Data;
-using QuantConnect.Orders;
 using System.Collections.Generic;
+using QuantConnect.Interfaces;
+using QuantConnect.Orders;
 
 namespace QuantConnect.Algorithm.CSharp
 {
@@ -26,20 +27,21 @@ namespace QuantConnect.Algorithm.CSharp
     /// <meta name="tag" content="using data" />
     /// <meta name="tag" content="using quantconnect" />
     /// <meta name="tag" content="trading and orders" />
-    public class BasicTemplateIndiaAlgorithm : QCAlgorithm
+    public class BasicTemplateIndiaAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
         /// <summary>
         /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
         /// </summary>
         public override void Initialize()
         {
-            SetStartDate(2003, 10, 07);  //Set Start Date
-            SetEndDate(2003, 10, 11);    //Set End Date
-            SetCash(100000);             //Set Strategy Cash
+            SetAccountCurrency("INR");  //Set Account Currency
+            SetStartDate(2019, 1, 23);  //Set Start Date
+            SetEndDate(2019, 10, 31);   //Set End Date
+            SetCash(100000);            //Set Strategy Cash
 
             // Find more symbols here: http://quantconnect.com/data
             // Equities Resolutions: Tick, Second, Minute, Hour, Daily.
-            AddEquity("UNIONBANK", Resolution.Second, Market.India);
+            AddEquity("YESBANK", Resolution.Minute, Market.India);
 
             //Set Order Prperties as per the requirements for order placement
             DefaultOrderProperties = new IndiaOrderProperties(exchange: Exchange.NSE);
@@ -58,7 +60,7 @@ namespace QuantConnect.Algorithm.CSharp
         {
             if (!Portfolio.Invested)
             {
-                var marketTicket = MarketOrder("UNIONBANK", 1);
+                var marketTicket = MarketOrder("YESBANK", 1);
             }
         }
 
@@ -73,60 +75,60 @@ namespace QuantConnect.Algorithm.CSharp
         /// <summary>
         /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
         /// </summary>
-        public bool CanRunLocally { get; } = false;
+        public bool CanRunLocally { get; } = true;
 
         /// <summary>
         /// This is used by the regression test system to indicate which languages this algorithm is written in.
         /// </summary>
-        public Language[] Languages { get; } = { Language.CSharp };
+        public Language[] Languages { get; } = { Language.CSharp, Language.Python };
 
         /// <summary>
         /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
         /// </summary>
         public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
-            {"Total Trades", "3"},
+            {"Total Trades", "1"},
             {"Average Win", "0%"},
-            {"Average Loss", "-1.01%"},
-            {"Compounding Annual Return", "261.134%"},
-            {"Drawdown", "2.200%"},
-            {"Expectancy", "-1"},
-            {"Net Profit", "1.655%"},
-            {"Sharpe Ratio", "8.505"},
-            {"Probabilistic Sharpe Ratio", "66.840%"},
-            {"Loss Rate", "100%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "-0.010%"},
+            {"Drawdown", "0.000%"},
+            {"Expectancy", "0"},
+            {"Net Profit", "-0.008%"},
+            {"Sharpe Ratio", "-1.183"},
+            {"Probabilistic Sharpe Ratio", "0.001%"},
+            {"Loss Rate", "0%"},
             {"Win Rate", "0%"},
             {"Profit-Loss Ratio", "0"},
-            {"Alpha", "-0.091"},
-            {"Beta", "1.006"},
-            {"Annual Standard Deviation", "0.224"},
-            {"Annual Variance", "0.05"},
-            {"Information Ratio", "-33.445"},
-            {"Tracking Error", "0.002"},
-            {"Treynor Ratio", "1.893"},
-            {"Total Fees", "$10.32"},
-            {"Estimated Strategy Capacity", "$27000000.00"},
-            {"Lowest Capacity Asset", "SPY R735QTJ8XC9X"},
-            {"Fitness Score", "0.747"},
-            {"Kelly Criterion Estimate", "38.796"},
-            {"Kelly Criterion Probability Value", "0.228"},
-            {"Sortino Ratio", "79228162514264337593543950335"},
-            {"Return Over Maximum Drawdown", "85.095"},
-            {"Portfolio Turnover", "0.747"},
-            {"Total Insights Generated", "100"},
-            {"Total Insights Closed", "99"},
-            {"Total Insights Analysis Completed", "99"},
-            {"Long Insight Count", "100"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "-1.183"},
+            {"Tracking Error", "0"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "$6.00"},
+            {"Estimated Strategy Capacity", "$61000000000.00"},
+            {"Lowest Capacity Asset", "YESBANK UL"},
+            {"Fitness Score", "0"},
+            {"Kelly Criterion Estimate", "0"},
+            {"Kelly Criterion Probability Value", "0"},
+            {"Sortino Ratio", "-0.247"},
+            {"Return Over Maximum Drawdown", "-1.104"},
+            {"Portfolio Turnover", "0"},
+            {"Total Insights Generated", "0"},
+            {"Total Insights Closed", "0"},
+            {"Total Insights Analysis Completed", "0"},
+            {"Long Insight Count", "0"},
             {"Short Insight Count", "0"},
             {"Long/Short Ratio", "100%"},
-            {"Estimated Monthly Alpha Value", "$135639.1761"},
-            {"Total Accumulated Estimated Alpha Value", "$21852.9784"},
-            {"Mean Population Estimated Insight Value", "$220.7372"},
-            {"Mean Population Direction", "53.5354%"},
-            {"Mean Population Magnitude", "53.5354%"},
-            {"Rolling Averaged Population Direction", "58.2788%"},
-            {"Rolling Averaged Population Magnitude", "58.2788%"},
-            {"OrderListHash", "ad2216297c759d8e5aef48ff065f8919"}
+            {"Estimated Monthly Alpha Value", "₹0"},
+            {"Total Accumulated Estimated Alpha Value", "₹0"},
+            {"Mean Population Estimated Insight Value", "₹0"},
+            {"Mean Population Direction", "0%"},
+            {"Mean Population Magnitude", "0%"},
+            {"Rolling Averaged Population Direction", "0%"},
+            {"Rolling Averaged Population Magnitude", "0%"},
+            {"OrderListHash", "6cc69218edd7bd461678b9ee0c575db5"}
         };
     }
 }

Algorithm.CSharp/BasicTemplateIndiaIndexAlgorithm.cs

@@ -0,0 +1,158 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data;
+using System.Collections.Generic;
+using QuantConnect.Indicators;
+using QuantConnect.Interfaces;
+using QuantConnect.Orders;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// This example demonstrates how to add index asset types.
+    /// </summary>
+    /// <meta name="tag" content="using data" />
+    /// <meta name="tag" content="benchmarks" />
+    /// <meta name="tag" content="indexes" />
+    public class BasicTemplateIndiaIndexAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        protected Symbol Nifty;
+        protected Symbol NiftyETF;
+        private ExponentialMovingAverage _emaSlow;
+        private ExponentialMovingAverage _emaFast;
+
+        /// <summary>
+        /// Initialize your algorithm and add desired assets.
+        /// </summary>
+        public override void Initialize()
+        {
+            SetAccountCurrency("INR"); //Set Account Currency
+            SetStartDate(2019, 1, 1);  //Set End Date
+            SetEndDate(2019, 1, 5);    //Set End Date
+            SetCash(1000000);          //Set Strategy Cash
+
+            // Use indicator for signal; but it cannot be traded
+            Nifty = AddIndex("NIFTY50", Resolution.Minute, Market.India).Symbol;
+
+            //Trade Index based ETF
+            NiftyETF = AddEquity("JUNIORBEES", Resolution.Minute, Market.India).Symbol;
+
+            //Set Order Prperties as per the requirements for order placement
+            DefaultOrderProperties = new IndiaOrderProperties(exchange: Exchange.NSE);
+
+            _emaSlow = EMA(Nifty, 80);
+            _emaFast = EMA(Nifty, 200);
+        }
+
+        /// <summary>
+        /// Index EMA Cross trading underlying.
+        /// </summary>
+        public override void OnData(Slice slice)
+        {
+            if (!slice.Bars.ContainsKey(Nifty) || !slice.Bars.ContainsKey(NiftyETF))
+            {
+                return;
+            }
+
+            // Warm up indicators
+            if (!_emaSlow.IsReady)
+            {
+                return;
+            }
+
+            if (_emaFast > _emaSlow)
+            {
+                if (!Portfolio.Invested)
+                {
+                    var marketTicket = MarketOrder(NiftyETF, 1);
+                }
+            }
+            else
+            {
+                Liquidate();
+            }
+        }
+
+        public override void OnEndOfAlgorithm()
+        {
+            if (Portfolio[Nifty].TotalSaleVolume > 0)
+            {
+                throw new Exception("Index is not tradable.");
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public virtual bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public virtual Language[] Languages { get; } = { Language.CSharp, Language.Python };
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public virtual Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Trades", "6"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0.00%"},
+            {"Compounding Annual Return", "-0.395%"},
+            {"Drawdown", "0.000%"},
+            {"Expectancy", "-1"},
+            {"Net Profit", "-0.004%"},
+            {"Sharpe Ratio", "-23.595"},
+            {"Probabilistic Sharpe Ratio", "0%"},
+            {"Loss Rate", "100%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "-23.595"},
+            {"Tracking Error", "0"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "$36.00"},
+            {"Estimated Strategy Capacity", "$74000.00"},
+            {"Lowest Capacity Asset", "JUNIORBEES UL"},
+            {"Fitness Score", "0"},
+            {"Kelly Criterion Estimate", "0"},
+            {"Kelly Criterion Probability Value", "0"},
+            {"Sortino Ratio", "-29.6"},
+            {"Return Over Maximum Drawdown", "-123.624"},
+            {"Portfolio Turnover", "0"},
+            {"Total Insights Generated", "0"},
+            {"Total Insights Closed", "0"},
+            {"Total Insights Analysis Completed", "0"},
+            {"Long Insight Count", "0"},
+            {"Short Insight Count", "0"},
+            {"Long/Short Ratio", "100%"},
+            {"Estimated Monthly Alpha Value", "₹0"},
+            {"Total Accumulated Estimated Alpha Value", "₹0"},
+            {"Mean Population Estimated Insight Value", "₹0"},
+            {"Mean Population Direction", "0%"},
+            {"Mean Population Magnitude", "0%"},
+            {"Rolling Averaged Population Direction", "0%"},
+            {"Rolling Averaged Population Magnitude", "0%"},
+            {"OrderListHash", "4637f26543287548b28a3c296db055d3"}
+        };
+    }
+}

Algorithm.CSharp/BasicTemplateOptionsDailyAlgorithm.cs

@@ -69,7 +69,7 @@ namespace QuantConnect.Algorithm.CSharp
                     var contractsByExpiration = chain.Where(x => x.Expiry != Time.Date).OrderBy(x => x.Expiry);
                     var contract = contractsByExpiration.FirstOrDefault();
 
-                    if (contract != null)
+                    if (contract != null && IsMarketOpen(contract.Symbol))
                     {
                         // if found, trade it
                         MarketOrder(contract.Symbol, 1);

Algorithm.CSharp/BasicTemplateOptionsHourlyAlgorithm.cs

@@ -76,7 +76,7 @@ namespace QuantConnect.Algorithm.CSharp
                         .ThenByDescending(x => x.Right)
                         .FirstOrDefault();
 
-                    if (atmContract != null)
+                    if (atmContract != null && IsMarketOpen(atmContract.Symbol))
                     {
                         // if found, trade it
                         MarketOrder(atmContract.Symbol, 1);

Algorithm.CSharp/BinanceCashAccountFeeRegressionAlgorithm.cs

@@ -0,0 +1,85 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using QuantConnect.Brokerages;
+using System.Collections.Generic;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Binance cash account regression algorithm, reproduces issue https://github.com/QuantConnect/Lean/issues/6123
+    /// </summary>
+    public class BinanceCashAccountFeeRegressionAlgorithm : CryptoCashAccountFeeRegressionAlgorithm
+    {
+        public override void Initialize()
+        {
+            SetAccountCurrency("USDT");
+            SetStartDate(2018, 05, 02);
+            SetEndDate(2018, 05, 03);
+            BrokerageName = BrokerageName.Binance;
+            Pair = "BTCUSDT";
+            base.Initialize();
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public override Dictionary<string, string> ExpectedStatistics => new()
+        {
+            {"Total Trades", "49"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "0%"},
+            {"Drawdown", "0%"},
+            {"Expectancy", "0"},
+            {"Net Profit", "0%"},
+            {"Sharpe Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "0%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "0"},
+            {"Tracking Error", "0"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "$45.62"},
+            {"Estimated Strategy Capacity", "$220000.00"},
+            {"Lowest Capacity Asset", "BTCUSDT 18N"},
+            {"Fitness Score", "0.208"},
+            {"Kelly Criterion Estimate", "0"},
+            {"Kelly Criterion Probability Value", "0"},
+            {"Sortino Ratio", "79228162514264337593543950335"},
+            {"Return Over Maximum Drawdown", "26.189"},
+            {"Portfolio Turnover", "0.208"},
+            {"Total Insights Generated", "0"},
+            {"Total Insights Closed", "0"},
+            {"Total Insights Analysis Completed", "0"},
+            {"Long Insight Count", "0"},
+            {"Short Insight Count", "0"},
+            {"Long/Short Ratio", "100%"},
+            {"Estimated Monthly Alpha Value", "USDT0"},
+            {"Total Accumulated Estimated Alpha Value", "USDT0"},
+            {"Mean Population Estimated Insight Value", "USDT0"},
+            {"Mean Population Direction", "0%"},
+            {"Mean Population Magnitude", "0%"},
+            {"Rolling Averaged Population Direction", "0%"},
+            {"Rolling Averaged Population Magnitude", "0%"},
+            {"OrderListHash", "7417649395922ff3791471b4f3b5c021"}
+        };
+    }
+}

Algorithm.CSharp/BitfinexCashAccountFeeRegressionAlgorithm.cs

@@ -0,0 +1,84 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using QuantConnect.Brokerages;
+using System.Collections.Generic;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Bitfinex cash account regression algorithm, reproduces issue https://github.com/QuantConnect/Lean/issues/6123
+    /// </summary>
+    public class BitfinexCashAccountFeeRegressionAlgorithm : CryptoCashAccountFeeRegressionAlgorithm
+    {
+        public override void Initialize()
+        {
+            SetStartDate(2013, 10, 02);
+            SetEndDate(2013, 10, 03);
+            BrokerageName = BrokerageName.Bitfinex;
+            Pair = "BTCUSD";
+            base.Initialize();
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public override Dictionary<string, string> ExpectedStatistics => new()
+        {
+            {"Total Trades", "49"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "0%"},
+            {"Drawdown", "0%"},
+            {"Expectancy", "0"},
+            {"Net Profit", "0%"},
+            {"Sharpe Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "0%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "0"},
+            {"Tracking Error", "0"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "$1.13"},
+            {"Estimated Strategy Capacity", "$2000.00"},
+            {"Lowest Capacity Asset", "BTCUSD E3"},
+            {"Fitness Score", "0.002"},
+            {"Kelly Criterion Estimate", "0"},
+            {"Kelly Criterion Probability Value", "0"},
+            {"Sortino Ratio", "79228162514264337593543950335"},
+            {"Return Over Maximum Drawdown", "79228162514264337593543950335"},
+            {"Portfolio Turnover", "0.002"},
+            {"Total Insights Generated", "0"},
+            {"Total Insights Closed", "0"},
+            {"Total Insights Analysis Completed", "0"},
+            {"Long Insight Count", "0"},
+            {"Short Insight Count", "0"},
+            {"Long/Short Ratio", "100%"},
+            {"Estimated Monthly Alpha Value", "$0"},
+            {"Total Accumulated Estimated Alpha Value", "$0"},
+            {"Mean Population Estimated Insight Value", "$0"},
+            {"Mean Population Direction", "0%"},
+            {"Mean Population Magnitude", "0%"},
+            {"Rolling Averaged Population Direction", "0%"},
+            {"Rolling Averaged Population Magnitude", "0%"},
+            {"OrderListHash", "7f892f0c42d8826ff770ee602fe207a2"}
+        };
+    }
+}

Algorithm.CSharp/CryptoCashAccountFeeRegressionAlgorithm.cs

@@ -0,0 +1,105 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Linq;
+using QuantConnect.Util;
+using QuantConnect.Data;
+using QuantConnect.Orders;
+using QuantConnect.Interfaces;
+using QuantConnect.Brokerages;
+using System.Collections.Generic;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Base crypto cash account regression algorithm trading in and out
+    /// </summary>
+    public abstract class CryptoCashAccountFeeRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        private Symbol _symbol;
+
+        /// <summary>
+        /// The target brokerage model name
+        /// </summary>
+        protected BrokerageName BrokerageName { get; set; }
+
+        /// <summary>
+        /// The pair to add and trade
+        /// </summary>
+        protected string Pair { get; set; }
+
+        /// <summary>
+        /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
+        /// </summary>
+        public override void Initialize()
+        {
+            SetBrokerageModel(BrokerageName, AccountType.Cash);
+            _symbol = AddCrypto(Pair, Resolution.Hour).Symbol;
+        }
+
+        /// <summary>
+        /// OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
+        /// </summary>
+        /// <param name="data">Slice object keyed by symbol containing the stock data</param>
+        public override void OnData(Slice data)
+        {
+            if (!Portfolio.Invested)
+            {
+                CurrencyPairUtil.DecomposeCurrencyPair(_symbol, out var baseCurrency, out var quoteCurrency);
+
+                var initialQuoteCurrency = Portfolio.CashBook[quoteCurrency].Amount;
+                var ticket = Buy(_symbol, 0.1m);
+                var filledEvent = ticket.OrderEvents.Single(orderEvent => orderEvent.Status == OrderStatus.Filled);
+
+                if (Portfolio.CashBook[baseCurrency].Amount != ticket.QuantityFilled
+                    || filledEvent.FillQuantity != ticket.QuantityFilled
+                    || (0.1m - filledEvent.OrderFee.Value.Amount) != ticket.QuantityFilled)
+                {
+                    throw new Exception($"Unexpected BaseCurrency porfoltio status. Event {filledEvent}. CashBook: {Portfolio.CashBook}. ");
+                }
+
+                if (Portfolio.CashBook[quoteCurrency].Amount != (initialQuoteCurrency - 0.1m * filledEvent.FillPrice))
+                {
+                    throw new Exception($"Unexpected QuoteCurrency porfoltio status. Event {filledEvent}. CashBook: {Portfolio.CashBook}. ");
+                }
+
+                if (Securities[_symbol].Holdings.Quantity != (0.1m - filledEvent.OrderFee.Value.Amount))
+                {
+                    throw new Exception($"Unexpected Holdings: {Securities[_symbol].Holdings}. Event {filledEvent}");
+                }
+            }
+            else
+            {
+                Liquidate();
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public Language[] Languages { get; } = { Language.CSharp };
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public abstract Dictionary<string, string> ExpectedStatistics { get; }
+    }
+}

Algorithm.CSharp/CustomDataIndicatorExtensionsAlgorithm.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -14,16 +14,17 @@
 */
 
 using System;
+using System.IO;
+using System.Globalization;
 using QuantConnect.Data;
-using QuantConnect.Data.Custom;
-using QuantConnect.Data.Market;
+using QuantConnect.Util;
 using QuantConnect.Indicators;
 
 namespace QuantConnect.Algorithm.CSharp
 {
     /// <summary>
     /// The algorithm creates new indicator value with the existing indicator method by Indicator Extensions
-    /// Demonstration of using the external custom datasource Quandl to request the VIX and VXV daily data
+    /// Demonstration of using local custom datasource CustomData to request the IBM and SPY daily data
     /// </summary>
     /// <meta name="tag" content="using data" />
     /// <meta name="tag" content="using quantconnect" />
@@ -34,10 +35,10 @@ namespace QuantConnect.Algorithm.CSharp
     /// <meta name="tag" content="charting" />
     public class CustomDataIndicatorExtensionsAlgorithm : QCAlgorithm
     {
-        private const string _vix = "CBOE/VIX";
-        private const string _vxv = "CBOE/VXV";
-        private SimpleMovingAverage _smaVIX;
-        private SimpleMovingAverage _smaVXV;
+        private const string _ibm = "IBM";
+        private const string _spy = "SPY";
+        private SimpleMovingAverage _smaIBM;
+        private SimpleMovingAverage _smaSPY;
         private IndicatorBase<IndicatorDataPoint> _ratio;
 
         /// <summary>
@@ -50,46 +51,82 @@ namespace QuantConnect.Algorithm.CSharp
             SetCash(25000);
 
             // Define the symbol and "type" of our generic data
-            AddData<QuandlVix>(_vix, Resolution.Daily);
-            AddData<Quandl>(_vxv, Resolution.Daily);
+            AddData<CustomData>(_ibm, Resolution.Daily);
+            AddData<CustomData>(_spy, Resolution.Daily);
             // Set up default Indicators, these are just 'identities' of the closing price
-            _smaVIX = SMA(_vix, 1);
-            _smaVXV = SMA(_vxv, 1);
-            // This will create a new indicator whose value is smaVXV / smaVIX
-            _ratio = _smaVXV.Over(_smaVIX);
+            _smaIBM = SMA(_ibm, 1);
+            _smaSPY = SMA(_spy, 1);
+            // This will create a new indicator whose value is smaSPY / smaIBM
+            _ratio = _smaSPY.Over(_smaIBM);
         }
 
         /// <summary>
         /// Custom data event handler:
         /// </summary>
-        /// <param name="data">Quandl - dictionary Bars of Quandl Data</param>
-        public void OnData(Quandl data)
+        /// <param name="data">CustomData - dictionary Bars of custom data</param>
+        public void OnData(CustomData data)
         {
             // Wait for all indicators to fully initialize
-            if (_smaVIX.IsReady && _smaVXV.IsReady && _ratio.IsReady)
+            if (_smaIBM.IsReady && _smaSPY.IsReady && _ratio.IsReady)
             {
                 if (!Portfolio.Invested && _ratio > 1)
                 {
-                    MarketOrder(_vix, 100);
+                    MarketOrder(_ibm, 100);
                 }
                 else if (_ratio < 1)
                 {
                     Liquidate();
                 }
                 // plot all indicators
-                PlotIndicator("SMA", _smaVIX, _smaVXV);
+                PlotIndicator("SMA", _smaIBM, _smaSPY);
                 PlotIndicator("Ratio", _ratio);
             }
         }
     }
 
     /// <summary>
-    /// In CBOE/VIX data, there is a "vix close" column instead of "close" which is the 
-    /// default column namein LEAN Quandl custom data implementation.
-    /// This class assigns new column name to match the the external datasource setting.
+    /// Custom data from local LEAN data
     /// </summary>
-    public class QuandlVix : Quandl
+    public class CustomData : BaseData
     {
-        public QuandlVix() : base(valueColumnName: "vix close") { }
+        public decimal Open;
+        public decimal High;
+        public decimal Low;
+        public decimal Close;
+
+        public override DateTime EndTime
+        {
+            get { return Time + Period; }
+            set { Time = value - Period; }
+        }
+
+        public TimeSpan Period
+        {
+            get { return QuantConnect.Time.OneDay; }
+        }
+
+        public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
+        {
+            var source = Path.Combine(Globals.DataFolder, "equity", "usa", config.Resolution.ToString().ToLower(), LeanData.GenerateZipFileName(config.Symbol, date, config.Resolution, config.TickType));
+            return new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.Csv);
+        }
+
+        public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
+        {
+            var csv = line.ToCsv(6);
+            var _scaleFactor = 1 / 10000m;
+
+            var custom = new CustomData
+            {
+                Symbol = config.Symbol,
+                Time = DateTime.ParseExact(csv[0], DateFormat.TwelveCharacter, CultureInfo.InvariantCulture),
+                Open = csv[1].ToDecimal() * _scaleFactor,
+                High = csv[2].ToDecimal() * _scaleFactor,
+                Low = csv[3].ToDecimal() * _scaleFactor,
+                Close = csv[4].ToDecimal() * _scaleFactor,
+                Value = csv[4].ToDecimal() * _scaleFactor
+            };
+            return custom;
+        }
     }
-}
+}

Algorithm.CSharp/CustomDataRegressionAlgorithm.cs

@@ -18,7 +18,9 @@ using System.Collections.Generic;
 using System.Globalization;
 using Newtonsoft.Json;
 using QuantConnect.Data;
+using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Interfaces;
+using QuantConnect.Securities;
 
 namespace QuantConnect.Algorithm.CSharp
 {
@@ -31,12 +33,14 @@ namespace QuantConnect.Algorithm.CSharp
     /// <meta name="tag" content="regression test" />
     public class CustomDataRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
+        private bool _warmedUpChecked = false;
+
         /// <summary>
         /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
         /// </summary>
         public override void Initialize()
         {
-            SetStartDate(2011, 9, 13);
+            SetStartDate(2011, 9, 14);
             SetEndDate(2015, 12, 01);
 
             //Set the cash for the strategy:
@@ -45,6 +49,9 @@ namespace QuantConnect.Algorithm.CSharp
             //Define the symbol and "type" of our generic data:
             var resolution = LiveMode ? Resolution.Second : Resolution.Daily;
             AddData<Bitcoin>("BTC", resolution);
+
+            var seeder = new FuncSecuritySeeder(GetLastKnownPrices);
+            SetSecurityInitializer(security => seeder.SeedSecurity(security));
         }
 
         /// <summary>
@@ -66,6 +73,30 @@ namespace QuantConnect.Algorithm.CSharp
             }
         }
 
+        public override void OnSecuritiesChanged(SecurityChanges changes)
+        {
+            changes.FilterCustomSecurities = false;
+            foreach (var addedSecurity in changes.AddedSecurities)
+            {
+                if (addedSecurity.Symbol.Value == "BTC")
+                {
+                    _warmedUpChecked = true;
+                }
+                if (!addedSecurity.HasData)
+                {
+                    throw new Exception($"Security {addedSecurity.Symbol} was not warmed up!");
+                }
+            }
+        }
+
+        public override void OnEndOfAlgorithm()
+        {
+            if (!_warmedUpChecked)
+            {
+                throw new Exception($"Security was not warmed up!");
+            }
+        }
+
         /// <summary>
         /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
         /// </summary>
@@ -84,30 +115,30 @@ namespace QuantConnect.Algorithm.CSharp
             {"Total Trades", "1"},
             {"Average Win", "0%"},
             {"Average Loss", "0%"},
-            {"Compounding Annual Return", "157.497%"},
+            {"Compounding Annual Return", "157.655%"},
             {"Drawdown", "84.800%"},
             {"Expectancy", "0"},
             {"Net Profit", "5319.007%"},
-            {"Sharpe Ratio", "2.086"},
-            {"Probabilistic Sharpe Ratio", "69.456%"},
+            {"Sharpe Ratio", "2.123"},
+            {"Probabilistic Sharpe Ratio", "70.581%"},
             {"Loss Rate", "0%"},
             {"Win Rate", "0%"},
             {"Profit-Loss Ratio", "0"},
-            {"Alpha", "1.747"},
-            {"Beta", "0.047"},
+            {"Alpha", "1.776"},
+            {"Beta", "0.059"},
             {"Annual Standard Deviation", "0.84"},
             {"Annual Variance", "0.706"},
-            {"Information Ratio", "1.922"},
-            {"Tracking Error", "0.848"},
-            {"Treynor Ratio", "37.473"},
+            {"Information Ratio", "1.962"},
+            {"Tracking Error", "0.847"},
+            {"Treynor Ratio", "30.455"},
             {"Total Fees", "$0.00"},
             {"Estimated Strategy Capacity", "$0"},
             {"Lowest Capacity Asset", "BTC.Bitcoin 2S"},
             {"Fitness Score", "0"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "2.269"},
-            {"Return Over Maximum Drawdown", "1.858"},
+            {"Sortino Ratio", "2.271"},
+            {"Return Over Maximum Drawdown", "1.86"},
             {"Portfolio Turnover", "0"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},

Algorithm.CSharp/HistoryAlgorithm.cs

@@ -14,10 +14,10 @@
 */
 
 using System;
+using QuantConnect.Util;
 using System.Collections.Generic;
 using System.Linq;
 using QuantConnect.Data;
-using QuantConnect.Data.Custom;
 using QuantConnect.Data.Market;
 using QuantConnect.Indicators;
 using QuantConnect.Securities.Equity;
@@ -36,7 +36,7 @@ namespace QuantConnect.Algorithm.CSharp
     public class HistoryAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
         private int _count;
-        private SimpleMovingAverage _spyDailySma;
+        private SimpleMovingAverage _dailySma;
 
         /// <summary>
         /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
@@ -49,12 +49,12 @@ namespace QuantConnect.Algorithm.CSharp
 
             // Find more symbols here: http://quantconnect.com/data
             var SPY = AddSecurity(SecurityType.Equity, "SPY", Resolution.Daily).Symbol;
-            var CME_SP1 = AddData<QuandlFuture>("CHRIS/CME_SP1", Resolution.Daily).Symbol;
+            var IBM = AddData<CustomData>("IBM", Resolution.Daily).Symbol;
             // specifying the exchange will allow the history methods that accept a number of bars to return to work properly
-            Securities["CHRIS/CME_SP1"].Exchange = new EquityExchange();
+            Securities["IBM"].Exchange = new EquityExchange();
 
             // we can get history in initialize to set up indicators and such
-            _spyDailySma = new SimpleMovingAverage(14);
+            _dailySma = new SimpleMovingAverage(14);
 
             // get the last calendar year's worth of SPY data at the configured resolution (daily)
             var tradeBarHistory = History<TradeBar>("SPY", TimeSpan.FromDays(365));
@@ -76,99 +76,97 @@ namespace QuantConnect.Algorithm.CSharp
             // we can use these TradeBars to initialize indicators or perform other math
             foreach (TradeBar tradeBar in tradeBarHistory)
             {
-                _spyDailySma.Update(tradeBar.EndTime, tradeBar.Close);
+                _dailySma.Update(tradeBar.EndTime, tradeBar.Close);
             }
 
-            // get the last calendar year's worth of quandl data at the configured resolution (daily)
-            var quandlHistory = History<QuandlFuture>("CHRIS/CME_SP1", TimeSpan.FromDays(365));
-            AssertHistoryCount("History<Quandl>(\"CHRIS/CME_SP1\", TimeSpan.FromDays(365))", quandlHistory, 250, CME_SP1);
+            // get the last calendar year's worth of IBM data at the configured resolution (daily)
+            var customDataHistory = History<CustomData>("IBM", TimeSpan.FromDays(365));
+            AssertHistoryCount("History<CustomData>(\"IBM\", TimeSpan.FromDays(365))", customDataHistory, 250, IBM);
 
-            // get the last 14 bars of SPY at the configured resolution (daily)
-            quandlHistory = History<QuandlFuture>("CHRIS/CME_SP1", 14);
-            AssertHistoryCount("History<Quandl>(\"CHRIS/CME_SP1\", 14)", quandlHistory, 14, CME_SP1);
+            // get the last 14 bars of IBM at the configured resolution (daily)
+            customDataHistory = History<CustomData>("IBM", 14);
+            AssertHistoryCount("History<CustomData>(\"IBM\", 14)", customDataHistory, 14, IBM);
 
-            // get the last 14 minute bars of SPY
-
-            // we can loop over the return values from these functions and we'll get Quandl data
+            // we can loop over the return values from these functions and we'll get custom data
             // this can be used in much the same way as the tradeBarHistory above
-            _spyDailySma.Reset();
-            foreach (QuandlFuture quandl in quandlHistory)
+            _dailySma.Reset();
+            foreach (CustomData customData in customDataHistory)
             {
-                _spyDailySma.Update(quandl.EndTime, quandl.Value);
+                _dailySma.Update(customData.EndTime, customData.Value);
             }
 
-            // get the last year's worth of all configured Quandl data at the configured resolution (daily)
-            var allQuandlData = History<QuandlFuture>(TimeSpan.FromDays(365));
-            AssertHistoryCount("History<QuandlFuture>(TimeSpan.FromDays(365))", allQuandlData, 250, CME_SP1);
+            // get the last year's worth of all configured custom data at the configured resolution (daily)
+            var allCustomData = History<CustomData>(TimeSpan.FromDays(365));
+            AssertHistoryCount("History<CustomData>(TimeSpan.FromDays(365))", allCustomData, 250, IBM);
 
-            // get the last 14 bars worth of Quandl data for the specified symbols at the configured resolution (daily)
-            allQuandlData = History<QuandlFuture>(Securities.Keys, 14);
-            AssertHistoryCount("History<QuandlFuture>(Securities.Keys, 14)", allQuandlData, 14, CME_SP1);
+            // get the last 14 bars worth of custom data for the specified symbols at the configured resolution (daily)
+            allCustomData = History<CustomData>(Securities.Keys, 14);
+            AssertHistoryCount("History<CustomData>(Securities.Keys, 14)", allCustomData, 14, IBM);
 
-            // NOTE: using different resolutions require that they are properly implemented in your data type, since
-            //  Quandl doesn't support minute data, this won't actually work, but if your custom data source has
-            //  different resolutions, it would need to be implemented in the GetSource and Reader methods properly
-            //quandlHistory = History<QuandlFuture>("CHRIS/CME_SP1", TimeSpan.FromDays(7), Resolution.Minute);
-            //quandlHistory = History<QuandlFuture>("CHRIS/CME_SP1", 14, Resolution.Minute);
-            //allQuandlData = History<QuandlFuture>(TimeSpan.FromDays(365), Resolution.Minute);
-            //allQuandlData = History<QuandlFuture>(Securities.Keys, 14, Resolution.Minute);
-            //allQuandlData = History<QuandlFuture>(Securities.Keys, TimeSpan.FromDays(1), Resolution.Minute);
-            //allQuandlData = History<QuandlFuture>(Securities.Keys, 14, Resolution.Minute);
+            // NOTE: Using different resolutions require that they are properly implemented in your data type. If your
+            // custom data source has different resolutions, it would need to be implemented in the GetSource and Reader
+            // methods properly.
+            //customDataHistory = History<CustomData>("IBM", TimeSpan.FromDays(7), Resolution.Minute);
+            //customDataHistory = History<CustomData>("IBM", 14, Resolution.Minute);
+            //allCustomData = History<CustomData>(TimeSpan.FromDays(365), Resolution.Minute);
+            //allCustomData = History<CustomData>(Securities.Keys, 14, Resolution.Minute);
+            //allCustomData = History<CustomData>(Securities.Keys, TimeSpan.FromDays(1), Resolution.Minute);
+            //allCustomData = History<CustomData>(Securities.Keys, 14, Resolution.Minute);
 
-            // get the last calendar year's worth of all quandl data
-            allQuandlData = History<QuandlFuture>(Securities.Keys, TimeSpan.FromDays(365));
-            AssertHistoryCount("History<QuandlFuture>(Securities.Keys, TimeSpan.FromDays(365))", allQuandlData, 250, CME_SP1);
+            // get the last calendar year's worth of all custom data
+            allCustomData = History<CustomData>(Securities.Keys, TimeSpan.FromDays(365));
+            AssertHistoryCount("History<CustomData>(Securities.Keys, TimeSpan.FromDays(365))", allCustomData, 250, IBM);
 
-            // the return is a series of dictionaries containing all quandl data at each time
+            // the return is a series of dictionaries containing all custom data at each time
             // we can loop over it to get the individual dictionaries
-            foreach (DataDictionary<QuandlFuture> quandlsDataDictionary in allQuandlData)
+            foreach (DataDictionary<CustomData> customDataDictionary in allCustomData)
             {
-                // we can access the dictionary to get the quandl data we want
-                var quandl = quandlsDataDictionary["CHRIS/CME_SP1"];
+                // we can access the dictionary to get the custom data we want
+                var customData = customDataDictionary["IBM"];
             }
 
             // we can also access the return value from the multiple symbol functions to request a single
             // symbol and then loop over it
-            var singleSymbolQuandl = allQuandlData.Get("CHRIS/CME_SP1");
-            AssertHistoryCount("allQuandlData.Get(\"CHRIS/CME_SP1\")", singleSymbolQuandl, 250, CME_SP1);
-            foreach (QuandlFuture quandl in singleSymbolQuandl)
+            var singleSymbolCustomData = allCustomData.Get("IBM");
+            AssertHistoryCount("allCustomData.Get(\"IBM\")", singleSymbolCustomData, 250, IBM);
+            foreach (CustomData customData in singleSymbolCustomData)
             {
-                // do something with 'CHRIS/CME_SP1' quandl data
+                // do something with 'IBM' custom data
             }
 
             // we can also access individual properties on our data, this will
-            // get the 'CHRIS/CME_SP1' quandls like above, but then only return the Low properties
-            var quandlSpyLows = allQuandlData.Get("CHRIS/CME_SP1", "Low");
-            AssertHistoryCount("allQuandlData.Get(\"CHRIS/CME_SP1\", \"Low\")", quandlSpyLows, 250);
-            foreach (decimal low in quandlSpyLows)
+            // get the 'IBM' CustomData objects like above, but then only return the Value properties
+            var customDataIbmValues = allCustomData.Get("IBM", "Value");
+            AssertHistoryCount("allCustomData.Get(\"IBM\", \"Value\")", customDataIbmValues, 250);
+            foreach (decimal value in customDataIbmValues)
             {
-                // do something with each low value
+                // do something with each value
             }
 
             // sometimes it's necessary to get the history for many configured symbols
 
             // request the last year's worth of history for all configured symbols at their configured resolutions
             var allHistory = History(TimeSpan.FromDays(365));
-            AssertHistoryCount("History(TimeSpan.FromDays(365))", allHistory, 250, SPY, CME_SP1);
+            AssertHistoryCount("History(TimeSpan.FromDays(365))", allHistory, 250, SPY, IBM);
 
             // request the last days's worth of history at the minute resolution
             allHistory = History(TimeSpan.FromDays(1), Resolution.Minute);
-            AssertHistoryCount("History(TimeSpan.FromDays(1), Resolution.Minute)", allHistory, 391, SPY, CME_SP1);
+            AssertHistoryCount("History(TimeSpan.FromDays(1), Resolution.Minute)", allHistory, 390, SPY, IBM);
 
             // request the last 100 bars for the specified securities at the configured resolution
             allHistory = History(Securities.Keys, 100);
-            AssertHistoryCount("History(Securities.Keys, 100)", allHistory, 100, SPY, CME_SP1);
+            AssertHistoryCount("History(Securities.Keys, 100)", allHistory, 100, SPY, IBM);
 
             // request the last 100 minute bars for the specified securities
             allHistory = History(Securities.Keys, 100, Resolution.Minute);
-            AssertHistoryCount("History(Securities.Keys, 100, Resolution.Minute)", allHistory, 101, SPY, CME_SP1);
+            AssertHistoryCount("History(Securities.Keys, 100, Resolution.Minute)", allHistory, 100, SPY, IBM);
 
             // request the last calendar years worth of history for the specified securities
             allHistory = History(Securities.Keys, TimeSpan.FromDays(365));
-            AssertHistoryCount("History(Securities.Keys, TimeSpan.FromDays(365))", allHistory, 250, SPY, CME_SP1);
+            AssertHistoryCount("History(Securities.Keys, TimeSpan.FromDays(365))", allHistory, 250, SPY, IBM);
             // we can also specify the resolution
             allHistory = History(Securities.Keys, TimeSpan.FromDays(1), Resolution.Minute);
-            AssertHistoryCount("History(Securities.Keys, TimeSpan.FromDays(1), Resolution.Minute)", allHistory, 391, SPY, CME_SP1);
+            AssertHistoryCount("History(Securities.Keys, TimeSpan.FromDays(1), Resolution.Minute)", allHistory, 390, SPY, IBM);
 
             // if we loop over this allHistory, we get Slice objects
             foreach (Slice slice in allHistory)
@@ -215,7 +213,7 @@ namespace QuantConnect.Algorithm.CSharp
 
             if (_count > 5)
             {
-                throw new Exception("Invalid number of bars arrived. Expected exactly 5");
+                throw new Exception($"Invalid number of bars arrived. Expected exactly 5, but received {_count}");
             }
 
             if (!Portfolio.Invested)
@@ -245,9 +243,9 @@ namespace QuantConnect.Algorithm.CSharp
             }
             else if (typeof(T).IsGenericType && typeof(T).GetGenericTypeDefinition() == typeof(DataDictionary<>))
             {
-                if (typeof(T).GetGenericArguments()[0] == typeof(QuandlFuture))
+                if (typeof(T).GetGenericArguments()[0] == typeof(CustomData))
                 {
-                    var dictionaries = (IEnumerable<DataDictionary<QuandlFuture>>) history;
+                    var dictionaries = (IEnumerable<DataDictionary<CustomData>>) history;
                     unexpectedSymbols = dictionaries.SelectMany(dd => dd.Keys)
                         .Distinct()
                         .Where(sym => !expectedSymbols.Contains(sym))
@@ -340,19 +338,5 @@ namespace QuantConnect.Algorithm.CSharp
             {"Rolling Averaged Population Magnitude", "0%"},
             {"OrderListHash", "33d01821923c397f999cfb2e5b5928ad"}
         };
-
-        /// <summary>
-        /// Custom quandl data type for setting customized value column name. Value column is used for the primary trading calculations and charting.
-        /// </summary>
-        public class QuandlFuture : Quandl
-        {
-            /// <summary>
-            /// Initializes a new instance of the <see cref="QuandlFuture"/> class.
-            /// </summary>
-            public QuandlFuture()
-                : base(valueColumnName: "Settle")
-            {
-            }
-        }
     }
-}
+}

Algorithm.CSharp/IndiaDataRegressionAlgorithm.cs

@@ -0,0 +1,202 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using QuantConnect.Interfaces;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Regression algorithm demonstrating use of map files with India data
+    /// </summary>
+    /// <meta name="tag" content="using data" />
+    /// <meta name="tag" content="India data" />
+    /// <meta name="tag" content="regression test" />
+    /// <meta name="tag" content="rename event" />
+    /// <meta name="tag" content="map" />
+    /// <meta name="tag" content="mapping" />
+    /// <meta name="tag" content="map files" />
+    public class IndiaDataRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        private Symbol _mappingSymbol, _splitAndDividendSymbol;
+        private bool _initialMapping;
+        private bool _executionMapping;
+        private bool _receivedWarningEvent;
+        private bool _receivedOccurredEvent;
+
+        /// <summary>
+        /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
+        /// </summary>
+        public override void Initialize()
+        {
+            SetAccountCurrency("INR");  //Set Account Currency 
+            SetStartDate(2004, 5, 20);  //Set Start Date
+            SetEndDate(2016, 7, 26);    //Set End Date
+            _mappingSymbol = AddEquity("3MINDIA", Resolution.Daily, Market.India).Symbol;
+            _splitAndDividendSymbol = AddEquity("CCCL", Resolution.Daily, Market.India).Symbol;
+        }
+
+        /// <summary>
+        /// Raises the data event.
+        /// </summary>
+        /// <param name="data">Data.</param>
+        public void OnData(Dividends data)
+        {
+            if (data.ContainsKey(_splitAndDividendSymbol))
+            {
+                var dividend = data[_splitAndDividendSymbol];
+                if (Time.Date == new DateTime(2010, 06, 15) &&
+                    (dividend.Price != 0.5m || dividend.ReferencePrice != 88.8m || dividend.Distribution != 0.5m))
+                {
+                    throw new Exception("Did not receive expected dividend values");
+                }
+            }
+        }
+
+        /// <summary>
+        /// Raises the data event.
+        /// </summary>
+        /// <param name="data">Data.</param>
+        public void OnData(Splits data)
+        {
+            if (data.ContainsKey(_splitAndDividendSymbol))
+            {
+                var split = data[_splitAndDividendSymbol];
+                if (split.Type == SplitType.Warning)
+                {
+                    _receivedWarningEvent = true;
+                }
+                else if (split.Type == SplitType.SplitOccurred)
+                {
+                    _receivedOccurredEvent = true;
+                    if (split.Price != 421m || split.ReferencePrice != 421m || split.SplitFactor != 0.2m)
+                    {
+                        throw new Exception("Did not receive expected split values");
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Checks the symbol change event
+        /// </summary>
+        public override void OnData(Slice slice)
+        {
+            if (slice.SymbolChangedEvents.ContainsKey(_mappingSymbol))
+            {
+                var mappingEvent = slice.SymbolChangedEvents.Single(x => x.Key.SecurityType == SecurityType.Equity).Value;
+                Log($"{Time} - Ticker changed from: {mappingEvent.OldSymbol} to {mappingEvent.NewSymbol}");
+                if (Time.Date == new DateTime(1999, 01, 01))
+                {
+                    _initialMapping = true;
+                }
+                else if (Time.Date == new DateTime(2004, 06, 15))
+                {
+                    if (mappingEvent.NewSymbol == "3MINDIA"
+                        && mappingEvent.OldSymbol == "BIRLA3M")
+                    {
+                        _executionMapping = true;
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Final step of the algorithm
+        /// </summary>
+        public override void OnEndOfAlgorithm()
+        {
+            if (_initialMapping)
+            {
+                throw new Exception("The ticker generated the initial rename event");
+            }
+            if (!_executionMapping)
+            {
+                throw new Exception("The ticker did not rename throughout the course of its life even though it should have");
+            }
+            if (!_receivedOccurredEvent)
+            {
+                throw new Exception("Did not receive expected split event");
+            }
+            if (!_receivedWarningEvent)
+            {
+                throw new Exception("Did not receive expected split warning event");
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public Language[] Languages { get; } = { Language.CSharp, Language.Python };
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Trades", "0"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "0%"},
+            {"Drawdown", "0%"},
+            {"Expectancy", "0"},
+            {"Net Profit", "0%"},
+            {"Sharpe Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "0%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "0"},
+            {"Tracking Error", "0"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "$0.00"},
+            {"Estimated Strategy Capacity", "$0"},
+            {"Lowest Capacity Asset", ""},
+            {"Fitness Score", "0"},
+            {"Kelly Criterion Estimate", "0"},
+            {"Kelly Criterion Probability Value", "0"},
+            {"Sortino Ratio", "79228162514264337593543950335"},
+            {"Return Over Maximum Drawdown", "79228162514264337593543950335"},
+            {"Portfolio Turnover", "0"},
+            {"Total Insights Generated", "0"},
+            {"Total Insights Closed", "0"},
+            {"Total Insights Analysis Completed", "0"},
+            {"Long Insight Count", "0"},
+            {"Short Insight Count", "0"},
+            {"Long/Short Ratio", "100%"},
+            {"Estimated Monthly Alpha Value", "₹0"},
+            {"Total Accumulated Estimated Alpha Value", "₹0"},
+            {"Mean Population Estimated Insight Value", "₹0"},
+            {"Mean Population Direction", "0%"},
+            {"Mean Population Magnitude", "0%"},
+            {"Rolling Averaged Population Direction", "0%"},
+            {"Rolling Averaged Population Magnitude", "0%"},
+            {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"}
+        };
+    }
+}

Algorithm.CSharp/IndicatorSuiteAlgorithm.cs

@@ -16,7 +16,6 @@
 using System;
 using System.Collections.Generic;
 using QuantConnect.Data;
-using QuantConnect.Data.Custom;
 using QuantConnect.Data.Market;
 using QuantConnect.Indicators;
 using QuantConnect.Interfaces;
@@ -34,7 +33,7 @@ namespace QuantConnect.Algorithm.CSharp
     public class IndicatorSuiteAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
         private string _ticker = "SPY";
-        private string _customTicker = "WIKI/FB";
+        private string _customTicker = "IBM";
 
         private Symbol _symbol;
         private Symbol _customSymbol;
@@ -64,7 +63,7 @@ namespace QuantConnect.Algorithm.CSharp
             _symbol = AddSecurity(SecurityType.Equity, _ticker, Resolution.Daily).Symbol;
 
             //Add the Custom Data:
-            _customSymbol = AddData<Quandl>(_customTicker, Resolution.Daily).Symbol;
+            _customSymbol = AddData<CustomData>(_customTicker, Resolution.Daily).Symbol;
 
             //Set up default Indicators, these indicators are defined on the Value property of incoming data (except ATR and AROON which use the full TradeBar object)
             _indicators = new Indicators
@@ -118,9 +117,9 @@ namespace QuantConnect.Algorithm.CSharp
             // these are indicators that require multiple inputs. the most common of which is a ratio.
             // suppose we seek the ratio of BTC to SPY, we could write the following:
             var spyClose = Identity(_symbol);
-            var fbClose = Identity(_customSymbol);
+            var ibmClose = Identity(_customSymbol);
             // this will create a new indicator whose value is FB/SPY
-            _ratio = fbClose.Over(spyClose);
+            _ratio = ibmClose.Over(spyClose);
             // we can also easily plot our indicators each time they update using th PlotIndicator function
             PlotIndicator("Ratio", _ratio);
         }
@@ -128,8 +127,8 @@ namespace QuantConnect.Algorithm.CSharp
         /// <summary>
         /// Custom data event handler:
         /// </summary>
-        /// <param name="data">Quandl - dictionary Bars of Quandl Data</param>
-        public void OnData(Quandl data)
+        /// <param name="data">CustomData - dictionary Bars of custom data</param>
+        public void OnData(CustomData data)
         {
         }
 

Algorithm.CSharp/ObjectStoreExampleAlgorithm.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -18,6 +18,7 @@ using QuantConnect.Indicators;
 using QuantConnect.Interfaces;
 using QuantConnect.Storage;
 using System;
+using System.Collections.Generic;
 using System.Linq;
 
 namespace QuantConnect.Algorithm.CSharp
@@ -29,7 +30,7 @@ namespace QuantConnect.Algorithm.CSharp
     /// history call. This pattern can be equally applied to a machine learning model being
     /// trained and then saving the model weights in the object store.
     /// </summary>
-    public class ObjectStoreExampleAlgorithm : QCAlgorithm
+    public class ObjectStoreExampleAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
         private const string SPY_Close_ObjectStore_Key = "spy_close";
         private Symbol SPY;
@@ -127,5 +128,64 @@ namespace QuantConnect.Algorithm.CSharp
                 }
             }
         }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public Language[] Languages { get; } = { Language.CSharp, Language.Python };
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Trades", "1"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "271.453%"},
+            {"Drawdown", "2.200%"},
+            {"Expectancy", "0"},
+            {"Net Profit", "1.692%"},
+            {"Sharpe Ratio", "8.888"},
+            {"Probabilistic Sharpe Ratio", "67.609%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "-0.005"},
+            {"Beta", "0.996"},
+            {"Annual Standard Deviation", "0.222"},
+            {"Annual Variance", "0.049"},
+            {"Information Ratio", "-14.565"},
+            {"Tracking Error", "0.001"},
+            {"Treynor Ratio", "1.978"},
+            {"Total Fees", "$3.44"},
+            {"Estimated Strategy Capacity", "$56000000.00"},
+            {"Lowest Capacity Asset", "SPY R735QTJ8XC9X"},
+            {"Fitness Score", "0.248"},
+            {"Kelly Criterion Estimate", "0"},
+            {"Kelly Criterion Probability Value", "0"},
+            {"Sortino Ratio", "79228162514264337593543950335"},
+            {"Return Over Maximum Drawdown", "93.728"},
+            {"Portfolio Turnover", "0.248"},
+            {"Total Insights Generated", "0"},
+            {"Total Insights Closed", "0"},
+            {"Total Insights Analysis Completed", "0"},
+            {"Long Insight Count", "0"},
+            {"Short Insight Count", "0"},
+            {"Long/Short Ratio", "100%"},
+            {"Estimated Monthly Alpha Value", "$0"},
+            {"Total Accumulated Estimated Alpha Value", "$0"},
+            {"Mean Population Estimated Insight Value", "$0"},
+            {"Mean Population Direction", "0%"},
+            {"Mean Population Magnitude", "0%"},
+            {"Rolling Averaged Population Direction", "0%"},
+            {"Rolling Averaged Population Magnitude", "0%"},
+            {"OrderListHash", "9e4bfd2eb0b81ee5bc1b197a87ccedbe"}
+        };
     }
 }

Algorithm.CSharp/ParameterizedAlgorithm.cs

@@ -28,9 +28,9 @@ namespace QuantConnect.Algorithm.CSharp
     /// <meta name="tag" content="using quantconnect" />
     public class ParameterizedAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
-        // we place attributes on top of our fields or properties that should receive
+        // We place attributes on top of our fields or properties that should receive
         // their values from the job. The values 100 and 200 are just default values that
-        // or only used if the parameters do not exist
+        // are only used if the parameters do not exist.
         [Parameter("ema-fast")]
         public int FastPeriod = 100;
 

Algorithm.CSharp/QuandlFuturesDataAlgorithm.cs

@@ -1,73 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using QuantConnect.Data.Custom;
-
-namespace QuantConnect.Algorithm.CSharp
-{
-    /// <summary>
-    /// Futures demonstration algorithm.
-    /// QuantConnect allows importing generic data sources! This example demonstrates importing a futures
-    /// data from the popular open data source Quandl. QuantConnect has a special deal with Quandl giving you access
-    /// to Stevens Continuous Futurs (SCF) for free. If you'd like to download SCF for local backtesting, you can download it through Quandl.com.
-    /// </summary>
-    /// <meta name="tag" content="using data" />
-    /// <meta name="tag" content="quandl" />
-    /// <meta name="tag" content="custom data" />
-    /// <meta name="tag" content="futures" />
-    public class QCUQuandlFutures : QCAlgorithm
-    {
-        private string _crude = "SCF/CME_CL1_ON";
-
-        /// <summary>
-        /// Initialize the data and resolution you require for your strategy
-        /// </summary>
-        public override void Initialize()
-        {
-            SetStartDate(2000, 1, 1);
-            SetEndDate(DateTime.Now.Date.AddDays(-1));
-            SetCash(25000);
-            AddData<QuandlFuture>(_crude, Resolution.Daily);
-        }
-
-        /// <summary>
-        /// Data Event Handler: New data arrives here. "TradeBars" type is a dictionary of strings so you can access it by symbol.
-        /// </summary>
-        /// <param name="data">Data.</param>
-        public void OnData(Quandl data)
-        {
-            if (!Portfolio.HoldStock)
-            {
-                SetHoldings(_crude, 1);
-                Debug(Time.ToStringInvariant("u") + " Purchased Crude Oil: " + _crude);
-            }
-        }
-
-        /// <summary>
-        /// Custom quandl data type for setting customized value column name. Value column is used for the primary trading calculations and charting.
-        /// </summary>
-        public class QuandlFuture : Quandl
-        {
-            /// <summary>
-            /// Initializes a new instance of the <see cref="QuandlFuture"/> class.
-            /// </summary>
-            public QuandlFuture()
-                : base(valueColumnName: "Settle")
-            {
-            }
-        }
-    }
-}

Algorithm.CSharp/QuandlImporterAlgorithm.cs

@@ -1,70 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using QuantConnect.Data.Custom;
-using QuantConnect.Indicators;
-
-namespace QuantConnect.Algorithm.CSharp
-{
-    /// <summary>
-    /// Using the underlying dynamic data class "Quandl" QuantConnect take care of the data
-    /// importing and definition for you. Simply point QuantConnect to the Quandl Short Code.
-    /// The Quandl object has properties which match the spreadsheet headers.
-    /// If you have multiple quandl streams look at data.Symbol to distinguish them.
-    /// </summary>
-    /// <meta name="tag" content="custom data" />
-    /// <meta name="tag" content="using data" />
-    /// <meta name="tag" content="quandl" />
-    public class QuandlImporterAlgorithm : QCAlgorithm
-    {
-        private SimpleMovingAverage _sma;
-        string _quandlCode = "WIKI/IBM";
-
-        /// Initialize the data and resolution you require for your strategy:
-        public override void Initialize()
-        {
-            //Start and End Date range for the backtest:
-            SetStartDate(2013, 1, 1);
-            SetEndDate(DateTime.Now.Date.AddDays(-1));
-
-            //Cash allocation
-            SetCash(25000);
-
-            // Optional argument - personal token necessary for restricted dataset
-            // Quandl.SetAuthCode("your-quandl-token");
-
-            //Add Generic Quandl Data:
-            AddData<Quandl>(_quandlCode, Resolution.Daily);
-
-            _sma = SMA(_quandlCode, 14);
-        }
-
-        /// Data Event Handler: New data arrives here. "TradeBars" type is a dictionary of strings so you can access it by symbol
-        public void OnData(Quandl data)
-        {
-            if (!Portfolio.HoldStock)
-            {
-                //Order function places trades: enter the string symbol and the quantity you want:
-                SetHoldings(_quandlCode, 1);
-
-                //Debug sends messages to the user console: "Time" is the algorithm time keeper object
-                Debug("Purchased " + _quandlCode + " >> " + Time.ToShortDateString());
-            }
-
-            Plot("SPY", _sma);
-        }
-    }
-}

Algorithm.CSharp/QuantConnect.Algorithm.CSharp.csproj

@@ -35,7 +35,7 @@
     <DebugType>portable</DebugType>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Accord" Version="3.6.0" />
     <PackageReference Include="Accord.Fuzzy" Version="3.6.0" />
     <PackageReference Include="Accord.MachineLearning" Version="3.6.0" />

Algorithm.Framework/QuantConnect.Algorithm.Framework.csproj

@@ -30,7 +30,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Accord" Version="3.6.0" />
     <PackageReference Include="Accord.Math" Version="3.6.0" />
     <PackageReference Include="Accord.Statistics" Version="3.6.0" />

Algorithm.Python/BasicTemplateFuturesAlgorithm.py

@@ -43,6 +43,8 @@ class BasicTemplateFuturesAlgorithm(QCAlgorithm):
         benchmark = self.AddEquity("SPY")
         self.SetBenchmark(benchmark.Symbol)
 
+        seeder = FuncSecuritySeeder(self.GetLastKnownPrices)
+        self.SetSecurityInitializer(lambda security: seeder.SeedSecurity(security))
 
     def OnData(self,slice):
         if not self.Portfolio.Invested:
@@ -70,3 +72,8 @@ class BasicTemplateFuturesAlgorithm(QCAlgorithm):
         maintenanceOvernight = buyingPowerModel.MaintenanceOvernightMarginRequirement
         initialIntraday = buyingPowerModel.InitialIntradayMarginRequirement
         maintenanceIntraday = buyingPowerModel.MaintenanceIntradayMarginRequirement
+
+    def OnSecuritiesChanged(self, changes):
+        for addedSecurity in changes.AddedSecurities:
+            if addedSecurity.Symbol.SecurityType == SecurityType.Future and not addedSecurity.Symbol.IsCanonical() and not addedSecurity.HasData:
+                raise Exception(f"Future contracts did not work up as expected: {addedSecurity.Symbol}")

Algorithm.Python/BasicTemplateFuturesDailyAlgorithm.py

@@ -23,7 +23,7 @@ class BasicTemplateFuturesDailyAlgorithm(QCAlgorithm):
 
     def Initialize(self):
         self.SetStartDate(2013, 10, 8)
-        self.SetEndDate(2013, 10, 10)
+        self.SetEndDate(2014, 10, 10)
         self.SetCash(1000000)
 
         self.contractSymbol = None
@@ -50,6 +50,7 @@ class BasicTemplateFuturesDailyAlgorithm(QCAlgorithm):
                 front = sorted(contracts, key = lambda x: x.Expiry, reverse=True)[0]
 
                 self.contractSymbol = front.Symbol
-                self.MarketOrder(front.Symbol , 1)
+                if self.IsMarketOpen(self.contractSymbol):
+                    self.MarketOrder(front.Symbol , 1)
         else:
             self.Liquidate()

Algorithm.Python/BasicTemplateFuturesHourlyAlgorithm.py

@@ -23,7 +23,7 @@ class BasicTemplateFuturesHourlyAlgorithm(QCAlgorithm):
 
     def Initialize(self):
         self.SetStartDate(2013, 10, 8)
-        self.SetEndDate(2013, 10, 10)
+        self.SetEndDate(2014, 10, 10)
         self.SetCash(1000000)
 
         self.contractSymbol = None
@@ -50,6 +50,7 @@ class BasicTemplateFuturesHourlyAlgorithm(QCAlgorithm):
                 front = sorted(contracts, key = lambda x: x.Expiry, reverse=True)[0]
 
                 self.contractSymbol = front.Symbol
-                self.MarketOrder(front.Symbol , 1)
+                if self.IsMarketOpen(self.contractSymbol):
+                    self.MarketOrder(front.Symbol , 1)
         else:
             self.Liquidate()

Algorithm.Python/BasicTemplateIndiaAlgorithm.py

@@ -0,0 +1,50 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+### <summary>
+### Basic template framework algorithm uses framework components to define the algorithm.
+### </summary>
+### <meta name="tag" content="using data" />
+### <meta name="tag" content="using quantconnect" />
+### <meta name="tag" content="trading and orders" />
+class BasicTemplateIndiaAlgorithm(QCAlgorithm):
+    '''Basic template framework algorithm uses framework components to define the algorithm.'''
+
+    def Initialize(self):
+        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
+
+        self.SetAccountCurrency("INR")  #Set Account Currency 
+        self.SetStartDate(2019, 1, 23)  #Set Start Date
+        self.SetEndDate(2019, 10, 31)   #Set End Date
+        self.SetCash(100000)            #Set Strategy Cash
+        # Find more symbols here: http://quantconnect.com/data
+        self.AddEquity("YESBANK", Resolution.Minute, Market.India)
+        self.Debug("numpy test >>> print numpy.pi: " + str(np.pi))
+
+        # Set Order Prperties as per the requirements for order placement
+        self.DefaultOrderProperties = IndiaOrderProperties(Exchange.NSE)
+
+    def OnData(self, data):
+        '''OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
+
+        Arguments:
+            data: Slice object keyed by symbol containing the stock data
+        '''
+        if not self.Portfolio.Invested:
+            self.MarketOrder("YESBANK", 1)
+
+    def OnOrderEvent(self, orderEvent):
+        if orderEvent.Status == OrderStatus.Filled:
+            self.Debug("Purchased Stock: {0}".format(orderEvent.Symbol))

Algorithm.Python/BasicTemplateIndiaIndexAlgorithm.py

@@ -0,0 +1,71 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+### <summary>
+### Basic Template India Index Algorithm uses framework components to define the algorithm.
+### </summary>
+### <meta name="tag" content="using data" />
+### <meta name="tag" content="using quantconnect" />
+### <meta name="tag" content="trading and orders" />
+class BasicTemplateIndiaIndexAlgorithm(QCAlgorithm):
+    '''Basic template framework algorithm uses framework components to define the algorithm.'''
+
+    def Initialize(self):
+        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
+
+        self.SetAccountCurrency("INR") #Set Account Currency
+        self.SetStartDate(2019, 1, 1)  #Set Start Date
+        self.SetEndDate(2019, 1, 5)    #Set End Date
+        self.SetCash(1000000)          #Set Strategy Cash
+
+        # Use indicator for signal; but it cannot be traded
+        self.Nifty = self.AddIndex("NIFTY50", Resolution.Minute, Market.India).Symbol
+        # Trade Index based ETF
+        self.NiftyETF = self.AddEquity("JUNIORBEES", Resolution.Minute, Market.India).Symbol
+   
+        # Set Order Prperties as per the requirements for order placement
+        self.DefaultOrderProperties = IndiaOrderProperties(Exchange.NSE)
+
+        # Define indicator
+        self._emaSlow = self.EMA(self.Nifty, 80);
+        self._emaFast = self.EMA(self.Nifty, 200);
+
+        self.Debug("numpy test >>> print numpy.pi: " + str(np.pi))
+
+
+    def OnData(self, data):
+        '''OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
+
+        Arguments:
+            data: Slice object keyed by symbol containing the stock data
+        '''
+
+        if not data.Bars.ContainsKey(self.Nifty) or not data.Bars.ContainsKey(self.NiftyETF):
+            return
+
+        if not self._emaSlow.IsReady:
+            return
+
+        if self._emaFast > self._emaSlow:
+            if not self.Portfolio.Invested:
+                self.marketTicket = self.MarketOrder(self.NiftyETF, 1)
+        else:
+            self.Liquidate()
+
+
+    def OnEndOfAlgorithm(self):
+        if self.Portfolio[self.Nifty].TotalSaleVolume > 0:
+            raise Exception("Index is not tradable.")
+

Algorithm.Python/BasicTemplateOptionsDailyAlgorithm.py

@@ -51,7 +51,7 @@ class BasicTemplateOptionsDailyAlgorithm(QCAlgorithm):
         contracts = sorted(chain, key = lambda x: x.Expiry)
 
         # if found, trade it
-        if len(contracts) == 0: return
+        if len(contracts) == 0 or not self.IsMarketOpen(contracts[0].Symbol): return
         symbol = contracts[0].Symbol
         self.MarketOrder(symbol, 1)
 

Algorithm.Python/BasicTemplateOptionsHourlyAlgorithm.py

@@ -57,7 +57,7 @@ class BasicTemplateOptionsHourlyAlgorithm(QCAlgorithm):
             key = lambda x: x.Right, reverse=True)
 
         # if found, trade it
-        if len(contracts) == 0: return
+        if len(contracts) == 0 or not self.IsMarketOpen(contracts[0].Symbol): return
         symbol = contracts[0].Symbol
         self.MarketOrder(symbol, 1)
         self.MarketOnCloseOrder(symbol, -1)

Algorithm.Python/CustomDataIndicatorExtensionsAlgorithm.py

@@ -12,10 +12,11 @@
 # limitations under the License.
 
 from AlgorithmImports import *
+from HistoryAlgorithm import *
 
 ### <summary>
 ### The algorithm creates new indicator value with the existing indicator method by Indicator Extensions
-### Demonstration of using the external custom datasource Quandl to request the VIX and VXV daily data
+### Demonstration of using the external custom data to request the IBM and SPY daily data
 ### </summary>
 ### <meta name="tag" content="using data" />
 ### <meta name="tag" content="using quantconnect" />
@@ -33,38 +34,30 @@ class CustomDataIndicatorExtensionsAlgorithm(QCAlgorithm):
         self.SetEndDate(2018,1,1)  
         self.SetCash(25000)
         
-        self.vix = 'CBOE/VIX'
-        self.vxv = 'CBOE/VXV'
+        self.ibm = 'IBM'
+        self.spy = 'SPY'
         
         # Define the symbol and "type" of our generic data
-        self.AddData(QuandlVix, self.vix, Resolution.Daily)
-        self.AddData(Quandl, self.vxv, Resolution.Daily)
+        self.AddData(CustomDataEquity, self.ibm, Resolution.Daily)
+        self.AddData(CustomDataEquity, self.spy, Resolution.Daily)
         
         # Set up default Indicators, these are just 'identities' of the closing price
-        self.vix_sma = self.SMA(self.vix, 1, Resolution.Daily)
-        self.vxv_sma = self.SMA(self.vxv, 1, Resolution.Daily)
+        self.ibm_sma = self.SMA(self.ibm, 1, Resolution.Daily)
+        self.spy_sma = self.SMA(self.spy, 1, Resolution.Daily)
         
-        # This will create a new indicator whose value is smaVXV / smaVIX
-        self.ratio = IndicatorExtensions.Over(self.vxv_sma, self.vix_sma)
+        # This will create a new indicator whose value is smaSPY / smaIBM
+        self.ratio = IndicatorExtensions.Over(self.spy_sma, self.ibm_sma)
         
         # Plot indicators each time they update using the PlotIndicator function
         self.PlotIndicator("Ratio", self.ratio)
-        self.PlotIndicator("Data", self.vix_sma, self.vxv_sma)
+        self.PlotIndicator("Data", self.ibm_sma, self.spy_sma)
     
     # OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
     def OnData(self, data):
         
         # Wait for all indicators to fully initialize
-        if not (self.vix_sma.IsReady and self.vxv_sma.IsReady and self.ratio.IsReady): return
+        if not (self.ibm_sma.IsReady and self.spy_sma.IsReady and self.ratio.IsReady): return
         if not self.Portfolio.Invested and self.ratio.Current.Value > 1:
-            self.MarketOrder(self.vix, 100)
+            self.MarketOrder(self.ibm, 100)
         elif self.ratio.Current.Value < 1:
                 self.Liquidate()
-
-# In CBOE/VIX data, there is a "vix close" column instead of "close" which is the 
-# default column namein LEAN Quandl custom data implementation.
-# This class assigns new column name to match the the external datasource setting.
-class QuandlVix(PythonQuandl):
-    
-    def __init__(self):
-        self.ValueColumnName = "VIX Close"

Algorithm.Python/CustomDataRegressionAlgorithm.py

@@ -25,18 +25,33 @@ class CustomDataRegressionAlgorithm(QCAlgorithm):
 
     def Initialize(self):
 
-        self.SetStartDate(2011,9,13)   # Set Start Date
+        self.SetStartDate(2011,9,14)   # Set Start Date
         self.SetEndDate(2015,12,1)     # Set End Date
         self.SetCash(100000)           # Set Strategy Cash
 
         resolution = Resolution.Second if self.LiveMode else Resolution.Daily
         self.AddData(Bitcoin, "BTC", resolution)
 
+        seeder = FuncSecuritySeeder(self.GetLastKnownPrices)
+        self.SetSecurityInitializer(lambda x: seeder.SeedSecurity(x))
+        self._warmedUpChecked = False
+
     def OnData(self, data):
         if not self.Portfolio.Invested:
             if data['BTC'].Close != 0 :
                 self.Order('BTC', self.Portfolio.MarginRemaining/abs(data['BTC'].Close + 1))
 
+    def OnSecuritiesChanged(self, changes):
+        changes.FilterCustomSecurities = False
+        for addedSecurity in changes.AddedSecurities:
+            if addedSecurity.Symbol.Value == "BTC":
+                self._warmedUpChecked = True
+            if not addedSecurity.HasData:
+                raise ValueError(f"Security {addedSecurity.Symbol} was not warmed up!")
+
+    def OnEndOfAlgorithm(self):
+        if not self._warmedUpChecked:
+            raise ValueError("Security was not warmed up!")
 
 class Bitcoin(PythonData):
     '''Custom Data Type: Bitcoin data from Quandl - http://www.quandl.com/help/api-for-bitcoin-data'''

Algorithm.Python/HistoryAlgorithm.py

@@ -30,12 +30,11 @@ class HistoryAlgorithm(QCAlgorithm):
         self.SetCash(100000)           #Set Strategy Cash
         # Find more symbols here: http://quantconnect.com/data
         self.AddEquity("SPY", Resolution.Daily)
-        self.AddData(QuandlFuture,"CHRIS/CME_SP1", Resolution.Daily)
+        self.AddData(CustomDataEquity, "IBM", Resolution.Daily)
         # specifying the exchange will allow the history methods that accept a number of bars to return to work properly
-        self.Securities["CHRIS/CME_SP1"].Exchange = EquityExchange()
 
         # we can get history in initialize to set up indicators and such
-        self.spyDailySma = SimpleMovingAverage(14)
+        self.dailySma = SimpleMovingAverage(14)
 
         # get the last calendar year's worth of SPY data at the configured resolution (daily)
         tradeBarHistory = self.History([self.Securities["SPY"].Symbol], timedelta(365))
@@ -56,56 +55,52 @@ class HistoryAlgorithm(QCAlgorithm):
         # we can loop over the return value from these functions and we get TradeBars
         # we can use these TradeBars to initialize indicators or perform other math
         for index, tradeBar in tradeBarHistory.loc["SPY"].iterrows():
-            self.spyDailySma.Update(index, tradeBar["close"])
+            self.dailySma.Update(index, tradeBar["close"])
 
-        # get the last calendar year's worth of quandl data at the configured resolution (daily)
-        quandlHistory = self.History(QuandlFuture, "CHRIS/CME_SP1", timedelta(365))
-        self.AssertHistoryCount("History(QuandlFuture, \"CHRIS/CME_SP1\", timedelta(365))", quandlHistory, 250)
+        # get the last calendar year's worth of customData data at the configured resolution (daily)
+        customDataHistory = self.History(CustomDataEquity, "IBM", timedelta(365))
+        self.AssertHistoryCount("History(CustomDataEquity, \"IBM\", timedelta(365))", customDataHistory, 10)
 
-        # get the last 14 bars of SPY at the configured resolution (daily)
-        quandlHistory = self.History(QuandlFuture, "CHRIS/CME_SP1", 14)
-        self.AssertHistoryCount("History(QuandlFuture, \"CHRIS/CME_SP1\", 14)", quandlHistory, 14)
+        # get the last 10 bars of IBM at the configured resolution (daily)
+        customDataHistory = self.History(CustomDataEquity, "IBM", 14)
+        self.AssertHistoryCount("History(CustomDataEquity, \"IBM\", 14)", customDataHistory, 10)
 
-        # we can loop over the return values from these functions and we'll get Quandl data
+        # we can loop over the return values from these functions and we'll get Custom data
         # this can be used in much the same way as the tradeBarHistory above
-        self.spyDailySma.Reset()
-        for index, quandl in quandlHistory.loc["CHRIS/CME_SP1"].iterrows():
-            self.spyDailySma.Update(index, quandl["settle"])
+        self.dailySma.Reset()
+        for index, customData in customDataHistory.loc["IBM"].iterrows():
+            self.dailySma.Update(index, customData["value"])
 
-        # get the last year's worth of all configured Quandl data at the configured resolution (daily)
-        #allQuandlData = self.History(QuandlFuture, timedelta(365))
-        #self.AssertHistoryCount("History(QuandlFuture, timedelta(365))", allQuandlData, 250)
+        # get the last 10 bars worth of Custom data for the specified symbols at the configured resolution (daily)
+        allCustomData = self.History(CustomDataEquity, self.Securities.Keys, 14)
+        self.AssertHistoryCount("History(CustomDataEquity, self.Securities.Keys, 14)", allCustomData, 10)
 
-        # get the last 14 bars worth of Quandl data for the specified symbols at the configured resolution (daily)
-        allQuandlData = self.History(QuandlFuture, self.Securities.Keys, 14)
-        self.AssertHistoryCount("History(QuandlFuture, self.Securities.Keys, 14)", allQuandlData, 14)
+        # NOTE: Using different resolutions require that they are properly implemented in your data type. If your
+        #  custom data source has different resolutions, it would need to be implemented in the GetSource and 
+        #  Reader methods properly.
+        #customDataHistory = self.History(CustomDataEquity, "IBM", timedelta(7), Resolution.Minute)
+        #customDataHistory = self.History(CustomDataEquity, "IBM", 14, Resolution.Minute)
+        #allCustomData = self.History(CustomDataEquity, timedelta(365), Resolution.Minute)
+        #allCustomData = self.History(CustomDataEquity, self.Securities.Keys, 14, Resolution.Minute)
+        #allCustomData = self.History(CustomDataEquity, self.Securities.Keys, timedelta(1), Resolution.Minute)
+        #allCustomData = self.History(CustomDataEquity, self.Securities.Keys, 14, Resolution.Minute)
 
-        # NOTE: using different resolutions require that they are properly implemented in your data type, since
-        #  Quandl doesn't support minute data, this won't actually work, but if your custom data source has
-        #  different resolutions, it would need to be implemented in the GetSource and Reader methods properly
-        #quandlHistory = self.History(QuandlFuture, "CHRIS/CME_SP1", timedelta(7), Resolution.Minute)
-        #quandlHistory = self.History(QuandlFuture, "CHRIS/CME_SP1", 14, Resolution.Minute)
-        #allQuandlData = self.History(QuandlFuture, timedelta(365), Resolution.Minute)
-        #allQuandlData = self.History(QuandlFuture, self.Securities.Keys, 14, Resolution.Minute)
-        #allQuandlData = self.History(QuandlFuture, self.Securities.Keys, timedelta(1), Resolution.Minute)
-        #allQuandlData = self.History(QuandlFuture, self.Securities.Keys, 14, Resolution.Minute)
-
-        # get the last calendar year's worth of all quandl data
-        allQuandlData = self.History(QuandlFuture, self.Securities.Keys, timedelta(365))
-        self.AssertHistoryCount("History(QuandlFuture, self.Securities.Keys, timedelta(365))", allQuandlData, 250)
+        # get the last calendar year's worth of all customData data
+        allCustomData = self.History(CustomDataEquity, self.Securities.Keys, timedelta(365))
+        self.AssertHistoryCount("History(CustomDataEquity, self.Securities.Keys, timedelta(365))", allCustomData, 10)
 
         # we can also access the return value from the multiple symbol functions to request a single
         # symbol and then loop over it
-        singleSymbolQuandl = allQuandlData.loc["CHRIS/CME_SP1"]
-        self.AssertHistoryCount("allQuandlData.loc[\"CHRIS/CME_SP1\"]", singleSymbolQuandl, 250)
-        for  quandl in singleSymbolQuandl:
-            # do something with 'CHRIS/CME_SP1.QuandlFuture' quandl data
+        singleSymbolCustom = allCustomData.loc["IBM"]
+        self.AssertHistoryCount("allCustomData.loc[\"IBM\"]", singleSymbolCustom, 10)
+        for  customData in singleSymbolCustom:
+            # do something with 'IBM.CustomDataEquity' customData data
             pass
 
-        quandlSpyLows = allQuandlData.loc["CHRIS/CME_SP1"]["low"]
-        self.AssertHistoryCount("allQuandlData.loc[\"CHRIS/CME_SP1\"][\"low\"]", quandlSpyLows, 250)
-        for  low in quandlSpyLows:
-            # do something with 'CHRIS/CME_SP1.QuandlFuture' quandl data
+        customDataSpyValues = allCustomData.loc["IBM"]["value"]
+        self.AssertHistoryCount("allCustomData.loc[\"IBM\"][\"value\"]", customDataSpyValues, 10)
+        for value in customDataSpyValues:
+            # do something with 'IBM.CustomDataEquity' value data
             pass
 
 
@@ -124,10 +119,20 @@ class HistoryAlgorithm(QCAlgorithm):
             raise Exception("{} expected {}, but received {}".format(methodCall, expected, count))
 
 
-class QuandlFuture(PythonQuandl):
-    '''Custom quandl data type for setting customized value column name. Value column is used for the primary trading calculations and charting.'''
-    def __init__(self):
-        # Define ValueColumnName: cannot be None, Empty or non-existant column name
-        # If ValueColumnName is "Close", do not use PythonQuandl, use Quandl:
-        # self.AddData[QuandlFuture](self.crude, Resolution.Daily)
-        self.ValueColumnName = "Settle"
+class CustomDataEquity(PythonData):
+    def GetSource(self, config, date, isLive):
+        source = "https://www.dl.dropboxusercontent.com/s/o6ili2svndzn556/custom_data.csv?dl=0"
+        return SubscriptionDataSource(source, SubscriptionTransportMedium.RemoteFile)
+
+    def Reader(self, config, line, date, isLive):
+        if line == None:
+            return None
+
+        customData = CustomDataEquity()
+        customData.Symbol = config.Symbol
+
+        csv = line.split(",")
+        customData.Time = datetime.strptime(csv[0], '%Y%m%d %H:%M')
+        customData.EndTime = customData.Time + timedelta(days=1)
+        customData.Value = float(csv[1])
+        return customData

Algorithm.Python/IndiaDataRegressionAlgorithm.py

@@ -0,0 +1,83 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+### <summary>
+### Basic template framework algorithm uses framework components to define the algorithm.
+### </summary>
+### <meta name="tag" content="using data" />
+### <meta name="tag" content="using quantconnect" />
+### <meta name="tag" content="trading and orders" />
+class IndiaDataRegressionAlgorithm(QCAlgorithm):
+    '''Basic template framework algorithm uses framework components to define the algorithm.'''
+
+    def Initialize(self):
+        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
+        
+        self.SetAccountCurrency("INR")
+        self.SetStartDate(2004, 5, 20)
+        self.SetEndDate(2016, 7, 26) 
+        self._mappingSymbol = self.AddEquity("3MINDIA", Resolution.Daily, Market.India).Symbol
+        self._splitAndDividendSymbol = self.AddEquity("CCCL", Resolution.Daily, Market.India).Symbol
+        self._receivedWarningEvent = False
+        self._receivedOccurredEvent = False
+        self._initialMapping = False
+        self._executionMapping = False
+        self.Debug("numpy test >>> print numpy.pi: " + str(np.pi))
+
+    def OnData(self, data):
+        '''OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
+
+        Arguments:
+            data: Slice object keyed by symbol containing the stock data
+        '''
+
+        # dividend
+        if data.Dividends.ContainsKey(self._splitAndDividendSymbol):
+            dividend = data.Dividends[self._splitAndDividendSymbol]
+            if ((self.Time.year == 2010 and self.Time.month == 6 and self.Time.day == 15) and
+                    (dividend.Price != 0.5 or dividend.ReferencePrice != 88.8 or dividend.Distribution != 0.5)):
+                raise Exception("Did not receive expected dividend values")
+
+        # split
+        if data.Splits.ContainsKey(self._splitAndDividendSymbol):
+            split = data.Splits[self._splitAndDividendSymbol]
+            if split.Type == SplitType.Warning:
+                self._receivedWarningEvent = True
+            elif split.Type == SplitType.SplitOccurred:
+                self._receivedOccurredEvent = True
+                if split.Price != 421.0 or split.ReferencePrice != 421.0 or split.SplitFactor != 0.2:
+                    raise Exception("Did not receive expected price values")
+
+        # mapping
+        if data.SymbolChangedEvents.ContainsKey(self._mappingSymbol):
+                mappingEvent = [x.Value for x in data.SymbolChangedEvents if x.Key.SecurityType == 1][0]
+                if self.Time.year == 1999 and self.Time.month == 1 and self.Time.day == 1:
+                    self._initialMapping = True
+                elif self.Time.year == 2004 and self.Time.month == 6 and self.Time.day == 15:
+                    if mappingEvent.NewSymbol == "3MINDIA" and mappingEvent.OldSymbol == "BIRLA3M":
+                        self._executionMapping = True
+    
+    def OnEndOfAlgorithm(self):
+        if self._initialMapping:
+            raise Exception("The ticker generated the initial rename event")
+
+        if not self._executionMapping:
+            raise Exception("The ticker did not rename throughout the course of its life even though it should have")
+        
+        if not self._receivedOccurredEvent:
+            raise Exception("Did not receive expected split event")
+
+        if not self._receivedWarningEvent:
+            raise Exception("Did not receive expected split warning event")

Algorithm.Python/IndicatorSuiteAlgorithm.py

@@ -12,6 +12,7 @@
 # limitations under the License.
 
 from AlgorithmImports import *
+from QuantConnect.Algorithm.CSharp import *
 
 ### <summary>
 ### Basic template algorithm simply initializes the date range and cash. This is a skeleton
@@ -27,7 +28,7 @@ class IndicatorSuiteAlgorithm(QCAlgorithm):
         '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
 
         self.symbol = "SPY"
-        self.customSymbol = "WIKI/FB"
+        self.customSymbol = "IBM"
         self.price = None
 
         self.SetStartDate(2013, 1, 1)  #Set Start Date
@@ -36,7 +37,7 @@ class IndicatorSuiteAlgorithm(QCAlgorithm):
         # Find more symbols here: http://quantconnect.com/data
 
         self.AddEquity(self.symbol, Resolution.Daily)
-        self.AddData(Quandl, self.customSymbol, Resolution.Daily)
+        self.AddData(CustomData, self.customSymbol, Resolution.Daily)
 
         # Set up default Indicators, these indicators are defined on the Value property of incoming data (except ATR and AROON which use the full TradeBar object)
         self.indicators = {
@@ -88,10 +89,10 @@ class IndicatorSuiteAlgorithm(QCAlgorithm):
         # these are indicators that require multiple inputs. the most common of which is a ratio.
         # suppose we seek the ratio of BTC to SPY, we could write the following:
         spyClose = Identity(self.symbol)
-        fbClose = Identity(self.customSymbol)
+        ibmClose = Identity(self.customSymbol)
 
-        # this will create a new indicator whose value is FB/SPY
-        self.ratio = IndicatorExtensions.Over(fbClose, spyClose)
+        # this will create a new indicator whose value is IBM/SPY
+        self.ratio = IndicatorExtensions.Over(ibmClose, spyClose)
 
         # we can also easily plot our indicators each time they update using th PlotIndicator function
         self.PlotIndicator("Ratio", self.ratio)

Algorithm.Python/KerasNeuralNetworkAlgorithm.py

@@ -13,11 +13,10 @@
 
 from AlgorithmImports import *
 
-from io import StringIO
-from keras.models import Sequential
+from keras.models import *
+from tensorflow import keras
 from keras.layers import Dense, Activation
 from keras.optimizers import SGD
-from keras.utils.generic_utils import serialize_keras_object
 
 class KerasNeuralNetworkAlgorithm(QCAlgorithm):
 
@@ -32,11 +31,13 @@ class KerasNeuralNetworkAlgorithm(QCAlgorithm):
             symbol = self.AddEquity(ticker).Symbol
 
             # Read the model saved in the ObjectStore
-            if self.ObjectStore.ContainsKey(f'{symbol}_model'):
-                modelStr = self.ObjectStore.Read(f'{symbol}_model')
-                config = json.loads(modelStr)['config']
-                self.modelBySymbol[symbol] = Sequential.from_config(config)
-                self.Debug(f'Model for {symbol} sucessfully retrieved from the ObjectStore')
+            for kvp in self.ObjectStore:
+                key = f'{symbol}_model'
+                if not key == kvp.Key or kvp.Value is None:
+                    continue
+                filePath = self.ObjectStore.GetFilePath(kvp.Key)
+                self.modelBySymbol[symbol] = keras.models.load_model(filePath)
+                self.Debug(f'Model for {symbol} sucessfully retrieved. File {filePath}. Size {kvp.Value.Length}. Weights {self.modelBySymbol[symbol].get_weights()}')
 
         # Look-back period for training set
         self.lookback = 30
@@ -51,19 +52,21 @@ class KerasNeuralNetworkAlgorithm(QCAlgorithm):
         self.Schedule.On(
             self.DateRules.EveryDay("SPY"),
             self.TimeRules.AfterMarketOpen("SPY", 30),
-            self.Trade) 
+            self.Trade)
 
 
     def OnEndOfAlgorithm(self):
         ''' Save the data and the mode using the ObjectStore '''
         for symbol, model in self.modelBySymbol.items():
-            modelStr = json.dumps(serialize_keras_object(model))
-            self.ObjectStore.Save(f'{symbol}_model', modelStr)
+            key = f'{symbol}_model'
+            file = self.ObjectStore.GetFilePath(key)
+            model.save(file)
+            self.ObjectStore.Save(key)
             self.Debug(f'Model for {symbol} sucessfully saved in the ObjectStore')
 
 
     def NeuralNetworkTraining(self):
-        '''Train the Neural Network and save the model in the ObjectStore'''        
+        '''Train the Neural Network and save the model in the ObjectStore'''
         symbols = self.Securities.keys()
 
         # Daily historical data is used to train the machine learning model
@@ -89,19 +92,18 @@ class KerasNeuralNetworkAlgorithm(QCAlgorithm):
             # choose loss function and optimizing method
             model.compile(loss='mse', optimizer=sgd)
 
-            # pick an iteration number large enough for convergence 
+            # pick an iteration number large enough for convergence
             for step in range(200):
                 # training the model
                 cost = model.train_on_batch(predictor, predictand)
 
             self.modelBySymbol[symbol] = model
 
-
     def Trade(self):
         '''
         Predict the price using the trained model and out-of-sample data
         Enter or exit positions based on relationship of the open price of the current bar and the prices defined by the machine learning model.
-        Liquidate if the open price is below the sell price and buy if the open price is above the buy price 
+        Liquidate if the open price is below the sell price and buy if the open price is above the buy price
         '''
         target = 1 / len(self.Securities)
 

Algorithm.Python/QuandlFuturesDataAlgorithm.py

@@ -1,53 +0,0 @@
-# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
-# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from AlgorithmImports import *
-
-### <summary>
-### Futures demonstration algorithm.
-### QuantConnect allows importing generic data sources! This example demonstrates importing a futures
-### data from the popular open data source Quandl. QuantConnect has a special deal with Quandl giving you access
-### to Stevens Continuous Futurs (SCF) for free. If you'd like to download SCF for local backtesting, you can download it through Quandl.com.
-### </summary>
-### <meta name="tag" content="using data" />
-### <meta name="tag" content="quandl" />
-### <meta name="tag" content="custom data" />
-### <meta name="tag" content="futures" />
-class QuandlFuturesDataAlgorithm(QCAlgorithm):
-
-    def Initialize(self):
-        ''' Initialize the data and resolution you require for your strategy '''
-        self.SetStartDate(2000, 1, 1)
-        self.SetEndDate(datetime.now().date() - timedelta(1))
-        self.SetCash(25000)
-
-        # Symbol corresponding to the quandl code
-        self.crude = "SCF/CME_CL1_ON"
-        self.AddData(QuandlFuture, self.crude, Resolution.Daily)
-
-
-    def OnData(self, data):
-        '''Data Event Handler: New data arrives here. "TradeBars" type is a dictionary of strings so you can access it by symbol.'''
-        if self.Portfolio.HoldStock: return
-
-        self.SetHoldings(self.crude, 1)
-        self.Debug(str(self.Time) + str(" Purchased Crude Oil: ") + self.crude)
-
-
-class QuandlFuture(PythonQuandl):
-    '''Custom quandl data type for setting customized value column name. Value column is used for the primary trading calculations and charting.'''
-    def __init__(self):
-        # Define ValueColumnName: cannot be None, Empty or non-existant column name
-        # If ValueColumnName is "Close", do not use PythonQuandl, use Quandl:
-        # self.AddData[QuandlFuture](self.crude, Resolution.Daily)
-        self.ValueColumnName = "Settle"

Algorithm.Python/QuandlImporterAlgorithm.py

@@ -1,51 +0,0 @@
-# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
-# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from AlgorithmImports import *
-
-### <summary>
-### Using the underlying dynamic data class "Quandl" QuantConnect take care of the data
-### importing and definition for you. Simply point QuantConnect to the Quandl Short Code.
-### The Quandl object has properties which match the spreadsheet headers.
-### If you have multiple quandl streams look at data.Symbol to distinguish them.
-### </summary>
-### <meta name="tag" content="custom data" />
-### <meta name="tag" content="using data" />
-### <meta name="tag" content="quandl" />
-class QuandlImporterAlgorithm(QCAlgorithm):
-
-    def Initialize(self):
-        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
-        self.quandlCode = "WIKI/IBM"
-        ## Optional argument - personal token necessary for restricted dataset
-        # Quandl.SetAuthCode("your-quandl-token")
-        self.SetStartDate(2014,4,1)                                 #Set Start Date
-        self.SetEndDate(datetime.today() - timedelta(1))            #Set End Date
-        self.SetCash(25000)                                         #Set Strategy Cash
-        self.AddData(QuandlCustomColumns, self.quandlCode, Resolution.Daily, TimeZones.NewYork)
-        self.sma = self.SMA(self.quandlCode, 14)
-
-    def OnData(self, data):
-        '''OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.'''
-        if not self.Portfolio.HoldStock:
-            self.SetHoldings(self.quandlCode, 1)
-            self.Debug("Purchased {0} >> {1}".format(self.quandlCode, self.Time))
-
-        self.Plot(self.quandlCode, "PriceSMA", self.sma.Current.Value)
-        
-# Quandl often doesn't use close columns so need to tell LEAN which is the "value" column.
-class QuandlCustomColumns(PythonQuandl):
-    '''Custom quandl data type for setting customized value column name. Value column is used for the primary trading calculations and charting.'''
-    def __init__(self):
-        # Define ValueColumnName: cannot be None, Empty or non-existant column name
-        self.ValueColumnName = "adj. close"

Algorithm.Python/QuantConnect.Algorithm.Python.csproj

@@ -37,7 +37,7 @@
     <Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
@@ -222,8 +222,6 @@
     <None Include="OptionSplitRegressionAlgorithm.py" />
     <None Include="OrderTicketDemoAlgorithm.py" />
     <None Include="ParameterizedAlgorithm.py" />
-    <None Include="QuandlFuturesDataAlgorithm.py" />
-    <None Include="QuandlImporterAlgorithm.py" />
     <None Include="readme.md" />
     <None Include="RawPricesCoarseUniverseAlgorithm.py" />
     <None Include="RegressionAlgorithm.py" />

Algorithm/QCAlgorithm.History.cs

@@ -513,16 +513,18 @@ namespace QuantConnect.Algorithm
             }
 
             var result = new Dictionary<TickType, BaseData>();
-            // For speed and memory usage, use Resolution.Minute as the minimum resolution
-            var resolution = (Resolution)Math.Max((int)Resolution.Minute,
-                (int)SubscriptionManager.SubscriptionDataConfigService.GetSubscriptionDataConfigs(symbol).GetHighestResolution());
+            Resolution? resolution = null;
             Func<int, bool> requestData = period =>
             {
-                var historyRequests = CreateBarCountHistoryRequests(new[] { symbol }, period, resolution)
+                var historyRequests = CreateBarCountHistoryRequests(new[] { symbol }, period)
                     .Select(request =>
                     {
+                        // For speed and memory usage, use Resolution.Minute as the minimum resolution
+                        request.Resolution = (Resolution)Math.Max((int)Resolution.Minute, (int)request.Resolution);
                         // force no fill forward behavior
                         request.FillForwardResolution = null;
+
+                        resolution = request.Resolution;
                         return request;
                     })
                     // request only those tick types we didn't get the data we wanted
@@ -547,12 +549,21 @@ namespace QuantConnect.Algorithm
 
             if (!requestData(5))
             {
-                // If the first attempt to get the last know price returns null, it maybe the case of an illiquid security.
-                // We increase the look-back period for this case accordingly to the resolution to cover 3 trading days
-                var periods =
-                    resolution == Resolution.Daily ? 3 :
-                    resolution == Resolution.Hour ? 24 : 1440;
-                requestData(periods);
+                if (resolution.HasValue)
+                {
+                    // If the first attempt to get the last know price returns null, it maybe the case of an illiquid security.
+                    // We increase the look-back period for this case accordingly to the resolution to cover 3 trading days
+                    var periods =
+                        resolution.Value == Resolution.Daily ? 3 :
+                        resolution.Value == Resolution.Hour ? 24 : 1440;
+                    requestData(periods);
+                }
+                else
+                {
+                    // this shouldn't happen but just in case
+                    QuantConnect.Logging.Log.Error(
+                        $"QCAlgorithm.GetLastKnownPrices(): no history request was created for symbol {symbol} at {Time}");
+                }
             }
             // return the data ordered by time ascending
             return result.Values.OrderBy(data => data.Time);
@@ -804,9 +815,30 @@ namespace QuantConnect.Algorithm
             Security security;
             if (Securities.TryGetValue(symbol, out security))
             {
-                return resolution ?? SubscriptionManager.SubscriptionDataConfigService
-                    .GetSubscriptionDataConfigs(symbol)
-                    .GetHighestResolution();
+                if (resolution != null)
+                {
+                    return resolution.Value;
+                }
+
+                Resolution? result = null;
+                var hasNonInternal = false;
+                foreach (var config in SubscriptionManager.SubscriptionDataConfigService
+                    .GetSubscriptionDataConfigs(symbol, includeInternalConfigs: true)
+                    // we process non internal configs first
+                    .OrderBy(config => config.IsInternalFeed ? 1 : 0))
+                {
+                    if (!config.IsInternalFeed || !hasNonInternal)
+                    {
+                        // once we find a non internal config we ignore internals
+                        hasNonInternal |= !config.IsInternalFeed;
+                        if (!result.HasValue || config.Resolution < result)
+                        {
+                            result = config.Resolution;
+                        }
+                    }
+                }
+
+                return result ?? UniverseSettings.Resolution;
             }
             else
             {

Algorithm/QCAlgorithm.Trading.cs

@@ -1269,13 +1269,11 @@ namespace QuantConnect.Algorithm
         [DocumentationAttribute(SecuritiesAndPortfolio)]
         public bool IsMarketOpen(Symbol symbol)
         {
-            var exchangeHours = MarketHoursDatabase
-                .FromDataFolder()
-                .GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
-
-            var time = UtcTime.ConvertFromUtc(exchangeHours.TimeZone);
-
-            return exchangeHours.IsOpen(time, false);
+            if (Securities.TryGetValue(symbol, out var security))
+            {
+                return security.IsMarketOpen(false);
+            }
+            return symbol.IsMarketOpen(UtcTime, false);
         }
 
         private SubmitOrderRequest CreateSubmitOrderRequest(OrderType orderType, Security security, decimal quantity, string tag, IOrderProperties properties, decimal stopPrice = 0m, decimal limitPrice = 0m,  decimal triggerPrice = 0m)

Algorithm/QCAlgorithm.Universe.cs

@@ -657,27 +657,9 @@ namespace QuantConnect.Algorithm
                 bar.Symbol = security.Symbol;
                 
                 var maxSupportedResolution = bar.SupportedResolutions().Max();
-
                 var updateFrequency = maxSupportedResolution.ToTimeSpan();
-                int periods;
-                switch (maxSupportedResolution)
-                {
-                    case Resolution.Tick:
-                    case Resolution.Second:
-                        periods = 600;
-                        break;
-                    case Resolution.Minute:
-                        periods = 60 * 24;
-                        break;
-                    case Resolution.Hour:
-                        periods = 24 * 30;
-                        break;
-                    default:
-                        periods = 30;
-                        break;
-                }
                 
-                security.VolatilityModel = new StandardDeviationOfReturnsVolatilityModel(periods, maxSupportedResolution, updateFrequency);
+                security.VolatilityModel = new StandardDeviationOfReturnsVolatilityModel(maxSupportedResolution, updateFrequency);
             }
         }
 

Algorithm/QCAlgorithm.cs

@@ -1680,6 +1680,12 @@ namespace QuantConnect.Algorithm
                             contractDepthOffset: contractOffset
                         );
 
+                        // let's add a MHDB entry for the continuous symbol using the associated security
+                        var continuousContractSymbol = ContinuousContractUniverse.CreateSymbol(security.Symbol);
+                        MarketHoursDatabase.SetEntry(continuousContractSymbol.ID.Market,
+                            continuousContractSymbol.ID.Symbol,
+                            continuousContractSymbol.ID.SecurityType,
+                            security.Exchange.Hours);
                         AddUniverse(new ContinuousContractUniverse(security, new UniverseSettings(settings)
                             {
                                 DataMappingMode = continuousConfigs.First().DataMappingMode,
@@ -1687,7 +1693,7 @@ namespace QuantConnect.Algorithm
                                 ContractDepthOffset = (int)continuousConfigs.First().ContractDepthOffset,
                                 SubscriptionDataTypes = dataTypes
                             }, LiveMode,
-                            new SubscriptionDataConfig(canonicalConfig, symbol: ContinuousContractUniverse.CreateSymbol(security.Symbol))));
+                            new SubscriptionDataConfig(canonicalConfig, symbol: continuousContractSymbol)));
 
                         universe = new FuturesChainUniverse((Future)security, settings);
                     }
@@ -1975,8 +1981,9 @@ namespace QuantConnect.Algorithm
             Universe universe;
             if (!UniverseManager.TryGetValue(universeSymbol, out universe))
             {
+                var settings = new UniverseSettings(UniverseSettings) { DataNormalizationMode = DataNormalizationMode.Raw, Resolution = underlyingConfigs.GetHighestResolution() };
                 universe = _pendingUniverseAdditions.FirstOrDefault(u => u.Configuration.Symbol == universeSymbol)
-                           ?? AddUniverse(new OptionContractUniverse(new SubscriptionDataConfig(configs.First(), symbol: universeSymbol), UniverseSettings));
+                           ?? AddUniverse(new OptionContractUniverse(new SubscriptionDataConfig(configs.First(), symbol: universeSymbol), settings));
             }
 
             // update the universe

Algorithm/QuantConnect.Algorithm.csproj

@@ -30,7 +30,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>

Algorithm/Selection/OptionContractUniverse.cs

@@ -65,7 +65,9 @@ namespace QuantConnect.Algorithm.Selection
                 _symbols.Remove(removedSymbol);
 
                 // the option has been removed! This can happen when the user manually removed the option contract we remove the underlying
-                if (removedSymbol.SecurityType.IsOption())
+                // but only if there isn't any other option selected using the same underlying!
+                if (removedSymbol.SecurityType.IsOption()
+                    && !_symbols.Any(symbol => symbol.SecurityType.IsOption() && symbol.Underlying == removedSymbol.Underlying))
                 {
                     Remove(removedSymbol.Underlying);
                 }

AlgorithmFactory/DebuggerHelper.cs

@@ -48,11 +48,18 @@ namespace QuantConnect.AlgorithmFactory
 
             /// <summary>
             ///  Python Tool for Visual Studio Debugger for remote python debugging.
-            /// <see cref="Language.Python"/> will use 'Python Extension in VS Code' 
-            ///or 'Python Tools in Visual Studio'
+            /// <see cref="Language.Python"/>. Deprecated, routes to DebugPy which
+            /// is it's replacement. Used in the same way.
             /// </summary>
             PTVSD,
 
+            /// <summary>
+            ///  DebugPy - a debugger for Python.
+            /// <see cref="Language.Python"/> can use  `Python Extension` in VS Code
+            /// or attach to Python in Visual Studio
+            /// </summary>
+            DebugPy,
+
             /// <summary>
             ///  PyCharm PyDev Debugger for remote python debugging.
             /// <see cref="Language.Python"/> will use 'Python Debug Server' in PyCharm
@@ -68,7 +75,7 @@ namespace QuantConnect.AlgorithmFactory
             if (language == Language.Python)
             {
                 DebuggingMethod debuggingType;
-                Enum.TryParse(Config.Get("debugging-method", DebuggingMethod.LocalCmdline.ToString()), out debuggingType);
+                Enum.TryParse(Config.Get("debugging-method", DebuggingMethod.LocalCmdline.ToString()), true, out debuggingType);
 
                 Log.Trace("DebuggerHelper.Initialize(): initializing python...");
                 PythonInitializer.Initialize();
@@ -91,8 +98,9 @@ while not sys.gettrace():
                             break;
 
                         case DebuggingMethod.PTVSD:
-                            Log.Trace("DebuggerHelper.Initialize(): waiting for PTVSD debugger to attach at localhost:5678...");
-                            PythonEngine.RunSimpleString("import ptvsd; ptvsd.enable_attach(); ptvsd.wait_for_attach()");
+                        case DebuggingMethod.DebugPy:
+                            Log.Trace("DebuggerHelper.Initialize(): debugpy waiting for attach at port 5678...");
+                            PythonEngine.RunSimpleString("import debugpy; debugpy.listen(('0.0.0.0', 5678)); debugpy.wait_for_client()");
                             break;
 
                         case DebuggingMethod.PyCharm:

AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs

@@ -83,7 +83,7 @@ namespace QuantConnect.AlgorithmFactory.Python.Wrappers
                         var repr = attr.Repr().GetStringBetweenChars('\'', '\'');
 
                         if (repr.StartsWith(moduleName) &&                // Must be defined in the module
-                            attr.TryConvert(out type) &&                  // Must be a Type
+                            attr.TryConvert(out type, true) &&                  // Must be a Type
                             typeof(QCAlgorithm).IsAssignableFrom(type))   // Must inherit from QCAlgorithm
                         {
                             Logging.Log.Trace("AlgorithmPythonWrapper(): Creating IAlgorithm instance.");

AlgorithmFactory/QuantConnect.AlgorithmFactory.csproj

@@ -29,7 +29,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

Api/QuantConnect.Api.csproj

@@ -41,7 +41,6 @@
     <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
     <PackageReference Include="NodaTime" Version="3.0.5" />
     <PackageReference Include="RestSharp" Version="106.12.0" />
-    <PackageReference Include="SharpZipLib" Version="1.2.0" />
   </ItemGroup>
   <ItemGroup>
     <Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />

Brokerages/Backtesting/BacktestingBrokerage.cs

@@ -279,12 +279,12 @@ namespace QuantConnect.Brokerages.Backtesting
                         continue;
                     }
 
-                    var fills = new OrderEvent[0];
+                    var fills = Array.Empty<OrderEvent>();
 
                     Security security;
                     if (!Algorithm.Securities.TryGetValue(order.Symbol, out security))
                     {
-                        Log.Error("BacktestingBrokerage.Scan(): Unable to process order: " + order.Id + ". The security no longer exists.");
+                        Log.Error($"BacktestingBrokerage.Scan(): Unable to process order: {order.Id}. The security no longer exists. UtcTime: {Algorithm.UtcTime}");
                         // invalidate the order in the algorithm before removing
                         OnOrderEvent(new OrderEvent(order,
                                 Algorithm.UtcTime,

Brokerages/BaseWebsocketsBrokerage.cs

@@ -149,7 +149,10 @@ namespace QuantConnect.Brokerages
             return SubscriptionManager?.GetSubscribedSymbols() ?? Enumerable.Empty<Symbol>();
         }
 
-        private void ConnectSync()
+        /// <summary>
+        /// Start websocket connect
+        /// </summary>
+        protected void ConnectSync()
         {
             var resetEvent = new ManualResetEvent(false);
             EventHandler triggerEvent = (o, args) => resetEvent.Set();

Brokerages/Binance/BinanceBrokerage.Messaging.cs

@@ -1,226 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using QuantConnect.Brokerages.Binance.Messages;
-using QuantConnect.Data.Market;
-using QuantConnect.Logging;
-using QuantConnect.Orders;
-using QuantConnect.Orders.Fees;
-using QuantConnect.Securities;
-using System;
-using System.Linq;
-using Newtonsoft.Json.Linq;
-using QuantConnect.Data;
-
-namespace QuantConnect.Brokerages.Binance
-{
-    public partial class BinanceBrokerage
-    {
-        private IDataAggregator _aggregator;
-
-        /// <summary>
-        /// Locking object for the Ticks list in the data queue handler
-        /// </summary>
-        protected readonly object TickLocker = new object();
-
-        private void OnUserMessage(WebSocketMessage webSocketMessage)
-        {
-            var e = (WebSocketClientWrapper.TextMessage)webSocketMessage.Data;
-
-            try
-            {
-                if (Log.DebuggingEnabled)
-                {
-                    Log.Debug($"BinanceBrokerage.OnUserMessage(): {e.Message}");
-                }
-
-                var obj = JObject.Parse(e.Message);
-
-                var objError = obj["error"];
-                if (objError != null)
-                {
-                    var error = objError.ToObject<ErrorMessage>();
-                    OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Error, error.Code, error.Message));
-                    return;
-                }
-
-                var objData = obj;
-
-                var objEventType = objData["e"];
-                if (objEventType != null)
-                {
-                    var eventType = objEventType.ToObject<string>();
-
-                    switch (eventType)
-                    {
-                        case "executionReport":
-                            var upd = objData.ToObject<Execution>();
-                            if (upd.ExecutionType.Equals("TRADE", StringComparison.OrdinalIgnoreCase))
-                            {
-                                OnFillOrder(upd);
-                            }
-                            break;
-                    }
-                }
-            }
-            catch (Exception exception)
-            {
-                OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Error, -1, $"Parsing wss message failed. Data: {e.Message} Exception: {exception}"));
-                throw;
-            }
-        }
-
-        private void OnDataMessage(WebSocketMessage webSocketMessage)
-        {
-            var e = (WebSocketClientWrapper.TextMessage)webSocketMessage.Data;
-
-            try
-            {
-                var obj = JObject.Parse(e.Message);
-
-                var objError = obj["error"];
-                if (objError != null)
-                {
-                    var error = objError.ToObject<ErrorMessage>();
-                    OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Error, error.Code, error.Message));
-                    return;
-                }
-
-                var objData = obj;
-
-                var objEventType = objData["e"];
-                if (objEventType != null)
-                {
-                    var eventType = objEventType.ToObject<string>();
-
-                    switch (eventType)
-                    {
-                        case "trade":
-                            var trade = objData.ToObject<Trade>();
-                            EmitTradeTick(
-                                _symbolMapper.GetLeanSymbol(trade.Symbol, SecurityType.Crypto, Market.Binance),
-                                Time.UnixMillisecondTimeStampToDateTime(trade.Time),
-                                trade.Price,
-                                trade.Quantity);
-                            break;
-                    }
-                }
-                else if (objData["u"] != null)
-                {
-                    var quote = objData.ToObject<BestBidAskQuote>();
-                    EmitQuoteTick(
-                        _symbolMapper.GetLeanSymbol(quote.Symbol, SecurityType.Crypto, Market.Binance),
-                        quote.BestBidPrice,
-                        quote.BestBidSize,
-                        quote.BestAskPrice,
-                        quote.BestAskSize);
-                }
-            }
-            catch (Exception exception)
-            {
-                OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Error, -1, $"Parsing wss message failed. Data: {e.Message} Exception: {exception}"));
-                throw;
-            }
-        }
-
-        private void EmitQuoteTick(Symbol symbol, decimal bidPrice, decimal bidSize, decimal askPrice, decimal askSize)
-        {
-            var tick = new Tick
-            {
-                AskPrice = askPrice,
-                BidPrice = bidPrice,
-                Time = DateTime.UtcNow,
-                Symbol = symbol,
-                TickType = TickType.Quote,
-                AskSize = askSize,
-                BidSize = bidSize
-            };
-            tick.SetValue();
-
-            lock (TickLocker)
-            {
-                _aggregator.Update(tick);
-            }
-        }
-
-        private void EmitTradeTick(Symbol symbol, DateTime time, decimal price, decimal quantity)
-        {
-            var tick = new Tick
-            {
-                Symbol = symbol,
-                Value = price,
-                Quantity = Math.Abs(quantity),
-                Time = time,
-                TickType = TickType.Trade
-            };
-
-            lock (TickLocker)
-            {
-                _aggregator.Update(tick);
-            }
-        }
-
-        private void OnFillOrder(Execution data)
-        {
-            try
-            {
-                var order = FindOrderByExternalId(data.OrderId);
-                if (order == null)
-                {
-                    // not our order, nothing else to do here
-                    return;
-                }
-
-                var fillPrice = data.LastExecutedPrice;
-                var fillQuantity = data.Direction == OrderDirection.Sell ? -data.LastExecutedQuantity : data.LastExecutedQuantity;
-                var updTime = Time.UnixMillisecondTimeStampToDateTime(data.TransactionTime);
-                var orderFee = new OrderFee(new CashAmount(data.Fee, data.FeeCurrency));
-                var status = ConvertOrderStatus(data.OrderStatus);
-                var orderEvent = new OrderEvent
-                (
-                    order.Id, order.Symbol, updTime, status,
-                    data.Direction, fillPrice, fillQuantity,
-                    orderFee, $"Binance Order Event {data.Direction}"
-                );
-
-                if (status == OrderStatus.Filled)
-                {
-                    Orders.Order outOrder;
-                    CachedOrderIDs.TryRemove(order.Id, out outOrder);
-                }
-
-                OnOrderEvent(orderEvent);
-            }
-            catch (Exception e)
-            {
-                Log.Error(e);
-                throw;
-            }
-        }
-
-        private Orders.Order FindOrderByExternalId(string brokerId)
-        {
-            var order = CachedOrderIDs
-                    .FirstOrDefault(o => o.Value.BrokerId.Contains(brokerId))
-                    .Value;
-            if (order == null)
-            {
-                order = _algorithm.Transactions.GetOrderByBrokerageId(brokerId);
-            }
-
-            return order;
-        }
-    }
-}

Brokerages/Binance/BinanceBrokerage.Utility.cs

@@ -1,53 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using QuantConnect.Orders;
-
-namespace QuantConnect.Brokerages.Binance
-{
-    /// <summary>
-    /// Binance utility methods
-    /// </summary>
-    public partial class BinanceBrokerage
-    {
-        private static OrderStatus ConvertOrderStatus(string raw)
-        {
-            switch (raw.LazyToUpper())
-            {
-                case "NEW":
-                    return OrderStatus.New;
-
-                case "PARTIALLY_FILLED":
-                    return OrderStatus.PartiallyFilled;
-
-                case "FILLED":
-                    return OrderStatus.Filled;
-
-                case "PENDING_CANCEL":
-                    return OrderStatus.CancelPending;
-
-                case "CANCELED":
-                    return OrderStatus.Canceled;
-
-                case "REJECTED":
-                case "EXPIRED":
-                    return OrderStatus.Invalid;
-
-                default:
-                    return OrderStatus.None;
-            }
-        }
-    }
-}

Brokerages/Binance/BinanceBrokerage.cs

@@ -1,596 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
-using QuantConnect.Configuration;
-using QuantConnect.Data;
-using QuantConnect.Data.Market;
-using QuantConnect.Interfaces;
-using QuantConnect.Logging;
-using QuantConnect.Orders;
-using QuantConnect.Packets;
-using QuantConnect.Securities;
-using QuantConnect.Util;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading;
-using Timer = System.Timers.Timer;
-
-namespace QuantConnect.Brokerages.Binance
-{
-    /// <summary>
-    /// Binance brokerage implementation
-    /// </summary>
-    [BrokerageFactory(typeof(BinanceBrokerageFactory))]
-    public partial class BinanceBrokerage : BaseWebsocketsBrokerage, IDataQueueHandler
-    {
-        private IAlgorithm _algorithm;
-        private readonly SymbolPropertiesDatabaseSymbolMapper _symbolMapper = new SymbolPropertiesDatabaseSymbolMapper(Market.Binance);
-
-        // Binance allows 5 messages per second, but we still get rate limited if we send a lot of messages at that rate
-        // By sending 3 messages per second, evenly spaced out, we can keep sending messages without being limited
-        private readonly RateGate _webSocketRateLimiter = new RateGate(1, TimeSpan.FromMilliseconds(330));
-
-        private long _lastRequestId;
-
-        private LiveNodePacket _job;
-        private string _webSocketBaseUrl;
-        private Timer _keepAliveTimer;
-        private Timer _reconnectTimer;
-        private BinanceRestApiClient _apiClient;
-        private BrokerageConcurrentMessageHandler<WebSocketMessage> _messageHandler;
-
-        private const int MaximumSymbolsPerConnection = 512;
-
-        /// <summary>
-        /// Constructor for brokerage
-        /// </summary>
-        public BinanceBrokerage() : base("Binance")
-        {
-        }
-
-        /// <summary>
-        /// Constructor for brokerage
-        /// </summary>
-        /// <param name="apiKey">api key</param>
-        /// <param name="apiSecret">api secret</param>
-        /// <param name="restApiUrl">The rest api url</param>
-        /// <param name="webSocketBaseUrl">The web socket base url</param>
-        /// <param name="algorithm">the algorithm instance is required to retrieve account type</param>
-        /// <param name="aggregator">the aggregator for consolidating ticks</param>
-        /// <param name="job">The live job packet</param>
-        public BinanceBrokerage(string apiKey, string apiSecret, string restApiUrl, string webSocketBaseUrl, IAlgorithm algorithm, IDataAggregator aggregator, LiveNodePacket job)
-            : base("Binance")
-        {
-            Initialize(
-                wssUrl: webSocketBaseUrl,
-                restApiUrl: restApiUrl,
-                apiKey: apiKey,
-                apiSecret: apiSecret,
-                algorithm: algorithm,
-                aggregator: aggregator,
-                job: job
-            );
-        }
-
-        #region IBrokerage
-
-        /// <summary>
-        /// Checks if the websocket connection is connected or in the process of connecting
-        /// </summary>
-        public override bool IsConnected => WebSocket.IsOpen;
-
-        /// <summary>
-        /// Creates wss connection
-        /// </summary>
-        public override void Connect()
-        {
-            if (IsConnected)
-                return;
-
-            _apiClient.CreateListenKey();
-            _reconnectTimer.Start();
-
-            WebSocket.Initialize($"{_webSocketBaseUrl}/{_apiClient.SessionId}");
-
-            base.Connect();
-        }
-
-        /// <summary>
-        /// Closes the websockets connection
-        /// </summary>
-        public override void Disconnect()
-        {
-            _reconnectTimer.Stop();
-
-            WebSocket?.Close();
-            _apiClient.StopSession();
-        }
-
-        /// <summary>
-        /// Gets all open positions
-        /// </summary>
-        /// <returns></returns>
-        public override List<Holding> GetAccountHoldings()
-        {
-            if (_algorithm.BrokerageModel.AccountType == AccountType.Cash)
-            {
-                return base.GetAccountHoldings(_job?.BrokerageData, _algorithm.Securities.Values);
-            }
-            return _apiClient.GetAccountHoldings();
-        }
-
-        /// <summary>
-        /// Gets the total account cash balance for specified account type
-        /// </summary>
-        /// <returns></returns>
-        public override List<CashAmount> GetCashBalance()
-        {
-            var account = _apiClient.GetCashBalance();
-            var balances = account.Balances?.Where(balance => balance.Amount > 0).ToList();
-            if (balances == null || !balances.Any())
-                return new List<CashAmount>();
-
-            return balances
-                .Select(b => new CashAmount(b.Amount, b.Asset.LazyToUpper()))
-                .ToList();
-        }
-
-        /// <summary>
-        /// Gets all orders not yet closed
-        /// </summary>
-        /// <returns></returns>
-        public override List<Order> GetOpenOrders()
-        {
-            var orders = _apiClient.GetOpenOrders();
-            List<Order> list = new List<Order>();
-            foreach (var item in orders)
-            {
-                Order order;
-                switch (item.Type.LazyToUpper())
-                {
-                    case "MARKET":
-                        order = new MarketOrder { Price = item.Price };
-                        break;
-
-                    case "LIMIT":
-                    case "LIMIT_MAKER":
-                        order = new LimitOrder { LimitPrice = item.Price };
-                        break;
-
-                    case "STOP_LOSS":
-                    case "TAKE_PROFIT":
-                        order = new StopMarketOrder { StopPrice = item.StopPrice, Price = item.Price };
-                        break;
-
-                    case "STOP_LOSS_LIMIT":
-                    case "TAKE_PROFIT_LIMIT":
-                        order = new StopLimitOrder { StopPrice = item.StopPrice, LimitPrice = item.Price };
-                        break;
-
-                    default:
-                        OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Error, -1,
-                            "BinanceBrokerage.GetOpenOrders: Unsupported order type returned from brokerage: " + item.Type));
-                        continue;
-                }
-
-                order.Quantity = item.Quantity;
-                order.BrokerId = new List<string> { item.Id };
-                order.Symbol = _symbolMapper.GetLeanSymbol(item.Symbol, SecurityType.Crypto, Market.Binance);
-                order.Time = Time.UnixMillisecondTimeStampToDateTime(item.Time);
-                order.Status = ConvertOrderStatus(item.Status);
-                order.Price = item.Price;
-
-                if (order.Status.IsOpen())
-                {
-                    var cached = CachedOrderIDs.Where(c => c.Value.BrokerId.Contains(order.BrokerId.First())).ToList();
-                    if (cached.Any())
-                    {
-                        CachedOrderIDs[cached.First().Key] = order;
-                    }
-                }
-
-                list.Add(order);
-            }
-
-            return list;
-        }
-
-        /// <summary>
-        /// Places a new order and assigns a new broker ID to the order
-        /// </summary>
-        /// <param name="order">The order to be placed</param>
-        /// <returns>True if the request for a new order has been placed, false otherwise</returns>
-        public override bool PlaceOrder(Order order)
-        {
-            var submitted = false;
-
-            _messageHandler.WithLockedStream(() =>
-            {
-                submitted = _apiClient.PlaceOrder(order);
-            });
-
-            return submitted;
-        }
-
-        /// <summary>
-        /// Updates the order with the same id
-        /// </summary>
-        /// <param name="order">The new order information</param>
-        /// <returns>True if the request was made for the order to be updated, false otherwise</returns>
-        public override bool UpdateOrder(Order order)
-        {
-            throw new NotSupportedException("BinanceBrokerage.UpdateOrder: Order update not supported. Please cancel and re-create.");
-        }
-
-        /// <summary>
-        /// Cancels the order with the specified ID
-        /// </summary>
-        /// <param name="order">The order to cancel</param>
-        /// <returns>True if the request was submitted for cancellation, false otherwise</returns>
-        public override bool CancelOrder(Order order)
-        {
-            var submitted = false;
-
-            _messageHandler.WithLockedStream(() =>
-            {
-                submitted = _apiClient.CancelOrder(order);
-            });
-
-            return submitted;
-        }
-
-        /// <summary>
-        /// Gets the history for the requested security
-        /// </summary>
-        /// <param name="request">The historical data request</param>
-        /// <returns>An enumerable of bars covering the span specified in the request</returns>
-        public override IEnumerable<BaseData> GetHistory(Data.HistoryRequest request)
-        {
-            if (request.Resolution == Resolution.Tick || request.Resolution == Resolution.Second)
-            {
-                OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution",
-                    $"{request.Resolution} resolution is not supported, no history returned"));
-                yield break;
-            }
-
-            if (request.TickType != TickType.Trade)
-            {
-                OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidTickType",
-                    $"{request.TickType} tick type not supported, no history returned"));
-                yield break;
-            }
-
-            var period = request.Resolution.ToTimeSpan();
-
-            foreach (var kline in _apiClient.GetHistory(request))
-            {
-                yield return new TradeBar()
-                {
-                    Time = Time.UnixMillisecondTimeStampToDateTime(kline.OpenTime),
-                    Symbol = request.Symbol,
-                    Low = kline.Low,
-                    High = kline.High,
-                    Open = kline.Open,
-                    Close = kline.Close,
-                    Volume = kline.Volume,
-                    Value = kline.Close,
-                    DataType = MarketDataType.TradeBar,
-                    Period = period
-                };
-            }
-        }
-
-        /// <summary>
-        /// Wss message handler
-        /// </summary>
-        /// <param name="sender"></param>
-        /// <param name="e"></param>
-        protected override void OnMessage(object sender, WebSocketMessage e)
-        {
-            _messageHandler.HandleNewMessage(e);
-        }
-
-        #endregion IBrokerage
-
-        #region IDataQueueHandler
-
-        /// <summary>
-        /// Sets the job we're subscribing for
-        /// </summary>
-        /// <param name="job">Job we're subscribing for</param>
-        public void SetJob(LiveNodePacket job)
-        {
-            var webSocketBaseUrl = job.BrokerageData["binance-websocket-url"];
-            var restApiUrl = job.BrokerageData["binance-api-url"];
-            var apiKey = job.BrokerageData["binance-api-key"];
-            var apiSecret = job.BrokerageData["binance-api-secret"];
-            var aggregator = Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(
-                Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"));
-
-            Initialize(
-                wssUrl: webSocketBaseUrl,
-                restApiUrl: restApiUrl,
-                apiKey: apiKey,
-                apiSecret: apiSecret,
-                algorithm: null,
-                aggregator: aggregator,
-                job: job
-            );
-
-            if (!IsConnected)
-            {
-                Connect();
-            }
-        }
-
-        /// <summary>
-        /// Subscribe to the specified configuration
-        /// </summary>
-        /// <param name="dataConfig">defines the parameters to subscribe to a data feed</param>
-        /// <param name="newDataAvailableHandler">handler to be fired on new data available</param>
-        /// <returns>The new enumerator for this subscription request</returns>
-        public IEnumerator<BaseData> Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler)
-        {
-            if (!CanSubscribe(dataConfig.Symbol))
-            {
-                return null;
-            }
-
-            var enumerator = _aggregator.Add(dataConfig, newDataAvailableHandler);
-            SubscriptionManager.Subscribe(dataConfig);
-
-            return enumerator;
-        }
-
-        /// <summary>
-        /// Removes the specified configuration
-        /// </summary>
-        /// <param name="dataConfig">Subscription config to be removed</param>
-        public void Unsubscribe(SubscriptionDataConfig dataConfig)
-        {
-            SubscriptionManager.Unsubscribe(dataConfig);
-            _aggregator.Remove(dataConfig);
-        }
-
-        /// <summary>
-        /// Checks if this brokerage supports the specified symbol
-        /// </summary>
-        /// <param name="symbol">The symbol</param>
-        /// <returns>returns true if brokerage supports the specified symbol; otherwise false</returns>
-        private static bool CanSubscribe(Symbol symbol)
-        {
-            return !symbol.Value.Contains("UNIVERSE") &&
-                   symbol.SecurityType == SecurityType.Crypto &&
-                   symbol.ID.Market == Market.Binance;
-        }
-
-        #endregion IDataQueueHandler
-
-        /// <summary>
-        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
-        /// </summary>
-        public override void Dispose()
-        {
-            _keepAliveTimer.DisposeSafely();
-            _reconnectTimer.DisposeSafely();
-            _apiClient.DisposeSafely();
-            _webSocketRateLimiter.DisposeSafely();
-        }
-
-        /// <summary>
-        /// Not used
-        /// </summary>
-        protected override bool Subscribe(IEnumerable<Symbol> symbols)
-        {
-            // NOP
-            return true;
-        }
-
-        /// <summary>
-        /// Initialize the instance of this class
-        /// </summary>
-        /// <param name="wssUrl">The web socket base url</param>
-        /// <param name="restApiUrl">The rest api url</param>
-        /// <param name="apiKey">api key</param>
-        /// <param name="apiSecret">api secret</param>
-        /// <param name="algorithm">the algorithm instance is required to retrieve account type</param>
-        /// <param name="aggregator">the aggregator for consolidating ticks</param>
-        /// <param name="job">The live job packet</param>
-        private void Initialize(string wssUrl, string restApiUrl,string apiKey, string apiSecret,
-            IAlgorithm algorithm, IDataAggregator aggregator, LiveNodePacket job)
-        {
-            if (IsInitialized)
-            {
-                return;
-            }
-            base.Initialize(wssUrl,  new WebSocketClientWrapper(), null, apiKey, apiSecret);
-            _job = job;
-            _algorithm = algorithm;
-            _aggregator = aggregator;
-            _webSocketBaseUrl = wssUrl;
-            _messageHandler = new BrokerageConcurrentMessageHandler<WebSocketMessage>(OnUserMessage);
-
-            var maximumWebSocketConnections = Config.GetInt("binance-maximum-websocket-connections");
-            var symbolWeights = maximumWebSocketConnections > 0 ? FetchSymbolWeights() : null;
-
-            var subscriptionManager = new BrokerageMultiWebSocketSubscriptionManager(
-                wssUrl,
-                MaximumSymbolsPerConnection,
-                maximumWebSocketConnections,
-                symbolWeights,
-                () => new BinanceWebSocketWrapper(null),
-                Subscribe,
-                Unsubscribe,
-                OnDataMessage,
-                new TimeSpan(23, 45, 0));
-
-            SubscriptionManager = subscriptionManager;
-
-            _apiClient = new BinanceRestApiClient(_symbolMapper,
-                algorithm?.Portfolio,
-                apiKey,
-                apiSecret,
-                restApiUrl);
-
-            _apiClient.OrderSubmit += (s, e) => OnOrderSubmit(e);
-            _apiClient.OrderStatusChanged += (s, e) => OnOrderEvent(e);
-            _apiClient.Message += (s, e) => OnMessage(e);
-
-            // User data streams will close after 60 minutes. It's recommended to send a ping about every 30 minutes.
-            // Source: https://github.com/binance-exchange/binance-official-api-docs/blob/master/user-data-stream.md#pingkeep-alive-a-listenkey
-            _keepAliveTimer = new Timer
-            {
-                // 30 minutes
-                Interval = 30 * 60 * 1000
-            };
-            _keepAliveTimer.Elapsed += (s, e) => _apiClient.SessionKeepAlive();
-
-            WebSocket.Open += (s, e) => { _keepAliveTimer.Start(); };
-            WebSocket.Closed += (s, e) => { _keepAliveTimer.Stop(); };
-
-            // A single connection to stream.binance.com is only valid for 24 hours; expect to be disconnected at the 24 hour mark
-            // Source: https://github.com/binance-exchange/binance-official-api-docs/blob/master/web-socket-streams.md#general-wss-information
-            _reconnectTimer = new Timer
-            {
-                // 23.5 hours
-                Interval = 23.5 * 60 * 60 * 1000
-            };
-            _reconnectTimer.Elapsed += (s, e) =>
-            {
-                Log.Trace("Daily websocket restart: disconnect");
-                Disconnect();
-
-                Log.Trace("Daily websocket restart: connect");
-                Connect();
-            };
-        }
-
-        /// <summary>
-        /// Subscribes to the requested symbol (using an individual streaming channel)
-        /// </summary>
-        /// <param name="webSocket">The websocket instance</param>
-        /// <param name="symbol">The symbol to subscribe</param>
-        private bool Subscribe(IWebSocket webSocket, Symbol symbol)
-        {
-            Send(webSocket,
-                new
-                {
-                    method = "SUBSCRIBE",
-                    @params = new[]
-                    {
-                        $"{symbol.Value.ToLowerInvariant()}@trade",
-                        $"{symbol.Value.ToLowerInvariant()}@bookTicker"
-                    },
-                    id = GetNextRequestId()
-                }
-            );
-
-            return true;
-        }
-
-        /// <summary>
-        /// Ends current subscription
-        /// </summary>
-        /// <param name="webSocket">The websocket instance</param>
-        /// <param name="symbol">The symbol to unsubscribe</param>
-        private bool Unsubscribe(IWebSocket webSocket, Symbol symbol)
-        {
-            Send(webSocket,
-                new
-                {
-                    method = "UNSUBSCRIBE",
-                    @params = new[]
-                    {
-                        $"{symbol.Value.ToLowerInvariant()}@trade",
-                        $"{symbol.Value.ToLowerInvariant()}@bookTicker"
-                    },
-                    id = GetNextRequestId()
-                }
-            );
-
-            return true;
-        }
-
-        private void Send(IWebSocket webSocket, object obj)
-        {
-            var json = JsonConvert.SerializeObject(obj);
-
-            _webSocketRateLimiter.WaitToProceed();
-
-            Log.Trace("Send: " + json);
-
-            webSocket.Send(json);
-        }
-
-        private long GetNextRequestId()
-        {
-            return Interlocked.Increment(ref _lastRequestId);
-        }
-
-        /// <summary>
-        /// Event invocator for the OrderFilled event
-        /// </summary>
-        /// <param name="e">The OrderEvent</param>
-        private void OnOrderSubmit(BinanceOrderSubmitEventArgs e)
-        {
-            var brokerId = e.BrokerId;
-            var order = e.Order;
-            if (CachedOrderIDs.ContainsKey(order.Id))
-            {
-                CachedOrderIDs[order.Id].BrokerId.Clear();
-                CachedOrderIDs[order.Id].BrokerId.Add(brokerId);
-            }
-            else
-            {
-                order.BrokerId.Add(brokerId);
-                CachedOrderIDs.TryAdd(order.Id, order);
-            }
-        }
-
-        /// <summary>
-        /// Returns the weights for each symbol (the weight value is the count of trades in the last 24 hours)
-        /// </summary>
-        private static Dictionary<Symbol, int> FetchSymbolWeights()
-        {
-            var dict = new Dictionary<Symbol, int>();
-
-            try
-            {
-                const string url = "https://api.binance.com/api/v3/ticker/24hr";
-                var json = url.DownloadData();
-
-                foreach (var row in JArray.Parse(json))
-                {
-                    var ticker = row["symbol"].ToObject<string>();
-                    var count = row["count"].ToObject<int>();
-
-                    var symbol = Symbol.Create(ticker, SecurityType.Crypto, Market.Binance);
-
-                    dict.Add(symbol, count);
-                }
-            }
-            catch (Exception exception)
-            {
-                Log.Error(exception);
-                throw;
-            }
-
-            return dict;
-        }
-    }
-}

Brokerages/Binance/BinanceBrokerageFactory.cs

@@ -1,98 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using System.Collections.Generic;
-using QuantConnect.Configuration;
-using QuantConnect.Data;
-using QuantConnect.Interfaces;
-using QuantConnect.Securities;
-using QuantConnect.Util;
-
-namespace QuantConnect.Brokerages.Binance
-{
-    /// <summary>
-    /// Factory method to create binance Websockets brokerage
-    /// </summary>
-    public class BinanceBrokerageFactory : BrokerageFactory
-    {
-        /// <summary>
-        /// Factory constructor
-        /// </summary>
-        public BinanceBrokerageFactory() : base(typeof(BinanceBrokerage))
-        {
-        }
-
-        /// <summary>
-        /// Not required
-        /// </summary>
-        public override void Dispose()
-        {
-        }
-
-        /// <summary>
-        /// provides brokerage connection data
-        /// </summary>
-        public override Dictionary<string, string> BrokerageData => new Dictionary<string, string>
-        {
-            { "binance-api-key", Config.Get("binance-api-key")},
-            { "binance-api-secret", Config.Get("binance-api-secret")},
-            // paper trading available using https://testnet.binance.vision
-            { "binance-api-url", Config.Get("binance-api-url", "https://api.binance.com")},
-            // paper trading available using wss://testnet.binance.vision/ws
-            { "binance-websocket-url", Config.Get("binance-websocket-url", "wss://stream.binance.com:9443/ws")},
-
-            // load holdings if available
-            { "live-holdings", Config.Get("live-holdings")},
-        };
-
-        /// <summary>
-        /// The brokerage model
-        /// </summary>
-        /// <param name="orderProvider">The order provider</param>
-        public override IBrokerageModel GetBrokerageModel(IOrderProvider orderProvider) => new BinanceBrokerageModel();
-
-        /// <summary>
-        /// Create the Brokerage instance
-        /// </summary>
-        /// <param name="job"></param>
-        /// <param name="algorithm"></param>
-        /// <returns></returns>
-        public override IBrokerage CreateBrokerage(Packets.LiveNodePacket job, IAlgorithm algorithm)
-        {
-            var required = new[] { "binance-api-secret", "binance-api-key", "binance-api-url", "binance-websocket-url" };
-
-            foreach (var item in required)
-            {
-                if (string.IsNullOrEmpty(job.BrokerageData[item]))
-                {
-                    throw new ArgumentException($"BinanceBrokerageFactory.CreateBrokerage: Missing {item} in config.json");
-                }
-            }
-
-            var brokerage = new BinanceBrokerage(
-                job.BrokerageData["binance-api-key"],
-                job.BrokerageData["binance-api-secret"],
-                job.BrokerageData["binance-api-url"],
-                job.BrokerageData["binance-websocket-url"],
-                algorithm,
-                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager")),
-                job);
-            Composer.Instance.AddPart<IDataQueueHandler>(brokerage);
-
-            return brokerage;
-        }
-    }
-}

Brokerages/Binance/BinanceOrderSubmitEventArgs.cs

@@ -1,46 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using QuantConnect.Orders;
-
-namespace QuantConnect.Brokerages.Binance
-{
-    /// <summary>
-    /// Represents a binance submit order event data
-    /// </summary>
-    public class BinanceOrderSubmitEventArgs
-    {
-        /// <summary>
-        /// Order Event Constructor.
-        /// </summary>
-        /// <param name="brokerId">Binance order id returned from brokerage</param>
-        /// <param name="order">Order for this order placement</param>
-        public BinanceOrderSubmitEventArgs(string brokerId, Order order)
-        {
-            BrokerId = brokerId;
-            Order = order;
-        }
-
-        /// <summary>
-        /// Original brokerage id
-        /// </summary>
-        public string BrokerId { get; set; }
-
-        /// <summary>
-        /// The lean order
-        /// </summary>
-        public Order Order { get; set; }
-    }
-}

Brokerages/Binance/BinanceRestApiClient.cs

@@ -1,606 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
-using QuantConnect.Logging;
-using QuantConnect.Orders;
-using QuantConnect.Orders.Fees;
-using QuantConnect.Securities;
-using QuantConnect.Util;
-using RestSharp;
-using System;
-using System.Collections.Generic;
-using System.Globalization;
-using System.Linq;
-using System.Net;
-using System.Security.Cryptography;
-using System.Text;
-
-namespace QuantConnect.Brokerages.Binance
-{
-    /// <summary>
-    /// Binance REST API implementation
-    /// </summary>
-    public class BinanceRestApiClient : IDisposable
-    {
-        private const string UserDataStreamEndpoint = "/api/v3/userDataStream";
-
-        private readonly SymbolPropertiesDatabaseSymbolMapper _symbolMapper;
-        private readonly ISecurityProvider _securityProvider;
-        private readonly IRestClient _restClient;
-        private readonly RateGate _restRateLimiter = new RateGate(10, TimeSpan.FromSeconds(1));
-        private readonly object _listenKeyLocker = new object();
-
-        /// <summary>
-        /// Event that fires each time an order is filled
-        /// </summary>
-        public event EventHandler<BinanceOrderSubmitEventArgs> OrderSubmit;
-
-        /// <summary>
-        /// Event that fires each time an order is filled
-        /// </summary>
-        public event EventHandler<OrderEvent> OrderStatusChanged;
-
-        /// <summary>
-        /// Event that fires when an error is encountered in the brokerage
-        /// </summary>
-        public event EventHandler<BrokerageMessageEvent> Message;
-
-        /// <summary>
-        /// Key Header
-        /// </summary>
-        public readonly string KeyHeader = "X-MBX-APIKEY";
-
-        /// <summary>
-        /// The api secret
-        /// </summary>
-        protected string ApiSecret;
-
-        /// <summary>
-        /// The api key
-        /// </summary>
-        protected string ApiKey;
-
-        /// <summary>
-        /// Represents UserData Session listen key
-        /// </summary>
-        public string SessionId { get; private set; }
-
-        /// <summary>
-        /// Initializes a new instance of the <see cref="BinanceRestApiClient"/> class.
-        /// </summary>
-        /// <param name="symbolMapper">The symbol mapper.</param>
-        /// <param name="securityProvider">The holdings provider.</param>
-        /// <param name="apiKey">The Binance API key</param>
-        /// <param name="apiSecret">The The Binance API secret</param>
-        /// <param name="restApiUrl">The Binance API rest url</param>
-        public BinanceRestApiClient(SymbolPropertiesDatabaseSymbolMapper symbolMapper, ISecurityProvider securityProvider,
-            string apiKey, string apiSecret, string restApiUrl)
-        {
-            _symbolMapper = symbolMapper;
-            _securityProvider = securityProvider;
-            _restClient = new RestClient(restApiUrl);
-            ApiKey = apiKey;
-            ApiSecret = apiSecret;
-        }
-
-        /// <summary>
-        /// Gets all open positions
-        /// </summary>
-        /// <returns></returns>
-        public List<Holding> GetAccountHoldings()
-        {
-            return new List<Holding>();
-        }
-
-        /// <summary>
-        /// Gets the total account cash balance for specified account type
-        /// </summary>
-        /// <returns></returns>
-        public Messages.AccountInformation GetCashBalance()
-        {
-            var queryString = $"timestamp={GetNonce()}";
-            var endpoint = $"/api/v3/account?{queryString}&signature={AuthenticationToken(queryString)}";
-            var request = new RestRequest(endpoint, Method.GET);
-            request.AddHeader(KeyHeader, ApiKey);
-
-            var response = ExecuteRestRequest(request);
-            if (response.StatusCode != HttpStatusCode.OK)
-            {
-                throw new Exception($"BinanceBrokerage.GetCashBalance: request failed: [{(int)response.StatusCode}] {response.StatusDescription}, Content: {response.Content}, ErrorMessage: {response.ErrorMessage}");
-            }
-
-            return JsonConvert.DeserializeObject<Messages.AccountInformation>(response.Content);
-        }
-
-        /// <summary>
-        /// Gets all orders not yet closed
-        /// </summary>
-        /// <returns></returns>
-        public IEnumerable<Messages.OpenOrder> GetOpenOrders()
-        {
-            var queryString = $"timestamp={GetNonce()}";
-            var endpoint = $"/api/v3/openOrders?{queryString}&signature={AuthenticationToken(queryString)}";
-            var request = new RestRequest(endpoint, Method.GET);
-            request.AddHeader(KeyHeader, ApiKey);
-
-            var response = ExecuteRestRequest(request);
-            if (response.StatusCode != HttpStatusCode.OK)
-            {
-                throw new Exception($"BinanceBrokerage.GetCashBalance: request failed: [{(int)response.StatusCode}] {response.StatusDescription}, Content: {response.Content}, ErrorMessage: {response.ErrorMessage}");
-            }
-
-            return JsonConvert.DeserializeObject<Messages.OpenOrder[]>(response.Content);
-        }
-
-        /// <summary>
-        /// Places a new order and assigns a new broker ID to the order
-        /// </summary>
-        /// <param name="order">The order to be placed</param>
-        /// <returns>True if the request for a new order has been placed, false otherwise</returns>
-        public bool PlaceOrder(Order order)
-        {
-            // supported time in force values {GTC, IOC, FOK}
-            // use GTC as LEAN doesn't support others yet
-            IDictionary<string, object> body = new Dictionary<string, object>()
-            {
-                { "symbol", _symbolMapper.GetBrokerageSymbol(order.Symbol) },
-                { "quantity", Math.Abs(order.Quantity).ToString(CultureInfo.InvariantCulture) },
-                { "side", ConvertOrderDirection(order.Direction) }
-            };
-
-            switch (order)
-            {
-                case LimitOrder limitOrder:
-                    body["type"] = (order.Properties as BinanceOrderProperties)?.PostOnly == true
-                        ? "LIMIT_MAKER"
-                        : "LIMIT";
-                    body["price"] = limitOrder.LimitPrice.ToString(CultureInfo.InvariantCulture);
-                    // timeInForce is not required for LIMIT_MAKER
-                    if (Equals(body["type"], "LIMIT"))
-                        body["timeInForce"] = "GTC";
-                    break;
-                case MarketOrder:
-                    body["type"] = "MARKET";
-                    break;
-                case StopLimitOrder stopLimitOrder:
-                    var ticker = GetTickerPrice(order);
-                    var stopPrice = stopLimitOrder.StopPrice;
-                    if (order.Direction == OrderDirection.Sell)
-                    {
-                        body["type"] = stopPrice <= ticker ? "STOP_LOSS_LIMIT" : "TAKE_PROFIT_LIMIT";
-                    }
-                    else
-                    {
-                        body["type"] = stopPrice <= ticker ? "TAKE_PROFIT_LIMIT" : "STOP_LOSS_LIMIT";
-                    }
-
-                    body["timeInForce"] = "GTC";
-                    body["stopPrice"] = stopPrice.ToStringInvariant();
-                    body["price"] = stopLimitOrder.LimitPrice.ToStringInvariant();
-                    break;
-                default:
-                    throw new NotSupportedException($"BinanceBrokerage.ConvertOrderType: Unsupported order type: {order.Type}");
-            }
-
-            const string endpoint = "/api/v3/order";
-            body["timestamp"] = GetNonce();
-            body["signature"] = AuthenticationToken(body.ToQueryString());
-            var request = new RestRequest(endpoint, Method.POST);
-            request.AddHeader(KeyHeader, ApiKey);
-            request.AddParameter(
-                "application/x-www-form-urlencoded",
-                Encoding.UTF8.GetBytes(body.ToQueryString()),
-                ParameterType.RequestBody
-            );
-
-            var response = ExecuteRestRequest(request);
-            if (response.StatusCode == HttpStatusCode.OK)
-            {
-                var raw = JsonConvert.DeserializeObject<Messages.NewOrder>(response.Content);
-
-                if (string.IsNullOrEmpty(raw?.Id))
-                {
-                    var errorMessage = $"Error parsing response from place order: {response.Content}";
-                    OnOrderEvent(new OrderEvent(
-                        order,
-                        DateTime.UtcNow,
-                        OrderFee.Zero,
-                        "Binance Order Event")
-                    { Status = OrderStatus.Invalid, Message = errorMessage });
-                    OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, (int)response.StatusCode, errorMessage));
-
-                    return true;
-                }
-
-                OnOrderSubmit(raw, order);
-                return true;
-            }
-
-            var message = $"Order failed, Order Id: {order.Id} timestamp: {order.Time} quantity: {order.Quantity} content: {response.Content}";
-            OnOrderEvent(new OrderEvent(
-                order,
-                DateTime.UtcNow,
-                OrderFee.Zero,
-                "Binance Order Event")
-            { Status = OrderStatus.Invalid });
-            OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, -1, message));
-
-            return true;
-        }
-
-        /// <summary>
-        /// Cancels the order with the specified ID
-        /// </summary>
-        /// <param name="order">The order to cancel</param>
-        /// <returns>True if the request was submitted for cancellation, false otherwise</returns>
-        public bool CancelOrder(Order order)
-        {
-            var success = new List<bool>();
-            IDictionary<string, object> body = new Dictionary<string, object>()
-            {
-                { "symbol", _symbolMapper.GetBrokerageSymbol(order.Symbol) }
-            };
-            foreach (var id in order.BrokerId)
-            {
-                if (body.ContainsKey("signature"))
-                {
-                    body.Remove("signature");
-                }
-                body["orderId"] = id;
-                body["timestamp"] = GetNonce();
-                body["signature"] = AuthenticationToken(body.ToQueryString());
-
-                var request = new RestRequest("/api/v3/order", Method.DELETE);
-                request.AddHeader(KeyHeader, ApiKey);
-                request.AddParameter(
-                    "application/x-www-form-urlencoded",
-                    Encoding.UTF8.GetBytes(body.ToQueryString()),
-                    ParameterType.RequestBody
-                );
-
-                var response = ExecuteRestRequest(request);
-                success.Add(response.StatusCode == HttpStatusCode.OK);
-            }
-
-            var canceled = false;
-            if (success.All(a => a))
-            {
-                OnOrderEvent(new OrderEvent(order,
-                    DateTime.UtcNow,
-                    OrderFee.Zero,
-                    "Binance Order Event")
-                { Status = OrderStatus.Canceled });
-
-                canceled = true;
-            }
-            return canceled;
-        }
-
-        /// <summary>
-        /// Gets the history for the requested security
-        /// </summary>
-        /// <param name="request">The historical data request</param>
-        /// <returns>An enumerable of bars covering the span specified in the request</returns>
-        public IEnumerable<Messages.Kline> GetHistory(Data.HistoryRequest request)
-        {
-            var resolution = ConvertResolution(request.Resolution);
-            var resolutionInMs = (long)request.Resolution.ToTimeSpan().TotalMilliseconds;
-            var symbol = _symbolMapper.GetBrokerageSymbol(request.Symbol);
-            var startMs = (long)Time.DateTimeToUnixTimeStamp(request.StartTimeUtc) * 1000;
-            var endMs = (long)Time.DateTimeToUnixTimeStamp(request.EndTimeUtc) * 1000;
-            // we always use the real endpoint for history requests
-            var endpoint = $"https://api.binance.com/api/v3/klines?symbol={symbol}&interval={resolution}&limit=1000";
-
-            while (endMs - startMs >= resolutionInMs)
-            {
-                var timeframe = $"&startTime={startMs}&endTime={endMs}";
-
-                var restRequest = new RestRequest(endpoint + timeframe, Method.GET);
-                var response = ExecuteRestRequest(restRequest);
-
-                if (response.StatusCode != HttpStatusCode.OK)
-                {
-                    throw new Exception($"BinanceBrokerage.GetHistory: request failed: [{(int)response.StatusCode}] {response.StatusDescription}, Content: {response.Content}, ErrorMessage: {response.ErrorMessage}");
-                }
-
-                var klines = JsonConvert.DeserializeObject<object[][]>(response.Content)
-                    .Select(entries => new Messages.Kline(entries))
-                    .ToList();
-
-                if (klines.Count > 0)
-                {
-                    var lastValue = klines[klines.Count - 1];
-                    if (Log.DebuggingEnabled)
-                    {
-                        var windowStartTime = Time.UnixMillisecondTimeStampToDateTime(klines[0].OpenTime);
-                        var windowEndTime = Time.UnixMillisecondTimeStampToDateTime(lastValue.OpenTime + resolutionInMs);
-                        Log.Debug($"BinanceRestApiClient.GetHistory(): Received [{symbol}] data for timeperiod from {windowStartTime.ToStringInvariant()} to {windowEndTime.ToStringInvariant()}..");
-                    }
-                    startMs = lastValue.OpenTime + resolutionInMs;
-
-                    foreach (var kline in klines)
-                    {
-                        yield return kline;
-                    }
-                }
-                else
-                {
-                    // if there is no data just break
-                    break;
-                }
-            }
-        }
-
-        /// <summary>
-        /// Check User Data stream listen key is alive
-        /// </summary>
-        /// <returns></returns>
-        public bool SessionKeepAlive()
-        {
-            if (string.IsNullOrEmpty(SessionId))
-            {
-                throw new Exception("BinanceBrokerage:UserStream. listenKey wasn't allocated or has been refused.");
-            }
-
-            var ping = new RestRequest(UserDataStreamEndpoint, Method.PUT);
-            ping.AddHeader(KeyHeader, ApiKey);
-            ping.AddParameter(
-                "application/x-www-form-urlencoded",
-                Encoding.UTF8.GetBytes($"listenKey={SessionId}"),
-                ParameterType.RequestBody
-            );
-
-            var pong = ExecuteRestRequest(ping);
-            return pong.StatusCode == HttpStatusCode.OK;
-        }
-
-        /// <summary>
-        /// Stops the session
-        /// </summary>
-        public void StopSession()
-        {
-            if (!string.IsNullOrEmpty(SessionId))
-            {
-                var request = new RestRequest(UserDataStreamEndpoint, Method.DELETE);
-                request.AddHeader(KeyHeader, ApiKey);
-                request.AddParameter(
-                    "application/x-www-form-urlencoded",
-                    Encoding.UTF8.GetBytes($"listenKey={SessionId}"),
-                    ParameterType.RequestBody
-                );
-                ExecuteRestRequest(request);
-            }
-        }
-
-        /// <summary>
-        /// Provides the current tickers price
-        /// </summary>
-        /// <returns></returns>
-        public Messages.PriceTicker[] GetTickers()
-        {
-            const string endpoint = "/api/v3/ticker/price";
-            var req = new RestRequest(endpoint, Method.GET);
-            var response = ExecuteRestRequest(req);
-            if (response.StatusCode != HttpStatusCode.OK)
-            {
-                throw new Exception($"BinanceBrokerage.GetTick: request failed: [{(int)response.StatusCode}] {response.StatusDescription}, Content: {response.Content}, ErrorMessage: {response.ErrorMessage}");
-            }
-
-            return JsonConvert.DeserializeObject<Messages.PriceTicker[]>(response.Content);
-        }
-
-        /// <summary>
-        /// Start user data stream
-        /// </summary>
-        public void CreateListenKey()
-        {
-            var request = new RestRequest(UserDataStreamEndpoint, Method.POST);
-            request.AddHeader(KeyHeader, ApiKey);
-
-            var response = ExecuteRestRequest(request);
-            if (response.StatusCode != HttpStatusCode.OK)
-            {
-                throw new Exception($"BinanceBrokerage.StartSession: request failed: [{(int)response.StatusCode}] {response.StatusDescription}, Content: {response.Content}, ErrorMessage: {response.ErrorMessage}");
-            }
-
-            var content = JObject.Parse(response.Content);
-            lock (_listenKeyLocker)
-            {
-                SessionId = content.Value<string>("listenKey");
-            }
-        }
-
-        /// <summary>
-        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
-        /// </summary>
-        public void Dispose()
-        {
-            _restRateLimiter.DisposeSafely();
-        }
-
-        /// <summary>
-        /// If an IP address exceeds a certain number of requests per minute
-        /// HTTP 429 return code is used when breaking a request rate limit.
-        /// </summary>
-        /// <param name="request"></param>
-        /// <returns></returns>
-        private IRestResponse ExecuteRestRequest(IRestRequest request)
-        {
-            const int maxAttempts = 10;
-            var attempts = 0;
-            IRestResponse response;
-
-            do
-            {
-                if (!_restRateLimiter.WaitToProceed(TimeSpan.Zero))
-                {
-                    Log.Trace("Brokerage.OnMessage(): " + new BrokerageMessageEvent(BrokerageMessageType.Warning, "RateLimit",
-                        "The API request has been rate limited. To avoid this message, please reduce the frequency of API calls."));
-
-                    _restRateLimiter.WaitToProceed();
-                }
-
-                response = _restClient.Execute(request);
-                // 429 status code: Too Many Requests
-            } while (++attempts < maxAttempts && (int)response.StatusCode == 429);
-
-            return response;
-        }
-
-        private decimal GetTickerPrice(Order order)
-        {
-            var security = _securityProvider.GetSecurity(order.Symbol);
-            var tickerPrice = order.Direction == OrderDirection.Buy ? security.AskPrice : security.BidPrice;
-            if (tickerPrice == 0)
-            {
-                var brokerageSymbol = _symbolMapper.GetBrokerageSymbol(order.Symbol);
-                var tickers = GetTickers();
-                var ticker = tickers.FirstOrDefault(t => t.Symbol == brokerageSymbol);
-                if (ticker == null)
-                {
-                    throw new KeyNotFoundException($"BinanceBrokerage: Unable to resolve currency conversion pair: {order.Symbol}");
-                }
-                tickerPrice = ticker.Price;
-            }
-            return tickerPrice;
-        }
-
-        /// <summary>
-        /// Timestamp in milliseconds
-        /// </summary>
-        /// <returns></returns>
-        private long GetNonce()
-        {
-            return (long)(Time.TimeStamp() * 1000);
-        }
-
-        /// <summary>
-        /// Creates a signature for signed endpoints
-        /// </summary>
-        /// <param name="payload">the body of the request</param>
-        /// <returns>a token representing the request params</returns>
-        private string AuthenticationToken(string payload)
-        {
-            using (HMACSHA256 hmac = new HMACSHA256(Encoding.UTF8.GetBytes(ApiSecret)))
-            {
-                return hmac.ComputeHash(Encoding.UTF8.GetBytes(payload)).ToHexString();
-            }
-        }
-
-        private static string ConvertOrderDirection(OrderDirection orderDirection)
-        {
-            if (orderDirection == OrderDirection.Buy || orderDirection == OrderDirection.Sell)
-            {
-                return orderDirection.ToString().LazyToUpper();
-            }
-
-            throw new NotSupportedException($"BinanceBrokerage.ConvertOrderDirection: Unsupported order direction: {orderDirection}");
-        }
-
-
-        private readonly Dictionary<Resolution, string> _knownResolutions = new Dictionary<Resolution, string>()
-        {
-            { Resolution.Minute, "1m" },
-            { Resolution.Hour,   "1h" },
-            { Resolution.Daily,  "1d" }
-        };
-
-        private string ConvertResolution(Resolution resolution)
-        {
-            if (_knownResolutions.ContainsKey(resolution))
-            {
-                return _knownResolutions[resolution];
-            }
-            else
-            {
-                throw new ArgumentException($"BinanceBrokerage.ConvertResolution: Unsupported resolution type: {resolution}");
-            }
-        }
-
-        /// <summary>
-        /// Event invocator for the OrderFilled event
-        /// </summary>
-        /// <param name="newOrder">The brokerage order submit result</param>
-        /// <param name="order">The lean order</param>
-        private void OnOrderSubmit(Messages.NewOrder newOrder, Order order)
-        {
-            try
-            {
-                OrderSubmit?.Invoke(
-                    this,
-                    new BinanceOrderSubmitEventArgs(newOrder.Id, order));
-
-                // Generate submitted event
-                OnOrderEvent(new OrderEvent(
-                    order,
-                    Time.UnixMillisecondTimeStampToDateTime(newOrder.TransactionTime),
-                    OrderFee.Zero,
-                    "Binance Order Event")
-                { Status = OrderStatus.Submitted }
-                );
-                Log.Trace($"Order submitted successfully - OrderId: {order.Id}");
-            }
-            catch (Exception err)
-            {
-                Log.Error(err);
-            }
-        }
-
-        /// <summary>
-        /// Event invocator for the OrderFilled event
-        /// </summary>
-        /// <param name="e">The OrderEvent</param>
-        private void OnOrderEvent(OrderEvent e)
-        {
-            try
-            {
-                Log.Debug("Brokerage.OnOrderEvent(): " + e);
-
-                OrderStatusChanged?.Invoke(this, e);
-            }
-            catch (Exception err)
-            {
-                Log.Error(err);
-            }
-        }
-
-        /// <summary>
-        /// Event invocator for the Message event
-        /// </summary>
-        /// <param name="e">The error</param>
-        protected virtual void OnMessage(BrokerageMessageEvent e)
-        {
-            try
-            {
-                if (e.Type == BrokerageMessageType.Error)
-                {
-                    Log.Error("Brokerage.OnMessage(): " + e);
-                }
-                else
-                {
-                    Log.Trace("Brokerage.OnMessage(): " + e);
-                }
-
-                Message?.Invoke(this, e);
-            }
-            catch (Exception err)
-            {
-                Log.Error(err);
-            }
-        }
-    }
-}

Brokerages/Binance/Messages.cs

@@ -1,209 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using Newtonsoft.Json;
-using QuantConnect.Orders;
-using System;
-using System.Globalization;
-
-namespace QuantConnect.Brokerages.Binance.Messages
-{
-#pragma warning disable 1591
-
-    public class AccountInformation
-    {
-        public Balance[] Balances { get; set; }
-
-        public class Balance
-        {
-            public string Asset { get; set; }
-            public decimal Free { get; set; }
-            public decimal Locked { get; set; }
-            public decimal Amount => Free + Locked;
-        }
-    }
-
-    public class PriceTicker
-    {
-        public string Symbol { get; set; }
-        public decimal Price { get; set; }
-    }
-
-    public class Order
-    {
-        [JsonProperty("orderId")]
-        public string Id { get; set; }
-        public string Symbol { get; set; }
-        public decimal Price { get; set; }
-        public decimal StopPrice { get; set; }
-        [JsonProperty("origQty")]
-        public decimal OriginalAmount { get; set; }
-        [JsonProperty("executedQty")]
-        public decimal ExecutedAmount { get; set; }
-        public string Status { get; set; }
-        public string Type { get; set; }
-        public string Side { get; set; }
-
-        public decimal Quantity => string.Equals(Side, "buy", StringComparison.OrdinalIgnoreCase) ? OriginalAmount : -OriginalAmount;
-    }
-
-    public class OpenOrder : Order
-    {
-        public long Time { get; set; }
-    }
-
-    public class NewOrder : Order
-    {
-        [JsonProperty("transactTime")]
-        public long TransactionTime { get; set; }
-    }
-
-    public enum EventType
-    {
-        None,
-        OrderBook,
-        Trade,
-        Execution
-    }
-
-    public class ErrorMessage
-    {
-        [JsonProperty("code")]
-        public int Code { get; set; }
-
-        [JsonProperty("msg")]
-        public string Message { get; set; }
-    }
-
-    public class BestBidAskQuote
-    {
-        [JsonProperty("u")]
-        public long OrderBookUpdateId { get; set; }
-
-        [JsonProperty("s")]
-        public string Symbol { get; set; }
-
-        [JsonProperty("b")]
-        public decimal BestBidPrice { get; set; }
-
-        [JsonProperty("B")]
-        public decimal BestBidSize { get; set; }
-
-        [JsonProperty("a")]
-        public decimal BestAskPrice { get; set; }
-
-        [JsonProperty("A")]
-        public decimal BestAskSize { get; set; }
-    }
-
-    public class BaseMessage
-    {
-        public virtual EventType @Event { get; } = EventType.None;
-
-        [JsonProperty("e")]
-        public string EventName { get; set; }
-
-        [JsonProperty("E")]
-        public long Time { get; set; }
-
-        [JsonProperty("s")]
-        public string Symbol { get; set; }
-    }
-
-    public class Trade : BaseMessage
-    {
-        public override EventType @Event => EventType.Trade;
-
-        [JsonProperty("T")]
-        public new long Time { get; set; }
-
-        [JsonProperty("p")]
-        public decimal Price { get; private set; }
-
-        [JsonProperty("q")]
-        public decimal Quantity { get; private set; }
-    }
-
-    public class Execution : BaseMessage
-    {
-        public override EventType @Event => EventType.Execution;
-
-        [JsonProperty("i")]
-        public string OrderId { get; set; }
-
-        [JsonProperty("t")]
-        public string TradeId { get; set; }
-
-        [JsonProperty("I")]
-        public string Ignore { get; set; }
-
-        [JsonProperty("x")]
-        public string ExecutionType { get; private set; }
-
-        [JsonProperty("X")]
-        public string OrderStatus { get; private set; }
-
-        [JsonProperty("T")]
-        public long TransactionTime { get; set; }
-
-        [JsonProperty("L")]
-        public decimal LastExecutedPrice { get; set; }
-
-        [JsonProperty("l")]
-        public decimal LastExecutedQuantity { get; set; }
-
-        [JsonProperty("S")]
-        public string Side { get; set; }
-
-        [JsonProperty("n")]
-        public decimal Fee { get; set; }
-
-        [JsonProperty("N")]
-        public string FeeCurrency { get; set; }
-
-        public OrderDirection Direction => Side.Equals("BUY", StringComparison.OrdinalIgnoreCase) ? OrderDirection.Buy : OrderDirection.Sell;
-    }
-
-    public class Kline
-    {
-        public long OpenTime { get; }
-        public decimal Open { get; }
-        public decimal Close { get; }
-        public decimal High { get; }
-        public decimal Low { get; }
-        public decimal Volume { get; }
-
-        public Kline() { }
-
-        public Kline(long msts, decimal close)
-        {
-            OpenTime = msts;
-            Open = Close = High = Low = close;
-            Volume = 0;
-        }
-
-        public Kline(object[] entries)
-        {
-            OpenTime = Convert.ToInt64(entries[0], CultureInfo.InvariantCulture);
-            Open = ((string)entries[1]).ToDecimal();
-            High = ((string)entries[2]).ToDecimal();
-            Low = ((string)entries[3]).ToDecimal();
-            Close = ((string)entries[4]).ToDecimal();
-            Volume = ((string)entries[5]).ToDecimal();
-        }
-    }
-
-#pragma warning restore 1591
-}

Brokerages/Bitfinex/BitfinexBrokerage.Messaging.cs

@@ -24,7 +24,6 @@ using QuantConnect.Orders;
 using QuantConnect.Orders.Fees;
 using QuantConnect.Packets;
 using QuantConnect.Securities;
-using QuantConnect.Securities.Crypto;
 using QuantConnect.Util;
 using RestSharp;
 using System;
@@ -49,7 +48,6 @@ namespace QuantConnect.Brokerages.Bitfinex
         private IAlgorithm _algorithm;
         private readonly RateGate _restRateLimiter = new RateGate(10, TimeSpan.FromMinutes(1));
         private readonly ConcurrentDictionary<int, decimal> _fills = new ConcurrentDictionary<int, decimal>();
-        private SymbolPropertiesDatabase _symbolPropertiesDatabase;
         private IDataAggregator _aggregator;
 
         // map Bitfinex ClientOrderId -> LEAN order (only used for orders submitted in PlaceOrder, not for existing orders)
@@ -188,7 +186,6 @@ namespace QuantConnect.Brokerages.Bitfinex
                 TimeSpan.Zero,
                 _connectionRateLimiter);
 
-            _symbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
             _algorithm = algorithm;
             _aggregator = aggregator;
 
@@ -473,27 +470,14 @@ namespace QuantConnect.Brokerages.Bitfinex
                         : OrderStatus.PartiallyFilled;
                 }
 
-                if (_algorithm.BrokerageModel.AccountType == AccountType.Cash &&
-                    order.Direction == OrderDirection.Buy)
+                if (_algorithm.BrokerageModel.AccountType == AccountType.Cash && order.Direction == OrderDirection.Buy)
                 {
-                    var symbolProperties = _symbolPropertiesDatabase.GetSymbolProperties(symbol.ID.Market,
-                        symbol,
-                        symbol.SecurityType,
-                        AccountBaseCurrency);
-                    Crypto.DecomposeCurrencyPair(symbol, symbolProperties, out var baseCurrency, out var _);
-
+                    CurrencyPairUtil.DecomposeCurrencyPair(symbol, out var baseCurrency, out _);
                     if (orderFee.Value.Currency != baseCurrency)
                     {
                         OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Error, "UnexpectedFeeCurrency", $"Unexpected fee currency {orderFee.Value.Currency} for symbol {symbol}. OrderId {order.Id}. BrokerageOrderId {brokerId}. " +
                             "This error can happen because your account is Margin type and Lean is configured to be Cash type or while using Cash type the Bitfinex account fee settings are set to 'Asset Trading Fee' and should be set to 'Currency Exchange Fee'."));
                     }
-                    else
-                    {
-                        // fees are debited in the base currency, so we have to subtract them from the filled quantity
-                        fillQuantity -= orderFee.Value.Amount;
-
-                        orderFee = new ModifiedFillQuantityOrderFee(orderFee.Value);
-                    }
                 }
 
                 var orderEvent = new OrderEvent

Brokerages/Bitfinex/BitfinexBrokerage.cs

@@ -23,7 +23,6 @@ using QuantConnect.Logging;
 using QuantConnect.Orders;
 using QuantConnect.Packets;
 using QuantConnect.Securities;
-using QuantConnect.Securities.Crypto;
 using QuantConnect.Util;
 using RestSharp;
 using System;
@@ -308,16 +307,7 @@ namespace QuantConnect.Brokerages.Bitfinex
                 foreach (var holding in GetAccountHoldings().Where(x => x.Symbol.SecurityType == SecurityType.Crypto))
                 {
                     var defaultQuoteCurrency = _algorithm.Portfolio.CashBook.AccountCurrency;
-
-                    var symbolProperties = _symbolPropertiesDatabase.GetSymbolProperties(
-                        holding.Symbol.ID.Market,
-                        holding.Symbol,
-                        holding.Symbol.SecurityType,
-                        defaultQuoteCurrency);
-
-                    string baseCurrency;
-                    string quoteCurrency;
-                    Crypto.DecomposeCurrencyPair(holding.Symbol, symbolProperties, out baseCurrency, out quoteCurrency);
+                    CurrencyPairUtil.DecomposeCurrencyPair(holding.Symbol, out var baseCurrency, out var quoteCurrency, defaultQuoteCurrency);
 
                     var baseQuantity = holding.Quantity;
                     CashAmount baseCurrencyAmount;
@@ -443,7 +433,7 @@ namespace QuantConnect.Brokerages.Bitfinex
             var apiKey = job.BrokerageData["bitfinex-api-key"];
             var apiSecret = job.BrokerageData["bitfinex-api-secret"];
             var aggregator = Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(
-                Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"));
+                Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false);
 
             Initialize(
                 wssUrl: WebSocketUrl,

Brokerages/Bitfinex/BitfinexBrokerageFactory.cs

@@ -84,7 +84,7 @@ namespace QuantConnect.Brokerages.Bitfinex
                 job.BrokerageData["bitfinex-api-secret"],
                 algorithm,
                 priceProvider,
-                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager")),
+                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false),
                 job);
             Composer.Instance.AddPart<IDataQueueHandler>(brokerage);
 

Brokerages/GDAX/GDAXBrokerageFactory.cs

@@ -91,7 +91,7 @@ namespace QuantConnect.Brokerages.GDAX
             var restClient = new RestClient(restApi);
             var webSocketClient = new WebSocketClientWrapper();
             var priceProvider = new ApiPriceProvider(job.UserId, job.UserToken);
-            var aggregator = Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"));
+            var aggregator = Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false);
 
             IBrokerage brokerage;
             if (job.DataQueueHandler.Contains("GDAXDataQueueHandler"))

Brokerages/GDAX/GDAXDataQueueHandler.cs

@@ -97,7 +97,7 @@ namespace QuantConnect.Brokerages.GDAX
             var apiSecret = job.BrokerageData["gdax-api-secret"];
             var priceProvider = new ApiPriceProvider(job.UserId, job.UserToken);
             var aggregator = Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(
-                Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"));
+                Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false);
 
             Initialize(
                 wssUrl: wssUrl,

Brokerages/InteractiveBrokers/InteractiveBrokersBrokerage.cs

@@ -55,7 +55,7 @@ namespace QuantConnect.Brokerages.InteractiveBrokers
         /// <summary>
         /// The default gateway version to use
         /// </summary>
-        public static string DefaultVersion { get; } = "985";
+        public static string DefaultVersion { get; } = "1012";
 
         private IBAutomater.IBAutomater _ibAutomater;
 
@@ -2614,7 +2614,7 @@ namespace QuantConnect.Brokerages.InteractiveBrokers
             Initialize(null,
                 null,
                 null,
-                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager")),
+                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false),
                 Composer.Instance.GetExportedValueByTypeName<IMapFileProvider>(Config.Get("map-file-provider", "QuantConnect.Data.Auxiliary.LocalDiskMapFileProvider")),
                 account,
                 host,

Brokerages/InteractiveBrokers/InteractiveBrokersBrokerageFactory.cs

@@ -102,7 +102,7 @@ namespace QuantConnect.Brokerages.InteractiveBrokers
                 algorithm,
                 algorithm.Transactions,
                 algorithm.Portfolio,
-                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager")),
+                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false),
                 Composer.Instance.GetExportedValueByTypeName<IMapFileProvider>(Config.Get("map-file-provider", "QuantConnect.Data.Auxiliary.LocalDiskMapFileProvider")),
                 account,
                 host,

Brokerages/Oanda/OandaBrokerage.cs

@@ -261,7 +261,7 @@ namespace QuantConnect.Brokerages.Oanda
             Initialize(
                 null,
                 null,
-                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager")),
+                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false),
                 environment,
                 accessToken,
                 accountId,

Brokerages/Oanda/OandaBrokerageFactory.cs

@@ -96,7 +96,7 @@ namespace QuantConnect.Brokerages.Oanda
             var brokerage = new OandaBrokerage(
                 algorithm.Transactions, 
                 algorithm.Portfolio,
-                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager")),
+                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false),
                 environment, 
                 accessToken, 
                 accountId, 

Brokerages/QuantConnect.Brokerages.csproj

@@ -37,7 +37,6 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
-    <PackageReference Include="NodaTime" Version="3.0.5" />
     <PackageReference Include="QuantConnect.IBAutomater" Version="2.0.64" />
     <PackageReference Include="RestSharp" Version="106.12.0" />
   </ItemGroup>

Brokerages/Tradier/TradierBrokerage.DataQueueHandler.cs

@@ -52,7 +52,7 @@ namespace QuantConnect.Brokerages.Tradier
             var useSandbox = bool.Parse(job.BrokerageData["tradier-use-sandbox"]);
             var accountId = job.BrokerageData["tradier-account-id"];
             var accessToken = job.BrokerageData["tradier-access-token"];
-            var aggregator = Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"));
+            var aggregator = Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false);
 
             Initialize(
                 wssUrl: WebSocketUrl,

Brokerages/Tradier/TradierBrokerage.cs

@@ -468,7 +468,8 @@ namespace QuantConnect.Brokerages.Tradier
             request.AddParameter("symbols", csvSymbols, ParameterType.QueryString);
 
             var dataContainer = Execute<TradierQuoteContainer>(request, TradierApiRequestType.Data, "quotes");
-            return dataContainer.Quotes;
+            // can return null quotes and not really be failing for cases where the provided symbols do not match
+            return dataContainer.Quotes ?? new List<TradierQuote>();
         }
 
         /// <summary>

Brokerages/Tradier/TradierBrokerageFactory.cs

@@ -101,7 +101,7 @@ namespace QuantConnect.Brokerages.Tradier
                 algorithm,
                 algorithm.Transactions,
                 algorithm.Portfolio,
-                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager")),
+                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false),
                 useSandbox,
                 accountId,
                 accessToken);

Brokerages/WebSocketClientWrapper.cs

@@ -148,7 +148,7 @@ namespace QuantConnect.Brokerages
         /// <summary>
         /// Wraps IsAlive
         /// </summary>
-        public bool IsOpen => _client != null && _client.State == WebSocketState.Open;
+        public bool IsOpen => _client?.State == WebSocketState.Open;
 
         /// <summary>
         /// Wraps message event

Brokerages/Zerodha/Constants.cs

@@ -1,120 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-
-namespace QuantConnect.Brokerages.Zerodha
-{
-#pragma warning disable 1591
-    /// <summary>
-    /// Types of product supported by Kite
-    /// </summary>
-    public enum KiteProductType
-    {
-        MIS,
-        CNC,
-        NRML
-    }
-
-    /// <summary>
-    /// Types of order supported by Kite
-    /// </summary>
-    public enum KiteOrderType
-    {
-        MARKET,
-        LIMIT,
-        SLM,
-        SL
-    }
-
-    public class Constants
-    {
-
-        // Products
-        public const string PRODUCT_MIS = "MIS";
-        public const string PRODUCT_CNC = "CNC";
-        public const string PRODUCT_NRML = "NRML";
-
-        // Order types
-        public const string ORDER_TYPE_MARKET = "MARKET";
-        public const string ORDER_TYPE_LIMIT = "LIMIT";
-        public const string ORDER_TYPE_SLM = "SL-M";
-        public const string ORDER_TYPE_SL = "SL";
-
-        // Order status
-        public const string ORDER_STATUS_COMPLETE = "COMPLETE";
-        public const string ORDER_STATUS_CANCELLED = "CANCELLED";
-        public const string ORDER_STATUS_REJECTED = "REJECTED";
-
-        // Varities
-        public const string VARIETY_REGULAR = "regular";
-        public const string VARIETY_BO = "bo";
-        public const string VARIETY_CO = "co";
-        public const string VARIETY_AMO = "amo";
-
-        // Transaction type
-        public const string TRANSACTION_TYPE_BUY = "BUY";
-        public const string TRANSACTION_TYPE_SELL = "SELL";
-
-        // Validity
-        public const string VALIDITY_DAY = "DAY";
-        public const string VALIDITY_IOC = "IOC";
-
-        // Exchanges
-        public const string EXCHANGE_NSE = "NSE";
-        public const string EXCHANGE_BSE = "BSE";
-        public const string EXCHANGE_NFO = "NFO";
-        public const string EXCHANGE_CDS = "CDS";
-        public const string EXCHANGE_BFO = "BFO";
-        public const string EXCHANGE_MCX = "MCX";
-
-        // Margins segments
-        public const string MARGIN_EQUITY = "equity";
-        public const string MARGIN_COMMODITY = "commodity";
-
-        // Ticker modes
-        public const string MODE_FULL = "full";
-        public const string MODE_QUOTE = "quote";
-        public const string MODE_LTP = "ltp";
-
-        // Positions
-        public const string POSITION_DAY = "day";
-        public const string POSITION_OVERNIGHT = "overnight";
-
-        // Historical intervals
-        public const string INTERVAL_MINUTE = "minute";
-        public const string INTERVAL_3MINUTE = "3minute";
-        public const string INTERVAL_5MINUTE = "5minute";
-        public const string INTERVAL_10MINUTE = "10minute";
-        public const string INTERVAL_15MINUTE = "15minute";
-        public const string INTERVAL_30MINUTE = "30minute";
-        public const string INTERVAL_60MINUTE = "60minute";
-        public const string INTERVAL_DAY = "day";
-
-        // GTT status
-        public const string GTT_ACTIVE = "active";
-        public const string GTT_TRIGGERED = "triggered";
-        public const string GTT_DISABLED = "disabled";
-        public const string GTT_EXPIRED = "expired";
-        public const string GTT_CANCELLED = "cancelled";
-        public const string GTT_REJECTED = "rejected";
-        public const string GTT_DELETED = "deleted";
-
-
-        // GTT trigger type
-        public const string GTT_TRIGGER_OCO = "two-leg";
-        public const string GTT_TRIGGER_SINGLE = "single";
-    }
-#pragma warning restore 1591
-}

Brokerages/Zerodha/ExceptionMessages.cs

@@ -1,60 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-
-namespace QuantConnect.Brokerages.Zerodha
-{
-    internal static partial class ExceptionExtensions
-    {
-
-        /// <summary>
-        /// Returns a list of all the exception messages from the top-level
-        /// exception down through all the inner exceptions. Useful for making
-        /// logs and error pages easier to read when dealing with exceptions.
-        /// Usage: Exception.Messages()
-        /// </summary>
-        public static IEnumerable<string> Messages(this Exception ex)
-        {
-            // return an empty sequence if the provided exception is null
-            if (ex == null) { yield break; }
-            // first return THIS exception's message at the beginning of the list
-            yield return ex.Message;
-            // then get all the lower-level exception messages recursively (if any)
-            IEnumerable<Exception> innerExceptions = Enumerable.Empty<Exception>();
-
-            if (ex is AggregateException && (ex as AggregateException).InnerExceptions.Any())
-            {
-                innerExceptions = (ex as AggregateException).InnerExceptions;
-            }
-            else if (ex.InnerException != null)
-            {
-                innerExceptions = new Exception[] { ex.InnerException };
-            }
-
-            foreach (var innerEx in innerExceptions)
-            {
-                foreach (string msg in innerEx.Messages())
-                {
-                    yield return msg;
-                }
-            }
-        }
-
-    }
-}

Brokerages/Zerodha/Exceptions.cs

@@ -1,87 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using System.Net;
-
-namespace QuantConnect.Brokerages.Zerodha
-{
-#pragma warning disable 1591
-    /// <summary>
-    /// KiteAPI Exceptions
-    /// </summary>
-    public class KiteException : Exception
-    {
-        HttpStatusCode status;
-        public KiteException(string message, HttpStatusCode httpStatus, Exception innerException = null) : base(message, innerException) { status = httpStatus; }
-    }
-
-    /// <summary>
-    /// General Exceptions
-    /// </summary>
-    public class GeneralException : KiteException
-    {
-        public GeneralException(string message, HttpStatusCode httpStatus = HttpStatusCode.InternalServerError, Exception innerException = null) : base(message, httpStatus, innerException) { }
-    }
-
-    /// <summary>
-    /// Token Exceptions
-    /// </summary>
-    public class TokenException : KiteException
-    {
-        public TokenException(string message, HttpStatusCode httpStatus = HttpStatusCode.Forbidden, Exception innerException = null) : base(message, httpStatus, innerException) { }
-    }
-
-    /// <summary>
-    /// Permission Exceptions
-    /// </summary>
-    public class PermissionException : KiteException
-    {
-        public PermissionException(string message, HttpStatusCode httpStatus = HttpStatusCode.Forbidden, Exception innerException = null) : base(message, httpStatus, innerException) { }
-    }
-
-    /// <summary>
-    /// Order Exceptions
-    /// </summary>
-    public class OrderException : KiteException
-    {
-        public OrderException(string message, HttpStatusCode httpStatus = HttpStatusCode.BadRequest, Exception innerException = null) : base(message, httpStatus, innerException) { }
-    }
-
-    /// <summary>
-    /// InputExceptions
-    /// </summary>
-    public class InputException : KiteException
-    {
-        public InputException(string message, HttpStatusCode httpStatus = HttpStatusCode.BadRequest, Exception innerException = null) : base(message, httpStatus, innerException) { }
-    }
-
-    /// <summary>
-    /// DataExceptions 
-    /// </summary>
-    public class DataException : KiteException
-    {
-        public DataException(string message, HttpStatusCode httpStatus = HttpStatusCode.BadGateway, Exception innerException = null) : base(message, httpStatus, innerException) { }
-    }
-
-    /// <summary>
-    /// Network Exceptions
-    /// </summary>
-    public class NetworkException : KiteException
-    {
-        public NetworkException(string message, HttpStatusCode httpStatus = HttpStatusCode.ServiceUnavailable, Exception innerException = null) : base(message, httpStatus, innerException) { }
-    }
-#pragma warning restore 1591
-}

Brokerages/Zerodha/Utils.cs

@@ -1,189 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-
-using System.IO;
-using System.Web;
-using System.Text.RegularExpressions;
-using Newtonsoft.Json;
-using System.Globalization;
-using Newtonsoft.Json.Linq;
-
-namespace QuantConnect.Brokerages.Zerodha
-{
-    /// <summary>
-    /// Zerodha utility class
-    /// </summary>
-    public class Utils
-    {
-        /// <summary>
-        /// Convert string to Date object
-        /// </summary>
-        /// <param name="dateString">Date string.</param>
-        /// <returns>Date object/</returns>
-        public static DateTime? StringToDate(string dateString)
-        {
-            if (dateString == null)
-            {
-                return null;
-            }
-
-            try
-            {
-                return DateTime.ParseExact(dateString, dateString.Length == 10 ? "yyyy-MM-dd" : "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
-            }
-            catch (Exception)
-            {
-                return null;
-            }
-        }
-
-        /// <summary>
-        /// Serialize C# object to JSON string.
-        /// </summary>
-        /// <param name="obj">C# object to serialize.</param>
-        /// <returns>JSON string/</returns>
-        public static string JsonSerialize(object obj)
-        {
-            string json = JsonConvert.SerializeObject(obj);
-            MatchCollection mc = Regex.Matches(json, @"\\/Date\((\d*?)\)\\/");
-            foreach (Match m in mc)
-            {
-                var unix = Convert.ToInt64(m.Groups[1].Value,CultureInfo.InvariantCulture) / 1000;
-                json = json.Replace(m.Groups[0].Value, UnixToDateTime(unix).ToStringInvariant());
-            }
-            return json;
-        }
-
-        /// <summary>
-        /// Deserialize Json string to nested string dictionary.
-        /// </summary>
-        /// <param name="Json">Json string to deserialize.</param>
-        /// <returns>Json in the form of nested string dictionary.</returns>
-        public static JObject JsonDeserialize(string Json)
-        {
-            JObject jObject = JObject.Parse(Json);
-            return jObject;
-        }
-
-        /// <summary>
-        /// Recursively traverses an object and converts double fields to decimal.
-        /// This is used in Json deserialization. JavaScriptSerializer converts floats
-        /// in exponential notation to double and everthing else to double. This function
-        /// makes everything decimal. Currently supports only Dictionary and Array as input.
-        /// </summary>
-        /// <param name="obj">Input object.</param>
-        /// <returns>Object with decimals instead of doubles</returns>
-        public static dynamic DoubleToDecimal(dynamic obj)
-        {
-            if (obj is double)
-            {
-                obj = Convert.ToDecimal(obj);
-            }
-            else if (obj is IDictionary)
-            {
-                var keys = new List<string>(obj.Keys);
-                for (int i = 0; i < keys.Count; i++)
-                {
-                    obj[keys[i]] = DoubleToDecimal(obj[keys[i]]);
-                }
-            }
-            else if (obj is ICollection)
-            {
-                obj = new ArrayList(obj);
-                for (int i = 0; i < obj.Count; i++)
-                {
-                    obj[i] = DoubleToDecimal(obj[i]);
-                }
-            }
-            return obj;
-        }
-
-        /// <summary>
-        /// Wraps a string inside a stream
-        /// </summary>
-        /// <param name="value">string data</param>
-        /// <returns>Stream that reads input string</returns>
-        public static MemoryStream StreamFromString(string value)
-        {
-            return new MemoryStream(Encoding.UTF8.GetBytes(value ?? ""));
-        }
-
-        /// <summary>
-        /// Helper function to add parameter to the request only if it is not null or empty
-        /// </summary>
-        /// <param name="Params">Dictionary to add the key-value pair</param>
-        /// <param name="Key">Key of the parameter</param>
-        /// <param name="Value">Value of the parameter</param>
-        public static void AddIfNotNull(Dictionary<string, dynamic> Params, string Key, string Value)
-        {
-            if (!String.IsNullOrEmpty(Value))
-                Params.Add(Key, Value);
-        }
-
-
-        /// <summary>
-        /// Creates key=value with url encoded value
-        /// </summary>
-        /// <param name="Key">Key</param>
-        /// <param name="Value">Value</param>
-        /// <returns>Combined string</returns>
-        public static string BuildParam(string Key, dynamic Value)
-        {
-            if (Value is string)
-            {
-                return HttpUtility.UrlEncode(Key) + "=" + HttpUtility.UrlEncode((string)Value);
-            }
-            else
-            {
-                string[] values = (string[])Value;
-                return String.Join("&", values.Select(x => HttpUtility.UrlEncode(Key) + "=" + HttpUtility.UrlEncode(x)));
-            }
-        }
-
-        /// <summary>
-        /// Convert Unix TimeStamp to DateTime
-        /// </summary>
-        /// <param name="unixTimeStamp">Timestamp to convert</param>
-        /// <returns><see cref="DateTime"/> object representing the timestamp</returns>
-        public static DateTime UnixToDateTime(long unixTimeStamp)
-        {
-            // Unix timestamp is seconds past epoch
-            DateTime dateTime = new DateTime(1970, 1, 1, 5, 30, 0, 0, DateTimeKind.Unspecified);
-            dateTime = dateTime.AddSeconds(unixTimeStamp);
-            return dateTime;
-        }
-
-        /// <summary>
-        /// Convert ArrayList to list of <see cref="decimal"/>
-        /// </summary>
-        /// <param name="arrayList"><see cref="ArrayList"/> to convert</param>
-        /// <returns>List of <see cref="decimal"/>s</returns>
-        public static List<decimal> ToDecimalList(ArrayList arrayList)
-        {
-            var res = new List<decimal>();
-            foreach(var i in arrayList)
-            {
-                res.Add(Convert.ToDecimal(i,CultureInfo.InvariantCulture));
-            }
-            return res;
-        }
-    }
-}

Brokerages/Zerodha/ZerodhaBrokerage.DataQueueHandler.cs

@@ -1,102 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using QuantConnect.Configuration;
-using QuantConnect.Data;
-using QuantConnect.Packets;
-using QuantConnect.Util;
-using System;
-using System.Collections.Generic;
-
-namespace QuantConnect.Brokerages.Zerodha
-{
-    /// <summary>
-    /// ZerodhaBrokerage: IDataQueueHandler implementation
-    /// </summary>
-    public partial class ZerodhaBrokerage
-    {
-        #region IDataQueueHandler implementation
-
-        /// <summary>
-        /// Sets the job we're subscribing for
-        /// </summary>
-        /// <param name="job">Job we're subscribing for</param>
-        public void SetJob(LiveNodePacket job)
-        {
-            Initialize(
-                job.BrokerageData["zerodha-trading-segment"],
-                job.BrokerageData["zerodha-product-type"],
-                job.BrokerageData["zerodha-api-key"],
-                job.BrokerageData["zerodha-access-token"],
-                null,
-                null,
-                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"))
-            );
-
-            if (!IsConnected)
-            {
-                Connect();
-            }
-        }
-
-        /// <summary>
-        /// Subscribe to the specified configuration
-        /// </summary>
-        /// <param name="dataConfig">defines the parameters to subscribe to a data feed</param>
-        /// <param name="newDataAvailableHandler">handler to be fired on new data available</param>
-        /// <returns>The new enumerator for this subscription request</returns>
-        public IEnumerator<BaseData> Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler)
-        {
-            var symbol = dataConfig.Symbol;
-            if (!CanSubscribe(symbol))
-            {
-                return null;
-            }
-
-            var enumerator = _aggregator.Add(dataConfig, newDataAvailableHandler);
-            SubscriptionManager.Subscribe(dataConfig);
-
-            return enumerator;
-        }
-
-        /// <summary>
-        /// UnSubscribe to the specified configuration
-        /// </summary>
-        /// <param name="dataConfig">defines the parameters to subscribe to a data feed</param>
-        public void Unsubscribe(SubscriptionDataConfig dataConfig)
-        {
-            SubscriptionManager.Unsubscribe(dataConfig);
-            _aggregator.Remove(dataConfig);
-        }
-
-        /// <summary>
-        /// Returns true if this data provide can handle the specified symbol
-        /// </summary>
-        /// <param name="symbol">The symbol to be handled</param>
-        /// <returns>True if this data provider can get data for the symbol, false otherwise</returns>
-        private static bool CanSubscribe(Symbol symbol)
-        {
-            var market = symbol.ID.Market;
-            var securityType = symbol.ID.SecurityType;
-            if (symbol.Value.IndexOfInvariant("universe", true) != -1) return false;
-            // Include future options as a special case with no matching market, otherwise
-            // our subscriptions are removed without any sort of notice.
-            return
-                (securityType == SecurityType.Equity) && (market == Market.India);
-        }
-
-        #endregion IDataQueueHandler implementation
-    }
-}

Brokerages/Zerodha/ZerodhaBrokerageFactory.cs

@@ -1,92 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using System.Collections.Generic;
-using QuantConnect.Configuration;
-using QuantConnect.Data;
-using QuantConnect.Interfaces;
-using QuantConnect.Securities;
-using QuantConnect.Util;
-
-namespace QuantConnect.Brokerages.Zerodha
-{
-    /// <summary>
-    /// Factory method to create Zerodha Websockets brokerage
-    /// </summary>
-    public class ZerodhaBrokerageFactory : BrokerageFactory
-    {
-        /// <summary>
-        /// Factory constructor
-        /// </summary>
-        public ZerodhaBrokerageFactory() : base(typeof(ZerodhaBrokerage))
-        {
-        }
-
-        /// <summary>
-        /// Not required
-        /// </summary>
-        public override void Dispose()
-        {
-        }
-
-        /// <summary>
-        /// provides brokerage connection data
-        /// </summary>
-        public override Dictionary<string, string> BrokerageData => new Dictionary<string, string>
-        {
-            { "zerodha-api-key", Config.Get("zerodha-api-key")},
-            { "zerodha-access-token", Config.Get("zerodha-access-token")},
-            { "zerodha-trading-segment", Config.Get("zerodha-trading-segment")},
-            { "zerodha-product-type", Config.Get("zerodha-product-type")},
-        };
-
-        /// <summary>
-        /// The brokerage model
-        /// </summary>
-        /// <param name="orderProvider">The order provider</param>
-        public override IBrokerageModel GetBrokerageModel(IOrderProvider orderProvider) => new ZerodhaBrokerageModel();
-
-        /// <summary>
-        /// Create the Brokerage instance
-        /// </summary>
-        /// <param name="job"></param>
-        /// <param name="algorithm"></param>
-        /// <returns></returns>
-        public override IBrokerage CreateBrokerage(Packets.LiveNodePacket job, IAlgorithm algorithm)
-        {
-            var required = new[] { "zerodha-api-key", "zerodha-access-token", "zerodha-trading-segment"};
-
-            foreach (var item in required)
-            {
-                if (string.IsNullOrEmpty(job.BrokerageData[item]))
-                    throw new Exception($"ZerodhaBrokerageFactory.CreateBrokerage: Missing {item} in config.json");
-            }
-
-            var brokerage = new ZerodhaBrokerage(
-                job.BrokerageData["zerodha-trading-segment"],
-                job.BrokerageData["zerodha-product-type"],
-                job.BrokerageData["zerodha-api-key"],
-                job.BrokerageData["zerodha-access-token"],
-                algorithm,
-                algorithm.Portfolio,
-                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"))
-               );
-            //Add the brokerage to the composer to ensure its accessible to the live data feed.
-            Composer.Instance.AddPart<IDataQueueHandler>(brokerage);
-            return brokerage;
-        }
-    }
-}

Brokerages/Zerodha/ZerodhaSymbolMapper.cs

@@ -1,393 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using System.Collections.Generic;
-using System.Globalization;
-using System.Linq;
-using QuantConnect.Brokerages.Zerodha.Messages;
-using QuantConnect.Util;
-
-namespace QuantConnect.Brokerages.Zerodha
-{
-    /// <summary>
-    /// Provides the mapping between Lean symbols and Zerodha symbols.
-    /// </summary>
-    public class ZerodhaSymbolMapper : ISymbolMapper
-    {
-
-        /// <summary>
-        /// Symbols that are Tradable
-        /// </summary>
-        public List<Symbol> KnownSymbols
-        {
-            get
-            {
-                return KnownSymbolsList;
-            }
-        }
-
-        /// <summary>
-        /// Custom class to store information about symbols
-        /// </summary>
-        private class SymbolData 
-        { 
-            /// <summary>
-            /// Stores exchange name for the tradingSymbol
-            /// </summary>
-            public string Exchange { get; set;}
-
-            /// <summary>
-            /// Stores instrumentToken name for the tradingSymbol
-            /// </summary>
-            public uint InstrumentToken {get; set;}
-
-            /// <summary>
-            /// Initalize values to the class attributes
-            /// </summary>
-            public SymbolData(uint token, string exchangeName)
-            {
-                Exchange = exchangeName;
-                InstrumentToken = token;
-            }
-        }
-
-        /// <summary>
-        /// The list of known Zerodha symbols.
-        /// </summary>
-        private List<Symbol> KnownSymbolsList = new List<Symbol>();
-
-        /// <summary>
-        /// Mapping between brokerageSymbol and a list of all available SymbolData objects for the brokerageSymbol.
-        /// </summary>
-        private Dictionary<string, List<SymbolData>> ZerodhaInstrumentsList = new Dictionary<string, List<SymbolData>>();
-
-        /// <summary>
-        /// Mapping between instrumentToken and it's market segment ( E.g: 408065-> nse)
-        /// </summary>
-        private Dictionary<uint,string> ZerodhaInstrumentsExchangeMapping = new Dictionary<uint,string>();
-
-        /// <summary>
-        /// Constructs default instance of the Zerodha Sybol Mapper
-        /// </summary>
-        public ZerodhaSymbolMapper(Kite kite, string exchange = "")
-        {
-            KnownSymbolsList = GetTradableInstrumentsList(kite, exchange);
-        }
-        /// <summary>
-        /// Get list of tradable symbol
-        /// </summary>
-        /// <param name="kite">Kite</param>
-        /// <param name="exchange">Exchange</param>
-        /// <returns></returns>
-        private List<Symbol> GetTradableInstrumentsList(Kite kite, string exchange = "")
-        {
-
-            var tradableInstruments = kite.GetInstruments(exchange);
-            var symbols = new List<Symbol>();
-            var zerodhaInstrumentsMapping = new Dictionary<string, List<SymbolData>>();
-            var zerodhaTokenExchangeDict = new Dictionary<uint,string>();
-
-            foreach (var tp in tradableInstruments)
-            {
-                var securityType = SecurityType.Equity;
-                var market = Market.India;
-                zerodhaTokenExchangeDict[tp.InstrumentToken] = tp.Exchange.ToLowerInvariant();
-                OptionRight optionRight = 0;
-
-                switch (tp.InstrumentType)
-                {
-                    //Equities
-                    case "EQ":
-                        securityType = SecurityType.Equity;
-                        break;
-                    //Call Options
-                    case "CE":
-                        securityType = SecurityType.Option;
-                        optionRight = OptionRight.Call;
-                        break;
-                    //Put Options
-                    case "PE":
-                        securityType = SecurityType.Option;
-                        optionRight = OptionRight.Put;
-                        break;
-                    //Stock Futures
-                    case "FUT":
-                        securityType = SecurityType.Future;
-                        break;
-                    default:
-                        securityType = SecurityType.Base;
-                        break;
-                }
-                
-                if (securityType == SecurityType.Option)
-                {
-                    var strikePrice = tp.Strike;
-                    var expiryDate = tp.Expiry;
-                    //TODO: Handle parsing of BCDOPT strike price
-                    if(tp.Segment!= "BCD-OPT")
-                    {
-                        var symbol = GetLeanSymbol(tp.Name.Trim().Replace(" ", ""), securityType, market, (DateTime)expiryDate, GetStrikePrice(tp), optionRight);
-                        symbols.Add(symbol);
-                        var cleanSymbol = tp.TradingSymbol.Trim().Replace(" ", "");
-                        if (!zerodhaInstrumentsMapping.ContainsKey(cleanSymbol))
-                        {
-                            zerodhaInstrumentsMapping[cleanSymbol] = new List<SymbolData>();
-                        }
-                        zerodhaInstrumentsMapping[cleanSymbol].Add(new SymbolData(tp.InstrumentToken,market));
-                    }                    
-                }
-                if (securityType == SecurityType.Future)
-                {
-                    var expiryDate = tp.Expiry;
-                    var cleanSymbol = tp.TradingSymbol.Trim().Replace(" ", "");
-                    var symbol = GetLeanSymbol(cleanSymbol, securityType, market, (DateTime)expiryDate);
-                    symbols.Add(symbol);
-                    if (!zerodhaInstrumentsMapping.ContainsKey(cleanSymbol))
-                    {
-                        zerodhaInstrumentsMapping[cleanSymbol] = new List<SymbolData>();
-                    }
-                    zerodhaInstrumentsMapping[cleanSymbol].Add(new SymbolData(tp.InstrumentToken,market));
-                }
-                if (securityType == SecurityType.Equity)
-                {
-                    var cleanSymbol = tp.TradingSymbol.Trim().Replace(" ", "");
-                    var symbol = GetLeanSymbol(cleanSymbol, securityType, market);
-                    symbols.Add(symbol);
-                    if (!zerodhaInstrumentsMapping.ContainsKey(cleanSymbol))
-                    {
-                        zerodhaInstrumentsMapping[cleanSymbol] = new List<SymbolData>();
-                    }
-                    zerodhaInstrumentsMapping[cleanSymbol].Add(new SymbolData(tp.InstrumentToken,market));
-                }
-            }
-            ZerodhaInstrumentsList = zerodhaInstrumentsMapping;
-            ZerodhaInstrumentsExchangeMapping = zerodhaTokenExchangeDict;
-            return symbols;
-        }
-
-        private decimal GetStrikePrice(CsvInstrument scrip)
-        {
-            var strikePrice = scrip.TradingSymbol.Trim().Replace(" ", "").Replace(scrip.Name, "");
-            var strikePriceTemp = strikePrice.Substring(5, strikePrice.Length - 5);
-            var strikePriceResult = strikePriceTemp.Substring(0, strikePriceTemp.Length - 2);
-
-            return Convert.ToDecimal(strikePriceResult, CultureInfo.InvariantCulture);
-        }
-
-        /// <summary>
-        /// Converts a Lean symbol instance to an Zerodha symbol
-        /// </summary>
-        /// <param name="symbol">A Lean symbol instance</param>
-        /// <returns>The Zerodha symbol</returns>
-        public string GetBrokerageSymbol(Symbol symbol)
-        {
-            if (symbol == null || string.IsNullOrWhiteSpace(symbol.Value))
-            {
-                throw new ArgumentException("Invalid symbol: " + (symbol == null ? "null" : symbol.ToString()));
-            }
-
-            if (symbol.ID.SecurityType != SecurityType.Equity && symbol.ID.SecurityType != SecurityType.Future && symbol.ID.SecurityType != SecurityType.Option)
-            {
-                throw new ArgumentException("Invalid security type: " + symbol.ID.SecurityType);
-            }
-
-            var brokerageSymbol = ConvertLeanSymbolToZerodhaSymbol(symbol.Value);
-
-            return brokerageSymbol;
-        }
-
-
-        /// <summary>
-        /// Converts an Zerodha symbol to a Lean symbol instance
-        /// </summary>
-        /// <param name="brokerageSymbol">The Zerodha symbol</param>
-        /// <param name="securityType">The security type</param>
-        /// <param name="market">The market</param>
-        /// <param name="expirationDate">Expiration date of the security(if applicable)</param>
-        /// <param name="strike">The strike of the security (if applicable)</param>
-        /// <param name="optionRight">The option right of the security (if applicable)</param>
-        /// <returns>A new Lean Symbol instance</returns>
-        public Symbol GetLeanSymbol(string brokerageSymbol, SecurityType securityType, string market, DateTime expirationDate = default(DateTime), decimal strike = 0, OptionRight optionRight = OptionRight.Call)
-        {
-            if (string.IsNullOrWhiteSpace(brokerageSymbol))
-            {
-                throw new ArgumentException($"Invalid Zerodha symbol: {brokerageSymbol}");
-            }
-
-            if (securityType == SecurityType.Forex || securityType == SecurityType.Cfd || securityType == SecurityType.Commodity || securityType == SecurityType.Crypto)
-            {
-                throw new ArgumentException($"Invalid security type: {securityType}");
-            }
-
-            if (!Market.Encode(market).HasValue)
-            {
-                throw new ArgumentException($"Invalid market: {market}");
-            }
-            var cleanSymbol = brokerageSymbol.Replace(" ", "").Trim();
-
-            switch (securityType)
-            {
-                case SecurityType.Option:
-                    OptionStyle optionStyle = OptionStyle.European;
-                    return Symbol.CreateOption(cleanSymbol, market, optionStyle, optionRight, strike, expirationDate);
-                case SecurityType.Future:
-                    return Symbol.CreateFuture(cleanSymbol, market, expirationDate);
-                default:
-                    return Symbol.Create(cleanSymbol, securityType, market);
-            }
-
-        }
-
-
-        /// <summary>
-        /// Converts an Zerodha symbol to a Lean symbol instance
-        /// </summary>
-        /// <param name="brokerageSymbol">The Zerodha symbol</param>
-        /// <returns>A new Lean Symbol instance</returns>
-        public Symbol GetLeanSymbol(string brokerageSymbol)
-        {
-            if (string.IsNullOrWhiteSpace(brokerageSymbol))
-            {
-                throw new ArgumentException($"Invalid Zerodha symbol: {brokerageSymbol}");
-            }
-
-            var cleanSymbol = brokerageSymbol.Replace(" ", "").Trim();
-
-            if (IsKnownBrokerageSymbol(cleanSymbol))
-            {
-                throw new ArgumentException($"Symbol not present : {cleanSymbol}");
-            }
-
-            var symbol = KnownSymbols.FirstOrDefault(s => s.Value == cleanSymbol);
-            var exchange = GetZerodhaDefaultExchange(cleanSymbol);
-            return GetLeanSymbol(cleanSymbol, symbol.SecurityType, exchange);
-        }
-
-        /// <summary>
-        /// Fetches the trading segment inside India Market, E.g: NSE, BSE for the given Instrument Token
-        /// </summary>
-        /// <param name="Token">The Zerodha Instrument Token</param>
-        /// <returns>An exchange value for the given token</returns>
-        public string GetZerodhaExchangeFromToken(uint Token)
-        {   
-            string exchange = string.Empty;
-            if (ZerodhaInstrumentsExchangeMapping.ContainsKey(Token))
-            {
-                ZerodhaInstrumentsExchangeMapping.TryGetValue(Token, out exchange);
-            }
-            return exchange;
-        }
-        
-
-        /// <summary>
-        /// Fetches the first available Exchage value for the given symbol from list of possible exchanges
-        /// </summary>
-        /// <param name="brokerageSymbol">The Zerodha symbol</param>
-        /// <returns>A default exchange value for the given ticker</returns>
-        private string GetZerodhaDefaultExchange(string brokerageSymbol)
-        {   
-            if (string.IsNullOrWhiteSpace(brokerageSymbol))
-            {
-                throw new ArgumentException($"Invalid Zerodha symbol: {brokerageSymbol}");
-            }
-
-            var cleanSymbol = brokerageSymbol.Replace(" ", "").Trim();
-
-            List<SymbolData> tempSymbolDataList;
-            if (ZerodhaInstrumentsList.TryGetValue(cleanSymbol, out tempSymbolDataList))
-            {   
-                return tempSymbolDataList[0].Exchange;
-            }
-            return string.Empty;
-        }
-
-        /// <summary>
-        /// Converts Lean symbol to a List of Zerodha Instrument Tokens available from various exchange
-        /// </summary>
-        /// <param name="brokerageSymbol">The Zerodha symbol</param>
-        /// <returns>A list of Zerodha Instrument Tokens</returns>
-        public List<uint> GetZerodhaInstrumentTokenList(string brokerageSymbol)
-        {
-            if (string.IsNullOrWhiteSpace(brokerageSymbol))
-            {
-                throw new ArgumentException($"Invalid Zerodha symbol: {brokerageSymbol}");
-            }
-
-            var cleanSymbol = brokerageSymbol.Replace(" ", "").Trim();
-
-            List<uint> tokenList = new List<uint>();
-            List<SymbolData> tempSymbolDataList;
-            if (ZerodhaInstrumentsList.TryGetValue(cleanSymbol, out tempSymbolDataList))
-            {
-                foreach (var sd in tempSymbolDataList)
-                {
-                    tokenList.Add(sd.InstrumentToken);
-                }
-            }
-            return tokenList;
-        }
-
-        /// <summary>
-        /// Checks if the symbol is supported by Zerodha
-        /// </summary>
-        /// <param name="brokerageSymbol">The Zerodha symbol</param>
-        /// <returns>True if Zerodha supports the symbol</returns>
-        private bool IsKnownBrokerageSymbol(string brokerageSymbol)
-        {
-            if (string.IsNullOrWhiteSpace(brokerageSymbol))
-            {
-                return false;
-            }
-
-            return KnownSymbolsList.Where(x => x.Value.Contains(brokerageSymbol)).IsNullOrEmpty();
-        }
-
-        /// <summary>
-        /// Converts an Zerodha symbol to a Lean symbol string
-        /// </summary>
-        public Symbol ConvertZerodhaSymbolToLeanSymbol(uint ZerodhaSymbol)
-        {
-            var _symbol = string.Empty;
-            foreach (var item in ZerodhaInstrumentsList)
-            {
-                foreach( var sd in item.Value) 
-                { 
-                    if (sd.InstrumentToken == ZerodhaSymbol) 
-                    {
-                        _symbol = item.Key;
-                        break;
-                    }
-                }
-            }
-            // return as it is due to Zerodha has similar Symbol format
-            return KnownSymbolsList.Where(s => s.Value == _symbol).FirstOrDefault();
-        }
-
-        /// <summary>
-        /// Converts a Lean symbol string to an Zerodha symbol
-        /// </summary>
-        private static string ConvertLeanSymbolToZerodhaSymbol(string leanSymbol)
-        {
-            if (string.IsNullOrWhiteSpace(leanSymbol))
-            {
-                throw new ArgumentException($"Invalid Lean symbol: {leanSymbol}");
-            }
-
-            // return as it is due to Zerodha has similar Symbol format
-            return leanSymbol.ToUpperInvariant();
-        }
-    }
-}

Common/AlgorithmImports.py

@@ -40,6 +40,7 @@ from QuantConnect.Python import *
 from QuantConnect.Storage import *
 from QuantConnect.Research import *
 from QuantConnect.Algorithm import *
+from QuantConnect.Statistics import *
 from QuantConnect.Parameters import *
 from QuantConnect.Benchmarks import *
 from QuantConnect.Brokerages import *
@@ -69,6 +70,7 @@ from QuantConnect.Algorithm.Framework import *
 from QuantConnect.Securities.Volatility import *
 from QuantConnect.Securities.Interfaces import *
 from QuantConnect.Data.UniverseSelection import *
+from QuantConnect.Data.Custom.IconicTypes import *
 from QuantConnect.Data.Custom.AlphaStreams import *
 from QuantConnect.Algorithm.Framework.Risk import *
 from QuantConnect.Algorithm.Framework.Alphas import *

Common/Api/ParameterSetJsonConverter.cs

@@ -66,13 +66,25 @@ namespace QuantConnect.Api
         /// </returns>
         public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
         {
-            var jObject = JObject.Load(reader);
+            if (reader.TokenType == JsonToken.StartArray)
+            {
+                if (JArray.Load(reader).Count == 0)
+                {
+                    return new ParameterSet(-1, new Dictionary<string, string>());
+                }
+            }
+            else if (reader.TokenType == JsonToken.StartObject)
+            {
+                var jObject = JObject.Load(reader);
 
-            var value = jObject["parameterSet"] ?? jObject;
+                var value = jObject["parameterSet"] ?? jObject;
 
-            var parameterSet = new ParameterSet(-1, value.ToObject<Dictionary<string, string>>());
+                var parameterSet = new ParameterSet(-1, value.ToObject<Dictionary<string, string>>());
 
-            return parameterSet;
+                return parameterSet;
+            }
+
+            throw new ArgumentException($"Unexpected Tokentype {reader.TokenType}");
         }
     }
 }

Common/Brokerages/BinanceBrokerageModel.cs

@@ -18,7 +18,6 @@ using QuantConnect.Orders;
 using QuantConnect.Orders.Fees;
 using QuantConnect.Securities;
 using QuantConnect.Util;
-using System;
 using System.Collections.Generic;
 using System.Linq;
 using static QuantConnect.StringExtensions;
@@ -30,6 +29,8 @@ namespace QuantConnect.Brokerages
     /// </summary>
     public class BinanceBrokerageModel : DefaultBrokerageModel
     {
+        private const decimal _defaultLeverage = 3;
+
         /// <summary>
         /// Gets a map of the default markets to be used for each security type
         /// </summary>
@@ -41,22 +42,20 @@ namespace QuantConnect.Brokerages
         /// <param name="accountType">The type of account to be modeled, defaults to <see cref="AccountType.Cash"/></param>
         public BinanceBrokerageModel(AccountType accountType = AccountType.Cash) : base(accountType)
         {
-            if (accountType == AccountType.Margin)
-            {
-                throw new ArgumentException("The Binance brokerage does not currently support Margin trading.");
-            }
         }
 
         /// <summary>
         /// Gets a new buying power model for the security, returning the default model with the security's configured leverage.
         /// For cash accounts, leverage = 1 is used.
-        /// Margin trading is not currently supported
+        /// For standard account margin trading the leverage is 3x, leverage 5x only supported in the master account
         /// </summary>
         /// <param name="security">The security to get a buying power model for</param>
         /// <returns>The buying power model for this brokerage/security</returns>
         public override IBuyingPowerModel GetBuyingPowerModel(Security security)
         {
-            return new CashBuyingPowerModel();
+            return AccountType == AccountType.Cash
+                ? new CashBuyingPowerModel()
+                : new SecurityMarginModel(GetLeverage(security));
         }
 
         /// <summary>
@@ -66,8 +65,12 @@ namespace QuantConnect.Brokerages
         /// <returns></returns>
         public override decimal GetLeverage(Security security)
         {
-            // margin trading is not currently supported by Binance
-            return 1m;
+            if (AccountType == AccountType.Cash || security.IsInternalFeed() || security.Type == SecurityType.Base)
+            {
+                return 1m;
+            }
+
+            return _defaultLeverage;
         }
 
         /// <summary>

Common/Brokerages/BrokerageName.cs

@@ -95,7 +95,7 @@ namespace QuantConnect.Brokerages
         /// Transaction and submit/execution rules will use TradingTechnologies models
         /// </summary>
         TradingTechnologies,
-        
+
         /// <summary>
         /// Transaction and submit/execution rules will use Kraken models
         /// </summary>
@@ -104,6 +104,16 @@ namespace QuantConnect.Brokerages
         /// <summary>
         /// Transaction and submit/execution rules will use ftx models
         /// </summary>
-        FTX
+        FTX,
+
+        /// <summary>
+        /// Transaction and submit/execution rules will use ftx us models
+        /// </summary>
+        FTXUS,
+
+        /// <summary>
+        /// Transaction and submit/execution rules will use Exante models
+        /// </summary>
+        Exante,
     }
 }

Common/Brokerages/ExanteBrokerageModel.cs

@@ -0,0 +1,132 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Reflection;
+using System.Reflection.Emit;
+using QuantConnect.Benchmarks;
+using QuantConnect.Orders;
+using QuantConnect.Orders.Fees;
+using QuantConnect.Securities;
+using static QuantConnect.StringExtensions;
+using static QuantConnect.Util.SecurityExtensions;
+
+namespace QuantConnect.Brokerages
+{
+    /// <summary>
+    /// Exante Brokerage Model Implementation for Back Testing.
+    /// </summary>
+    public class ExanteBrokerageModel : DefaultBrokerageModel
+    {
+        private const decimal EquityLeverage = 1.2m;
+
+        /// <summary>
+        /// Constructor for Exante brokerage model
+        /// </summary>
+        /// <param name="accountType">Cash or Margin</param>
+        public ExanteBrokerageModel(AccountType accountType = AccountType.Cash)
+            : base(accountType)
+        {
+        }
+
+        /// <summary>
+        /// Get the benchmark for this model
+        /// </summary>
+        /// <param name="securities">SecurityService to create the security with if needed</param>
+        /// <returns>The benchmark for this brokerage</returns>
+        public override IBenchmark GetBenchmark(SecurityManager securities)
+        {
+            var symbol = Symbol.Create("SPY", SecurityType.Equity, Market.USA);
+            return SecurityBenchmark.CreateInstance(securities, symbol);
+        }
+
+        /// <summary>
+        /// Returns true if the brokerage could accept this order. This takes into account
+        /// order type, security type, and order size limits.
+        /// </summary>
+        /// <remarks>
+        /// For example, a brokerage may have no connectivity at certain times, or an order rate/size limit
+        /// </remarks>
+        /// <param name="security">The security being ordered</param>
+        /// <param name="order">The order to be processed</param>
+        /// <param name="message">If this function returns false, a brokerage message detailing why the order may not be submitted</param>
+        /// <returns>True if the brokerage could process the order, false otherwise</returns>
+        public override bool CanSubmitOrder(Security security, Order order, out BrokerageMessageEvent message)
+        {
+            message = null;
+
+            if (order == null)
+            {
+                message = new BrokerageMessageEvent(BrokerageMessageType.Warning, "NotSupported",
+                    Invariant($"Order is null.")
+                );
+                return false;
+            }
+
+            if (order.Price == 0m)
+            {
+                message = new BrokerageMessageEvent(BrokerageMessageType.Warning, "NotSupported",
+                    Invariant($"Price is not set.")
+                );
+                return false;
+            }
+
+            if (security.Type != SecurityType.Forex &&
+                security.Type != SecurityType.Equity &&
+                security.Type != SecurityType.Index &&
+                security.Type != SecurityType.Option &&
+                security.Type != SecurityType.Future &&
+                security.Type != SecurityType.Cfd &&
+                security.Type != SecurityType.Crypto &&
+                security.Type != SecurityType.Index)
+            {
+                message = new BrokerageMessageEvent(BrokerageMessageType.Warning, "NotSupported",
+                    Invariant(
+                        $"The {nameof(ExanteBrokerageModel)} does not support {security.Type} security type.")
+                );
+                return false;
+            }
+
+            return true;
+        }
+
+        /// <summary>
+        /// Gets a new fee model that represents this brokerage's fee structure
+        /// </summary>
+        /// <param name="security">The security to get a fee model for</param>
+        /// <returns>The new fee model for this brokerage</returns>
+        public override IFeeModel GetFeeModel(Security security) => new ExanteFeeModel();
+
+        /// <summary>
+        /// Exante global leverage rule
+        /// </summary>
+        /// <param name="security">The security's whose leverage we seek</param>
+        /// <returns>The leverage for the specified security</returns>
+        public override decimal GetLeverage(Security security)
+        {
+            if (AccountType == AccountType.Cash || security.IsInternalFeed() || security.Type == SecurityType.Base)
+            {
+                return 1m;
+            }
+
+            return security.Type switch
+            {
+                SecurityType.Forex => 1.05m,
+                SecurityType.Equity => EquityLeverage,
+                _ => 1.0m,
+            };
+        }
+    }
+}

Common/Brokerages/FTXBrokerageModel.cs

@@ -29,6 +29,16 @@ namespace QuantConnect.Brokerages
     {
         private const decimal _defaultLeverage = 3m;
 
+        /// <summary>
+        /// market name
+        /// </summary>
+        protected virtual string MarketName => Market.FTX;
+
+        /// <summary>
+        /// Gets a map of the default markets to be used for each security type
+        /// </summary>
+        public override IReadOnlyDictionary<SecurityType, string> DefaultMarkets { get; } = GetDefaultMarkets(Market.FTX);
+
         /// <summary>
         /// Creates an instance of <see cref="FTXBrokerageModel"/> class
         /// </summary>
@@ -36,13 +46,7 @@ namespace QuantConnect.Brokerages
         public FTXBrokerageModel(AccountType accountType = AccountType.Margin) : base(accountType)
         {
         }
-
-        /// <summary>
-        /// Gets a map of the default markets to be used for each security type
-        /// </summary>
-        public override IReadOnlyDictionary<SecurityType, string> DefaultMarkets { get; } = GetDefaultMarkets();
-
-
+        
         /// <summary>
         /// Gets a new buying power model for the security, returning the default model with the security's configured leverage.
         /// For cash accounts, leverage = 1 is used.
@@ -87,7 +91,7 @@ namespace QuantConnect.Brokerages
         /// <returns>The benchmark for this brokerage</returns>
         public override IBenchmark GetBenchmark(SecurityManager securities)
         {
-            var symbol = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.FTX);
+            var symbol = Symbol.Create("BTCUSD", SecurityType.Crypto, MarketName);
             return SecurityBenchmark.CreateInstance(securities, symbol);
         }
 
@@ -158,7 +162,7 @@ namespace QuantConnect.Brokerages
             if (security.Type != SecurityType.Crypto)
             {
                 message = new BrokerageMessageEvent(BrokerageMessageType.Warning, "NotSupported",
-                    StringExtensions.Invariant($"The {nameof(FTXBrokerageModel)} does not support {security.Type} security type.")
+                    StringExtensions.Invariant($"The {this.GetType().Name} does not support {security.Type} security type.")
                 );
 
                 return false;
@@ -186,10 +190,10 @@ namespace QuantConnect.Brokerages
             return false;
         }
 
-        private static IReadOnlyDictionary<SecurityType, string> GetDefaultMarkets()
+        protected static IReadOnlyDictionary<SecurityType, string> GetDefaultMarkets(string market)
         {
             var map = DefaultMarketMap.ToDictionary();
-            map[SecurityType.Crypto] = Market.FTX;
+            map[SecurityType.Crypto] = market;
             return map.ToReadOnlyDictionary();
         }
     }

Common/Brokerages/FTXUSBrokerageModel.cs

@@ -0,0 +1,53 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.Collections.Generic;
+using QuantConnect.Orders.Fees;
+using QuantConnect.Securities;
+
+namespace QuantConnect.Brokerages
+{
+    /// <summary>
+    /// FTX.US Brokerage model
+    /// </summary>
+    public class FTXUSBrokerageModel : FTXBrokerageModel
+    {
+        /// <summary>
+        /// Market name
+        /// </summary>
+        protected override string MarketName => Market.FTXUS;
+
+        /// <summary>
+        /// Gets a map of the default markets to be used for each security type
+        /// </summary>
+        public override IReadOnlyDictionary<SecurityType, string> DefaultMarkets { get; } = GetDefaultMarkets(Market.FTXUS);
+
+        /// <summary>
+        /// Creates an instance of <see cref="FTXUSBrokerageModel"/> class
+        /// </summary>
+        /// <param name="accountType">Cash or Margin</param>
+        public FTXUSBrokerageModel(AccountType accountType = AccountType.Margin) : base(accountType)
+        {
+        }
+
+        /// <summary>
+        /// Provides FTX.US fee model
+        /// </summary>
+        /// <param name="security">The security to get a fee model for</param>
+        /// <returns>The new fee model for this brokerage</returns>
+        public override IFeeModel GetFeeModel(Security security)
+            => new FTXUSFeeModel();
+    }
+}

Common/Brokerages/IBrokerageModel.cs

@@ -227,10 +227,16 @@ namespace QuantConnect.Brokerages
                 
                 case BrokerageName.Kraken:
                     return new KrakenBrokerageModel(accountType);
+                
+                case BrokerageName.Exante:
+                    return new ExanteBrokerageModel(accountType);
 
                 case BrokerageName.FTX:
                     return new FTXBrokerageModel(accountType);
 
+                case BrokerageName.FTXUS:
+                    return new FTXUSBrokerageModel(accountType);
+
                 default:
                     throw new ArgumentOutOfRangeException(nameof(brokerage), brokerage, null);
             }

Common/Brokerages/SamcoBrokerageModel.cs

@@ -38,7 +38,7 @@ namespace QuantConnect.Brokerages
             typeof(GoodTilDateTimeInForce)
         };
 
-        private const decimal _maxLeverage = 7m;
+        private const decimal _maxLeverage = 5m;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="SamcoBrokerageModel"/> class
@@ -167,7 +167,7 @@ namespace QuantConnect.Brokerages
                 return 1m;
             }
 
-            if (security.Type == SecurityType.Equity || security.Type == SecurityType.Future || security.Type == SecurityType.Option)
+            if (security.Type == SecurityType.Equity || security.Type == SecurityType.Future || security.Type == SecurityType.Option || security.Type == SecurityType.Index)
             {
                 return _maxLeverage;
             }

Common/Brokerages/ZerodhaBrokerageModel.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -38,7 +38,7 @@ namespace QuantConnect.Brokerages
             typeof(GoodTilDateTimeInForce)
         };
 
-        private const decimal _maxLeverage = 7m;
+        private const decimal _maxLeverage = 5m;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="ZerodhaBrokerageModel"/> class
@@ -164,7 +164,7 @@ namespace QuantConnect.Brokerages
                 return 1m;
             }
 
-            if (security.Type == SecurityType.Equity || security.Type == SecurityType.Future || security.Type == SecurityType.Option)
+            if (security.Type == SecurityType.Equity || security.Type == SecurityType.Future || security.Type == SecurityType.Option || security.Type == SecurityType.Index)
             {
                 return _maxLeverage;
             }

Common/Currencies.cs

@@ -23,24 +23,34 @@ namespace QuantConnect
     public static class Currencies
     {
         /// <summary>
-        /// USD currency string
+        /// USD (United States Dollar) currency string
         /// </summary>
-        public static string USD = "USD";
+        public const string USD = "USD";
 
         /// <summary>
-        /// EUR currency string
+        /// EUR (Euro) currency string
         /// </summary>
-        public static string EUR = "EUR";
+        public const string EUR = "EUR";
 
         /// <summary>
-        /// GBP currency string
+        /// GBP (British pound sterling) currency string
         /// </summary>
-        public static string GBP = "GBP";
+        public const string GBP = "GBP";
 
         /// <summary>
-        /// INR currency string
+        /// INR (Indian rupee) currency string
         /// </summary>
-        public static string INR = "INR";
+        public const string INR = "INR";
+
+        /// <summary>
+        /// CNH (Chinese Yuan Renminbi) currency string
+        /// </summary>
+        public const string CNH = "CNH";
+
+        /// <summary>
+        /// HKD (Hong Kong dollar) currency string
+        /// </summary>
+        public const string HKD = "HKD";
 
         /// <summary>
         /// Null currency used when a real one is not required
@@ -55,19 +65,19 @@ namespace QuantConnect
         /// </remarks>
         public static readonly IReadOnlyDictionary<string, string> CurrencySymbols = new Dictionary<string, string>
         {
-            {"USD", "$"},
-            {"GBP", "₤"},
+            {USD, "$"},
+            {GBP, "₤"},
             {"JPY", "¥"},
-            {"EUR", "€"},
+            {EUR, "€"},
             {"NZD", "$"},
             {"AUD", "$"},
             {"CAD", "$"},
             {"CHF", "Fr"},
-            {"HKD", "$"},
+            {HKD, "$"},
             {"SGD", "$"},
             {"XAG", "Ag"},
             {"XAU", "Au"},
-            {"CNH", "¥"},
+            {CNH, "¥"},
             {"CNY", "¥"},
             {"CZK", "Kč"},
             {"DKK", "kr"},

Common/Data/BaseDataRequest.cs

@@ -15,6 +15,7 @@
 
 using System;
 using QuantConnect.Securities;
+using System.Collections.Generic;
 
 namespace QuantConnect.Data
 {
@@ -51,6 +52,11 @@ namespace QuantConnect.Data
         /// </summary>
         public SecurityExchangeHours ExchangeHours { get; }
 
+        /// <summary>
+        /// Gets the tradable days specified by this request, in the security's data time zone
+        /// </summary>
+        public abstract IEnumerable<DateTime> TradableDays { get; }
+
         /// <summary>
         /// Initializes the base data request
         /// </summary>

Common/Data/Custom/Quandl.cs

@@ -1,156 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using System.Collections.Generic;
-using System.Globalization;
-using System.Net;
-
-namespace QuantConnect.Data.Custom
-{
-    /// <summary>
-    /// Quandl Data Type - Import generic data from quandl, without needing to define Reader methods.
-    /// This reads the headers of the data imported, and dynamically creates properties for the imported data.
-    /// </summary>
-    public class Quandl : DynamicData
-    {
-        private bool _isInitialized;
-        private readonly List<string> _propertyNames = new List<string>();
-        private readonly string _valueColumn;
-        private static string _authCode = "";
-
-        /// <summary>
-        /// Static constructor for the <see cref="Quandl"/> class
-        /// </summary>
-        static Quandl()
-        {
-            // The Quandl API now requires TLS 1.2 for API requests (since 9/18/2018).
-            // NET 4.5.2 and below does not enable this more secure protocol by default, so we add it in here
-            ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
-        }
-
-        /// <summary>
-        /// Flag indicating whether or not the Quanl auth code has been set yet
-        /// </summary>
-        public static bool IsAuthCodeSet
-        {
-            get;
-            private set;
-        }
-
-        /// <summary>
-        /// The end time of this data. Some data covers spans (trade bars) and as such we want
-        /// to know the entire time span covered
-        /// </summary>
-        public override DateTime EndTime
-        {
-            get { return Time + Period; }
-            set { Time = value - Period; }
-        }
-
-        /// <summary>
-        /// Gets a time span of one day
-        /// </summary>
-        public TimeSpan Period
-        {
-            get { return QuantConnect.Time.OneDay; }
-        }
-
-        /// <summary>
-        /// Default quandl constructor uses Close as its value column
-        /// </summary>
-        public Quandl() : this("Close")
-        {
-        }
-
-        /// <summary>
-        /// Constructor for creating customized quandl instance which doesn't use "Close" as its value item.
-        /// </summary>
-        /// <param name="valueColumnName"></param>
-        protected Quandl(string valueColumnName)
-        {
-            _valueColumn = valueColumnName;
-        }
-
-        /// <summary>
-        /// Generic Reader Implementation for Quandl Data.
-        /// </summary>
-        /// <param name="config">Subscription configuration</param>
-        /// <param name="line">CSV line of data from the souce</param>
-        /// <param name="date">Date of the requested line</param>
-        /// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
-        /// <returns></returns>
-        public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
-        {
-            // be sure to instantiate the correct type
-            var data = (Quandl) Activator.CreateInstance(GetType());
-            data.Symbol = config.Symbol;
-            var csv = line.Split(',');
-
-            if (!_isInitialized)
-            {
-                _isInitialized = true;
-                foreach (var propertyName in csv)
-                {
-                    var property = propertyName.Trim();
-                    // should we remove property names like Time?
-                    // do we need to alias the Time??
-                    data.SetProperty(property, 0m);
-                    _propertyNames.Add(property);
-                }
-                // Returns null at this point where we are only reading the properties names
-                return null;
-            }
-
-            data.Time = DateTime.ParseExact(csv[0], "yyyy-MM-dd", CultureInfo.InvariantCulture);
-
-            for (var i = 1; i < csv.Length; i++)
-            {
-                var value = csv[i].ToDecimal();
-                data.SetProperty(_propertyNames[i], value);
-            }
-
-            // we know that there is a close property, we want to set that to 'Value'
-            data.Value = (decimal)data.GetProperty(_valueColumn);
-
-            return data;
-        }
-
-        /// <summary>
-        /// Quandl Source Locator: Using the Quandl V1 API automatically set the URL for the dataset.
-        /// </summary>
-        /// <param name="config">Subscription configuration object</param>
-        /// <param name="date">Date of the data file we're looking for</param>
-        /// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
-        /// <returns>STRING API Url for Quandl.</returns>
-        public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
-        {
-            var source = $"https://www.quandl.com/api/v3/datasets/{config.Symbol.Value}.csv?order=asc&api_key={_authCode}";
-            return new SubscriptionDataSource(source, SubscriptionTransportMedium.RemoteFile);
-        }
-
-        /// <summary>
-        /// Set the auth code for the quandl set to the QuantConnect auth code.
-        /// </summary>
-        /// <param name="authCode"></param>
-        public static void SetAuthCode(string authCode)
-        {
-            if (string.IsNullOrWhiteSpace(authCode)) return;
-
-            _authCode = authCode;
-            IsAuthCodeSet = true;
-        }
-    }
-}