Add null check for Tradier GetQuotes (#6170)

- Fixing bug where tradier GetQuotes could return null in cases where
  the provided symbol would not match any. Adding unit test.

.vscode/launch.json

@@ -8,8 +8,8 @@
         marketplace.
 
         Attach to Python:
-        Will attempt to attach to LEAN running locally using PTVSD. Requires that the process is
-        actively running and config is set: "debugging": true, "debugging-method": "PTVSD",
+        Will attempt to attach to LEAN running locally using DebugPy. Requires that the process is
+        actively running and config is set: "debugging": true, "debugging-method": "DebugPy",
         Requires Python extension from the marketplace. Currently only works with algorithms in
         Algorithm.Python directory. This is because we map that directory to our build directory
         that contains the py file at runtime. If using another location change "localRoot" value 

.vscode/readme.md

@@ -95,7 +95,7 @@ Python algorithms require a little extra work in order to be able to debug them.
 First in order to debug a Python algorithm in VS Code we must make the following change to our configuration (Launcher\config.json) under the comment debugging configuration:
 
     "debugging": true,
-    "debugging-method": "PTVSD",
+    "debugging-method": "DebugPy,
 
 In setting this we are telling Lean to expect a debugger connection using ‘Python Tools for Visual Studio Debugger’. Once this is set Lean will stop upon initialization and await a connection to the debugger via port 5678.
 

Algorithm.CSharp/AddRemoveOptionUniverseRegressionAlgorithm.cs

@@ -40,8 +40,8 @@ namespace QuantConnect.Algorithm.CSharp
         private int _expectedContractIndex;
         private readonly List<Symbol> _expectedContracts = new List<Symbol>
         {
-            SymbolRepresentation.ParseOptionTickerOSI("GOOG  151224P00750000"),
             SymbolRepresentation.ParseOptionTickerOSI("GOOG  151224P00747500"),
+            SymbolRepresentation.ParseOptionTickerOSI("GOOG  151224P00750000"),
             SymbolRepresentation.ParseOptionTickerOSI("GOOG  151224P00752500")
         };
 
@@ -109,6 +109,11 @@ namespace QuantConnect.Algorithm.CSharp
                 var googOptionChain = AddOption(UnderlyingTicker);
                 googOptionChain.SetFilter(u =>
                 {
+                    // we added the universe at 10, the universe selection data should not be from before
+                    if (u.Underlying.EndTime.Hour < 10)
+                    {
+                        throw new Exception($"Unexpected underlying data point {u.Underlying.EndTime} {u.Underlying}");
+                    }
                     // find first put above market price
                     return u.IncludeWeeklys()
                         .Strikes(+1, +1)
@@ -231,7 +236,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Treynor Ratio", "0"},
             {"Total Fees", "$6.00"},
             {"Estimated Strategy Capacity", "$2000.00"},
-            {"Lowest Capacity Asset", "GOOCV 305RBQ2BZBZT2|GOOCV VP83T1ZUHROL"},
+            {"Lowest Capacity Asset", "GOOCV 305RBR0BSWIX2|GOOCV VP83T1ZUHROL"},
             {"Fitness Score", "0"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
@@ -251,7 +256,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "1e7b3e90918777b9dbf46353a96f3329"}
+            {"OrderListHash", "550a99c482106defd8ba15f48183768e"}
         };
     }
 }

Algorithm.CSharp/Alphas/MortgageRateVolatilityAlpha.cs

@@ -1,162 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-using System;
-using System.Linq;
-using QuantConnect.Data;
-using QuantConnect.Indicators;
-using QuantConnect.Orders.Fees;
-using QuantConnect.Data.Custom;
-using System.Collections.Generic;
-using QuantConnect.Algorithm.Framework.Alphas;
-using QuantConnect.Algorithm.Framework.Execution;
-using QuantConnect.Algorithm.Framework.Portfolio;
-using QuantConnect.Algorithm.Framework.Risk;
-using QuantConnect.Algorithm.Framework.Selection;
-
-namespace QuantConnect.Algorithm.CSharp.Alphas
-{
-    ///<summary>
-    /// This Alpha Model uses Wells Fargo 30-year Fixed Rate Mortgage data from Quandl to
-    /// generate Insights about the movement of Real Estate ETFs. Mortgage rates can provide information
-    /// regarding the general price trend of real estate, and ETFs provide good continuous-time instruments
-    /// to measure the impact against. Volatility in mortgage rates tends to put downward pressure on real
-    /// estate prices, whereas stable mortgage rates, regardless of true rate, lead to stable or higher real
-    /// estate prices. This Alpha model seeks to take advantage of this correlation by emitting insights
-    /// based on volatility and rate deviation from its historic mean.
-    ///
-    /// This alpha is part of the Benchmark Alpha Series created by QuantConnect which are open
-    /// sourced so the community and client funds can see an example of an alpha.
-    ///</summary>
-    public class MortgageRateVolatilityAlgorithm : QCAlgorithm
-    {
-        public override void Initialize()
-        {
-            SetStartDate(2017, 1, 1);  //Set Start Date
-            SetCash(100000);             //Set Strategy Cash
-
-            UniverseSettings.Resolution = Resolution.Daily;
-            SetSecurityInitializer(security => security.FeeModel = new ConstantFeeModel(0));
-
-            // Basket of 6 liquid real estate ETFs
-            Func<string, Symbol> toSymbol = x => QuantConnect.Symbol.Create(x, SecurityType.Equity, Market.USA);
-            var realEstateETFs = new[] { "VNQ", "REET", "TAO", "FREL", "SRET", "HIPS" }.Select(toSymbol).ToArray();
-            SetUniverseSelection(new ManualUniverseSelectionModel(realEstateETFs));
-
-            SetAlpha(new MortgageRateVolatilityAlphaModel(this));
-
-            SetPortfolioConstruction(new EqualWeightingPortfolioConstructionModel());
-
-            SetExecution(new ImmediateExecutionModel());
-
-            SetRiskManagement(new NullRiskManagementModel());
-
-        }
-
-        private class MortgageRateVolatilityAlphaModel : AlphaModel
-        {
-            private readonly int _indicatorPeriod;
-            private readonly Resolution _resolution;
-            private readonly TimeSpan _insightDuration;
-            private readonly int _deviations;
-            private readonly double _insightMagnitude;
-            private readonly Symbol _mortgageRate;
-            private readonly SimpleMovingAverage _mortgageRateSma;
-            private readonly StandardDeviation _mortgageRateStd;
-
-            public MortgageRateVolatilityAlphaModel(
-                QCAlgorithm algorithm,
-                int indicatorPeriod = 15,
-                double insightMagnitude = 0.0005,
-                int deviations = 2,
-                Resolution resolution = Resolution.Daily
-                )
-            {
-            	// Add Quandl data for a Well's Fargo 30-year Fixed Rate mortgage
-                _mortgageRate = algorithm.AddData<QuandlMortgagePriceColumns>("WFC/PR_GOV_30YFIXEDVA_APR").Symbol;
-                _indicatorPeriod = indicatorPeriod;
-                _resolution = resolution;
-                _insightDuration = resolution.ToTimeSpan().Multiply(indicatorPeriod);
-                _insightMagnitude = insightMagnitude;
-                _deviations = deviations;
-                
-                // Add indicators for the mortgage rate -- Standard Deviation and Simple Moving Average
-                _mortgageRateStd = algorithm.STD(_mortgageRate, _indicatorPeriod, resolution);
-                _mortgageRateSma = algorithm.SMA(_mortgageRate, _indicatorPeriod, resolution);
-                
-                // Use a history call to warm-up the indicators
-                WarmUpIndicators(algorithm);
-            }
-            
-            public override IEnumerable<Insight> Update(QCAlgorithm algorithm, Slice data)
-            {
-                var insights = new List<Insight>();
-                
-                // Return empty list if data slice doesn't contain monrtgage rate data
-                if (!data.Keys.Contains(_mortgageRate))
-                {
-                    return insights;
-                }
-				// Extract current mortgage rate, the current STD indicator value, and current SMA value
-                var rate = data[_mortgageRate].Value;
-                var deviation = _deviations * _mortgageRateStd;
-                var sma = _mortgageRateSma;
-
-				// Loop through all Active Securities to emit insights
-                foreach (var security in algorithm.ActiveSecurities.Keys)
-                {
-                	// Mortgage rate Symbol will be in the collection, so skip it
-                    if (security == _mortgageRate)
-                    {
-                        return insights;
-                    }
-
-					// If volatility in mortgage rates is high, then we emit an Insight to sell
-                    if ((rate < sma - deviation) || (rate > sma + deviation))
-                    {
-                        insights.Add(Insight.Price(security, _insightDuration, InsightDirection.Down, _insightMagnitude));
-                    }
-                    
-                    // If volatility in mortgage rates is low, then we emit an Insight to buy
-                    if ((rate < sma - (decimal)deviation/2) || (rate > sma + (decimal)deviation/2))
-                    {
-                        insights.Add(Insight.Price(security, _insightDuration, InsightDirection.Up, _insightMagnitude));
-                    }
-                }
-
-                return insights;
-            }
-
-            private void WarmUpIndicators(QCAlgorithm algorithm)
-            {
-            	// Make a history call and update the indicators
-                algorithm.History(new[] { _mortgageRate }, _indicatorPeriod, _resolution).PushThrough(bar =>
-                {
-                    _mortgageRateSma.Update(bar.EndTime, bar.Value);
-                    _mortgageRateStd.Update(bar.EndTime, bar.Value);
-                });
-            }
-        }
-        public class QuandlMortgagePriceColumns : Quandl
-        {
-            public QuandlMortgagePriceColumns()
-            
-            // Rename the Quandl object column to the data we want, which is the 'Value' column
-	        // of the CSV that our API call returns
-                : base(valueColumnName: "Value")
-            {
-            }
-        }
-    }
-}

Algorithm.CSharp/BasicTemplateFuturesAlgorithm.cs

@@ -18,6 +18,7 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using QuantConnect.Data;
+using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Interfaces;
 using QuantConnect.Securities;
 using QuantConnect.Securities.Future;
@@ -64,6 +65,9 @@ namespace QuantConnect.Algorithm.CSharp
 
             var benchmark = AddEquity("SPY");
             SetBenchmark(benchmark.Symbol);
+
+            var seeder = new FuncSecuritySeeder(GetLastKnownPrices);
+            SetSecurityInitializer(security => seeder.SeedSecurity(security));
         }
 
         /// <summary>
@@ -112,6 +116,19 @@ namespace QuantConnect.Algorithm.CSharp
             var maintenanceIntraday = futureMarginModel.MaintenanceIntradayMarginRequirement;
         }
 
+        public override void OnSecuritiesChanged(SecurityChanges changes)
+        {
+            foreach (var addedSecurity in changes.AddedSecurities)
+            {
+                if (addedSecurity.Symbol.SecurityType == SecurityType.Future
+                    && !addedSecurity.Symbol.IsCanonical()
+                    && !addedSecurity.HasData)
+                {
+                    throw new Exception($"Future contracts did not work up as expected: {addedSecurity.Symbol}");
+                }
+            }
+        }
+
         /// <summary>
         /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
         /// </summary>

Algorithm.CSharp/BasicTemplateIndiaAlgorithm.cs

@@ -14,8 +14,9 @@
 */
 
 using QuantConnect.Data;
-using QuantConnect.Orders;
 using System.Collections.Generic;
+using QuantConnect.Interfaces;
+using QuantConnect.Orders;
 
 namespace QuantConnect.Algorithm.CSharp
 {
@@ -26,20 +27,21 @@ namespace QuantConnect.Algorithm.CSharp
     /// <meta name="tag" content="using data" />
     /// <meta name="tag" content="using quantconnect" />
     /// <meta name="tag" content="trading and orders" />
-    public class BasicTemplateIndiaAlgorithm : QCAlgorithm
+    public class BasicTemplateIndiaAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
         /// <summary>
         /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
         /// </summary>
         public override void Initialize()
         {
-            SetStartDate(2003, 10, 07);  //Set Start Date
-            SetEndDate(2003, 10, 11);    //Set End Date
-            SetCash(100000);             //Set Strategy Cash
+            SetAccountCurrency("INR");  //Set Account Currency
+            SetStartDate(2019, 1, 23);  //Set Start Date
+            SetEndDate(2019, 10, 31);   //Set End Date
+            SetCash(100000);            //Set Strategy Cash
 
             // Find more symbols here: http://quantconnect.com/data
             // Equities Resolutions: Tick, Second, Minute, Hour, Daily.
-            AddEquity("UNIONBANK", Resolution.Second, Market.India);
+            AddEquity("YESBANK", Resolution.Minute, Market.India);
 
             //Set Order Prperties as per the requirements for order placement
             DefaultOrderProperties = new IndiaOrderProperties(exchange: Exchange.NSE);
@@ -58,7 +60,7 @@ namespace QuantConnect.Algorithm.CSharp
         {
             if (!Portfolio.Invested)
             {
-                var marketTicket = MarketOrder("UNIONBANK", 1);
+                var marketTicket = MarketOrder("YESBANK", 1);
             }
         }
 
@@ -73,60 +75,60 @@ namespace QuantConnect.Algorithm.CSharp
         /// <summary>
         /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
         /// </summary>
-        public bool CanRunLocally { get; } = false;
+        public bool CanRunLocally { get; } = true;
 
         /// <summary>
         /// This is used by the regression test system to indicate which languages this algorithm is written in.
         /// </summary>
-        public Language[] Languages { get; } = { Language.CSharp };
+        public Language[] Languages { get; } = { Language.CSharp, Language.Python };
 
         /// <summary>
         /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
         /// </summary>
         public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
-            {"Total Trades", "3"},
+            {"Total Trades", "1"},
             {"Average Win", "0%"},
-            {"Average Loss", "-1.01%"},
-            {"Compounding Annual Return", "261.134%"},
-            {"Drawdown", "2.200%"},
-            {"Expectancy", "-1"},
-            {"Net Profit", "1.655%"},
-            {"Sharpe Ratio", "8.505"},
-            {"Probabilistic Sharpe Ratio", "66.840%"},
-            {"Loss Rate", "100%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "-0.010%"},
+            {"Drawdown", "0.000%"},
+            {"Expectancy", "0"},
+            {"Net Profit", "-0.008%"},
+            {"Sharpe Ratio", "-1.183"},
+            {"Probabilistic Sharpe Ratio", "0.001%"},
+            {"Loss Rate", "0%"},
             {"Win Rate", "0%"},
             {"Profit-Loss Ratio", "0"},
-            {"Alpha", "-0.091"},
-            {"Beta", "1.006"},
-            {"Annual Standard Deviation", "0.224"},
-            {"Annual Variance", "0.05"},
-            {"Information Ratio", "-33.445"},
-            {"Tracking Error", "0.002"},
-            {"Treynor Ratio", "1.893"},
-            {"Total Fees", "$10.32"},
-            {"Estimated Strategy Capacity", "$27000000.00"},
-            {"Lowest Capacity Asset", "SPY R735QTJ8XC9X"},
-            {"Fitness Score", "0.747"},
-            {"Kelly Criterion Estimate", "38.796"},
-            {"Kelly Criterion Probability Value", "0.228"},
-            {"Sortino Ratio", "79228162514264337593543950335"},
-            {"Return Over Maximum Drawdown", "85.095"},
-            {"Portfolio Turnover", "0.747"},
-            {"Total Insights Generated", "100"},
-            {"Total Insights Closed", "99"},
-            {"Total Insights Analysis Completed", "99"},
-            {"Long Insight Count", "100"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "-1.183"},
+            {"Tracking Error", "0"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "$6.00"},
+            {"Estimated Strategy Capacity", "$61000000000.00"},
+            {"Lowest Capacity Asset", "YESBANK UL"},
+            {"Fitness Score", "0"},
+            {"Kelly Criterion Estimate", "0"},
+            {"Kelly Criterion Probability Value", "0"},
+            {"Sortino Ratio", "-0.247"},
+            {"Return Over Maximum Drawdown", "-1.104"},
+            {"Portfolio Turnover", "0"},
+            {"Total Insights Generated", "0"},
+            {"Total Insights Closed", "0"},
+            {"Total Insights Analysis Completed", "0"},
+            {"Long Insight Count", "0"},
             {"Short Insight Count", "0"},
             {"Long/Short Ratio", "100%"},
-            {"Estimated Monthly Alpha Value", "$135639.1761"},
-            {"Total Accumulated Estimated Alpha Value", "$21852.9784"},
-            {"Mean Population Estimated Insight Value", "$220.7372"},
-            {"Mean Population Direction", "53.5354%"},
-            {"Mean Population Magnitude", "53.5354%"},
-            {"Rolling Averaged Population Direction", "58.2788%"},
-            {"Rolling Averaged Population Magnitude", "58.2788%"},
-            {"OrderListHash", "ad2216297c759d8e5aef48ff065f8919"}
+            {"Estimated Monthly Alpha Value", "₹0"},
+            {"Total Accumulated Estimated Alpha Value", "₹0"},
+            {"Mean Population Estimated Insight Value", "₹0"},
+            {"Mean Population Direction", "0%"},
+            {"Mean Population Magnitude", "0%"},
+            {"Rolling Averaged Population Direction", "0%"},
+            {"Rolling Averaged Population Magnitude", "0%"},
+            {"OrderListHash", "6cc69218edd7bd461678b9ee0c575db5"}
         };
     }
 }

Algorithm.CSharp/BasicTemplateIndiaIndexAlgorithm.cs

@@ -0,0 +1,158 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using QuantConnect.Data;
+using System.Collections.Generic;
+using QuantConnect.Indicators;
+using QuantConnect.Interfaces;
+using QuantConnect.Orders;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// This example demonstrates how to add index asset types.
+    /// </summary>
+    /// <meta name="tag" content="using data" />
+    /// <meta name="tag" content="benchmarks" />
+    /// <meta name="tag" content="indexes" />
+    public class BasicTemplateIndiaIndexAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        protected Symbol Nifty;
+        protected Symbol NiftyETF;
+        private ExponentialMovingAverage _emaSlow;
+        private ExponentialMovingAverage _emaFast;
+
+        /// <summary>
+        /// Initialize your algorithm and add desired assets.
+        /// </summary>
+        public override void Initialize()
+        {
+            SetAccountCurrency("INR"); //Set Account Currency
+            SetStartDate(2019, 1, 1);  //Set End Date
+            SetEndDate(2019, 1, 5);    //Set End Date
+            SetCash(1000000);          //Set Strategy Cash
+
+            // Use indicator for signal; but it cannot be traded
+            Nifty = AddIndex("NIFTY50", Resolution.Minute, Market.India).Symbol;
+
+            //Trade Index based ETF
+            NiftyETF = AddEquity("JUNIORBEES", Resolution.Minute, Market.India).Symbol;
+
+            //Set Order Prperties as per the requirements for order placement
+            DefaultOrderProperties = new IndiaOrderProperties(exchange: Exchange.NSE);
+
+            _emaSlow = EMA(Nifty, 80);
+            _emaFast = EMA(Nifty, 200);
+        }
+
+        /// <summary>
+        /// Index EMA Cross trading underlying.
+        /// </summary>
+        public override void OnData(Slice slice)
+        {
+            if (!slice.Bars.ContainsKey(Nifty) || !slice.Bars.ContainsKey(NiftyETF))
+            {
+                return;
+            }
+
+            // Warm up indicators
+            if (!_emaSlow.IsReady)
+            {
+                return;
+            }
+
+            if (_emaFast > _emaSlow)
+            {
+                if (!Portfolio.Invested)
+                {
+                    var marketTicket = MarketOrder(NiftyETF, 1);
+                }
+            }
+            else
+            {
+                Liquidate();
+            }
+        }
+
+        public override void OnEndOfAlgorithm()
+        {
+            if (Portfolio[Nifty].TotalSaleVolume > 0)
+            {
+                throw new Exception("Index is not tradable.");
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public virtual bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public virtual Language[] Languages { get; } = { Language.CSharp, Language.Python };
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public virtual Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Trades", "6"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0.00%"},
+            {"Compounding Annual Return", "-0.395%"},
+            {"Drawdown", "0.000%"},
+            {"Expectancy", "-1"},
+            {"Net Profit", "-0.004%"},
+            {"Sharpe Ratio", "-23.595"},
+            {"Probabilistic Sharpe Ratio", "0%"},
+            {"Loss Rate", "100%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "-23.595"},
+            {"Tracking Error", "0"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "$36.00"},
+            {"Estimated Strategy Capacity", "$74000.00"},
+            {"Lowest Capacity Asset", "JUNIORBEES UL"},
+            {"Fitness Score", "0"},
+            {"Kelly Criterion Estimate", "0"},
+            {"Kelly Criterion Probability Value", "0"},
+            {"Sortino Ratio", "-29.6"},
+            {"Return Over Maximum Drawdown", "-123.624"},
+            {"Portfolio Turnover", "0"},
+            {"Total Insights Generated", "0"},
+            {"Total Insights Closed", "0"},
+            {"Total Insights Analysis Completed", "0"},
+            {"Long Insight Count", "0"},
+            {"Short Insight Count", "0"},
+            {"Long/Short Ratio", "100%"},
+            {"Estimated Monthly Alpha Value", "₹0"},
+            {"Total Accumulated Estimated Alpha Value", "₹0"},
+            {"Mean Population Estimated Insight Value", "₹0"},
+            {"Mean Population Direction", "0%"},
+            {"Mean Population Magnitude", "0%"},
+            {"Rolling Averaged Population Direction", "0%"},
+            {"Rolling Averaged Population Magnitude", "0%"},
+            {"OrderListHash", "4637f26543287548b28a3c296db055d3"}
+        };
+    }
+}

Algorithm.CSharp/CustomDataIndicatorExtensionsAlgorithm.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -14,16 +14,17 @@
 */
 
 using System;
+using System.IO;
+using System.Globalization;
 using QuantConnect.Data;
-using QuantConnect.Data.Custom;
-using QuantConnect.Data.Market;
+using QuantConnect.Util;
 using QuantConnect.Indicators;
 
 namespace QuantConnect.Algorithm.CSharp
 {
     /// <summary>
     /// The algorithm creates new indicator value with the existing indicator method by Indicator Extensions
-    /// Demonstration of using the external custom datasource Quandl to request the VIX and VXV daily data
+    /// Demonstration of using local custom datasource CustomData to request the IBM and SPY daily data
     /// </summary>
     /// <meta name="tag" content="using data" />
     /// <meta name="tag" content="using quantconnect" />
@@ -34,10 +35,10 @@ namespace QuantConnect.Algorithm.CSharp
     /// <meta name="tag" content="charting" />
     public class CustomDataIndicatorExtensionsAlgorithm : QCAlgorithm
     {
-        private const string _vix = "CBOE/VIX";
-        private const string _vxv = "CBOE/VXV";
-        private SimpleMovingAverage _smaVIX;
-        private SimpleMovingAverage _smaVXV;
+        private const string _ibm = "IBM";
+        private const string _spy = "SPY";
+        private SimpleMovingAverage _smaIBM;
+        private SimpleMovingAverage _smaSPY;
         private IndicatorBase<IndicatorDataPoint> _ratio;
 
         /// <summary>
@@ -50,46 +51,82 @@ namespace QuantConnect.Algorithm.CSharp
             SetCash(25000);
 
             // Define the symbol and "type" of our generic data
-            AddData<QuandlVix>(_vix, Resolution.Daily);
-            AddData<Quandl>(_vxv, Resolution.Daily);
+            AddData<CustomData>(_ibm, Resolution.Daily);
+            AddData<CustomData>(_spy, Resolution.Daily);
             // Set up default Indicators, these are just 'identities' of the closing price
-            _smaVIX = SMA(_vix, 1);
-            _smaVXV = SMA(_vxv, 1);
-            // This will create a new indicator whose value is smaVXV / smaVIX
-            _ratio = _smaVXV.Over(_smaVIX);
+            _smaIBM = SMA(_ibm, 1);
+            _smaSPY = SMA(_spy, 1);
+            // This will create a new indicator whose value is smaSPY / smaIBM
+            _ratio = _smaSPY.Over(_smaIBM);
         }
 
         /// <summary>
         /// Custom data event handler:
         /// </summary>
-        /// <param name="data">Quandl - dictionary Bars of Quandl Data</param>
-        public void OnData(Quandl data)
+        /// <param name="data">CustomData - dictionary Bars of custom data</param>
+        public void OnData(CustomData data)
         {
             // Wait for all indicators to fully initialize
-            if (_smaVIX.IsReady && _smaVXV.IsReady && _ratio.IsReady)
+            if (_smaIBM.IsReady && _smaSPY.IsReady && _ratio.IsReady)
             {
                 if (!Portfolio.Invested && _ratio > 1)
                 {
-                    MarketOrder(_vix, 100);
+                    MarketOrder(_ibm, 100);
                 }
                 else if (_ratio < 1)
                 {
                     Liquidate();
                 }
                 // plot all indicators
-                PlotIndicator("SMA", _smaVIX, _smaVXV);
+                PlotIndicator("SMA", _smaIBM, _smaSPY);
                 PlotIndicator("Ratio", _ratio);
             }
         }
     }
 
     /// <summary>
-    /// In CBOE/VIX data, there is a "vix close" column instead of "close" which is the 
-    /// default column namein LEAN Quandl custom data implementation.
-    /// This class assigns new column name to match the the external datasource setting.
+    /// Custom data from local LEAN data
     /// </summary>
-    public class QuandlVix : Quandl
+    public class CustomData : BaseData
     {
-        public QuandlVix() : base(valueColumnName: "vix close") { }
+        public decimal Open;
+        public decimal High;
+        public decimal Low;
+        public decimal Close;
+
+        public override DateTime EndTime
+        {
+            get { return Time + Period; }
+            set { Time = value - Period; }
+        }
+
+        public TimeSpan Period
+        {
+            get { return QuantConnect.Time.OneDay; }
+        }
+
+        public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
+        {
+            var source = Path.Combine(Globals.DataFolder, "equity", "usa", config.Resolution.ToString().ToLower(), LeanData.GenerateZipFileName(config.Symbol, date, config.Resolution, config.TickType));
+            return new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.Csv);
+        }
+
+        public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
+        {
+            var csv = line.ToCsv(6);
+            var _scaleFactor = 1 / 10000m;
+
+            var custom = new CustomData
+            {
+                Symbol = config.Symbol,
+                Time = DateTime.ParseExact(csv[0], DateFormat.TwelveCharacter, CultureInfo.InvariantCulture),
+                Open = csv[1].ToDecimal() * _scaleFactor,
+                High = csv[2].ToDecimal() * _scaleFactor,
+                Low = csv[3].ToDecimal() * _scaleFactor,
+                Close = csv[4].ToDecimal() * _scaleFactor,
+                Value = csv[4].ToDecimal() * _scaleFactor
+            };
+            return custom;
+        }
     }
-}
+}

Algorithm.CSharp/HistoryAlgorithm.cs

@@ -14,10 +14,10 @@
 */
 
 using System;
+using QuantConnect.Util;
 using System.Collections.Generic;
 using System.Linq;
 using QuantConnect.Data;
-using QuantConnect.Data.Custom;
 using QuantConnect.Data.Market;
 using QuantConnect.Indicators;
 using QuantConnect.Securities.Equity;
@@ -36,7 +36,7 @@ namespace QuantConnect.Algorithm.CSharp
     public class HistoryAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
         private int _count;
-        private SimpleMovingAverage _spyDailySma;
+        private SimpleMovingAverage _dailySma;
 
         /// <summary>
         /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
@@ -49,12 +49,12 @@ namespace QuantConnect.Algorithm.CSharp
 
             // Find more symbols here: http://quantconnect.com/data
             var SPY = AddSecurity(SecurityType.Equity, "SPY", Resolution.Daily).Symbol;
-            var CME_SP1 = AddData<QuandlFuture>("CHRIS/CME_SP1", Resolution.Daily).Symbol;
+            var IBM = AddData<CustomData>("IBM", Resolution.Daily).Symbol;
             // specifying the exchange will allow the history methods that accept a number of bars to return to work properly
-            Securities["CHRIS/CME_SP1"].Exchange = new EquityExchange();
+            Securities["IBM"].Exchange = new EquityExchange();
 
             // we can get history in initialize to set up indicators and such
-            _spyDailySma = new SimpleMovingAverage(14);
+            _dailySma = new SimpleMovingAverage(14);
 
             // get the last calendar year's worth of SPY data at the configured resolution (daily)
             var tradeBarHistory = History<TradeBar>("SPY", TimeSpan.FromDays(365));
@@ -76,99 +76,97 @@ namespace QuantConnect.Algorithm.CSharp
             // we can use these TradeBars to initialize indicators or perform other math
             foreach (TradeBar tradeBar in tradeBarHistory)
             {
-                _spyDailySma.Update(tradeBar.EndTime, tradeBar.Close);
+                _dailySma.Update(tradeBar.EndTime, tradeBar.Close);
             }
 
-            // get the last calendar year's worth of quandl data at the configured resolution (daily)
-            var quandlHistory = History<QuandlFuture>("CHRIS/CME_SP1", TimeSpan.FromDays(365));
-            AssertHistoryCount("History<Quandl>(\"CHRIS/CME_SP1\", TimeSpan.FromDays(365))", quandlHistory, 250, CME_SP1);
+            // get the last calendar year's worth of IBM data at the configured resolution (daily)
+            var customDataHistory = History<CustomData>("IBM", TimeSpan.FromDays(365));
+            AssertHistoryCount("History<CustomData>(\"IBM\", TimeSpan.FromDays(365))", customDataHistory, 250, IBM);
 
-            // get the last 14 bars of SPY at the configured resolution (daily)
-            quandlHistory = History<QuandlFuture>("CHRIS/CME_SP1", 14);
-            AssertHistoryCount("History<Quandl>(\"CHRIS/CME_SP1\", 14)", quandlHistory, 14, CME_SP1);
+            // get the last 14 bars of IBM at the configured resolution (daily)
+            customDataHistory = History<CustomData>("IBM", 14);
+            AssertHistoryCount("History<CustomData>(\"IBM\", 14)", customDataHistory, 14, IBM);
 
-            // get the last 14 minute bars of SPY
-
-            // we can loop over the return values from these functions and we'll get Quandl data
+            // we can loop over the return values from these functions and we'll get custom data
             // this can be used in much the same way as the tradeBarHistory above
-            _spyDailySma.Reset();
-            foreach (QuandlFuture quandl in quandlHistory)
+            _dailySma.Reset();
+            foreach (CustomData customData in customDataHistory)
             {
-                _spyDailySma.Update(quandl.EndTime, quandl.Value);
+                _dailySma.Update(customData.EndTime, customData.Value);
             }
 
-            // get the last year's worth of all configured Quandl data at the configured resolution (daily)
-            var allQuandlData = History<QuandlFuture>(TimeSpan.FromDays(365));
-            AssertHistoryCount("History<QuandlFuture>(TimeSpan.FromDays(365))", allQuandlData, 250, CME_SP1);
+            // get the last year's worth of all configured custom data at the configured resolution (daily)
+            var allCustomData = History<CustomData>(TimeSpan.FromDays(365));
+            AssertHistoryCount("History<CustomData>(TimeSpan.FromDays(365))", allCustomData, 250, IBM);
 
-            // get the last 14 bars worth of Quandl data for the specified symbols at the configured resolution (daily)
-            allQuandlData = History<QuandlFuture>(Securities.Keys, 14);
-            AssertHistoryCount("History<QuandlFuture>(Securities.Keys, 14)", allQuandlData, 14, CME_SP1);
+            // get the last 14 bars worth of custom data for the specified symbols at the configured resolution (daily)
+            allCustomData = History<CustomData>(Securities.Keys, 14);
+            AssertHistoryCount("History<CustomData>(Securities.Keys, 14)", allCustomData, 14, IBM);
 
-            // NOTE: using different resolutions require that they are properly implemented in your data type, since
-            //  Quandl doesn't support minute data, this won't actually work, but if your custom data source has
-            //  different resolutions, it would need to be implemented in the GetSource and Reader methods properly
-            //quandlHistory = History<QuandlFuture>("CHRIS/CME_SP1", TimeSpan.FromDays(7), Resolution.Minute);
-            //quandlHistory = History<QuandlFuture>("CHRIS/CME_SP1", 14, Resolution.Minute);
-            //allQuandlData = History<QuandlFuture>(TimeSpan.FromDays(365), Resolution.Minute);
-            //allQuandlData = History<QuandlFuture>(Securities.Keys, 14, Resolution.Minute);
-            //allQuandlData = History<QuandlFuture>(Securities.Keys, TimeSpan.FromDays(1), Resolution.Minute);
-            //allQuandlData = History<QuandlFuture>(Securities.Keys, 14, Resolution.Minute);
+            // NOTE: Using different resolutions require that they are properly implemented in your data type. If your
+            // custom data source has different resolutions, it would need to be implemented in the GetSource and Reader
+            // methods properly.
+            //customDataHistory = History<CustomData>("IBM", TimeSpan.FromDays(7), Resolution.Minute);
+            //customDataHistory = History<CustomData>("IBM", 14, Resolution.Minute);
+            //allCustomData = History<CustomData>(TimeSpan.FromDays(365), Resolution.Minute);
+            //allCustomData = History<CustomData>(Securities.Keys, 14, Resolution.Minute);
+            //allCustomData = History<CustomData>(Securities.Keys, TimeSpan.FromDays(1), Resolution.Minute);
+            //allCustomData = History<CustomData>(Securities.Keys, 14, Resolution.Minute);
 
-            // get the last calendar year's worth of all quandl data
-            allQuandlData = History<QuandlFuture>(Securities.Keys, TimeSpan.FromDays(365));
-            AssertHistoryCount("History<QuandlFuture>(Securities.Keys, TimeSpan.FromDays(365))", allQuandlData, 250, CME_SP1);
+            // get the last calendar year's worth of all custom data
+            allCustomData = History<CustomData>(Securities.Keys, TimeSpan.FromDays(365));
+            AssertHistoryCount("History<CustomData>(Securities.Keys, TimeSpan.FromDays(365))", allCustomData, 250, IBM);
 
-            // the return is a series of dictionaries containing all quandl data at each time
+            // the return is a series of dictionaries containing all custom data at each time
             // we can loop over it to get the individual dictionaries
-            foreach (DataDictionary<QuandlFuture> quandlsDataDictionary in allQuandlData)
+            foreach (DataDictionary<CustomData> customDataDictionary in allCustomData)
             {
-                // we can access the dictionary to get the quandl data we want
-                var quandl = quandlsDataDictionary["CHRIS/CME_SP1"];
+                // we can access the dictionary to get the custom data we want
+                var customData = customDataDictionary["IBM"];
             }
 
             // we can also access the return value from the multiple symbol functions to request a single
             // symbol and then loop over it
-            var singleSymbolQuandl = allQuandlData.Get("CHRIS/CME_SP1");
-            AssertHistoryCount("allQuandlData.Get(\"CHRIS/CME_SP1\")", singleSymbolQuandl, 250, CME_SP1);
-            foreach (QuandlFuture quandl in singleSymbolQuandl)
+            var singleSymbolCustomData = allCustomData.Get("IBM");
+            AssertHistoryCount("allCustomData.Get(\"IBM\")", singleSymbolCustomData, 250, IBM);
+            foreach (CustomData customData in singleSymbolCustomData)
             {
-                // do something with 'CHRIS/CME_SP1' quandl data
+                // do something with 'IBM' custom data
             }
 
             // we can also access individual properties on our data, this will
-            // get the 'CHRIS/CME_SP1' quandls like above, but then only return the Low properties
-            var quandlSpyLows = allQuandlData.Get("CHRIS/CME_SP1", "Low");
-            AssertHistoryCount("allQuandlData.Get(\"CHRIS/CME_SP1\", \"Low\")", quandlSpyLows, 250);
-            foreach (decimal low in quandlSpyLows)
+            // get the 'IBM' CustomData objects like above, but then only return the Value properties
+            var customDataIbmValues = allCustomData.Get("IBM", "Value");
+            AssertHistoryCount("allCustomData.Get(\"IBM\", \"Value\")", customDataIbmValues, 250);
+            foreach (decimal value in customDataIbmValues)
             {
-                // do something with each low value
+                // do something with each value
             }
 
             // sometimes it's necessary to get the history for many configured symbols
 
             // request the last year's worth of history for all configured symbols at their configured resolutions
             var allHistory = History(TimeSpan.FromDays(365));
-            AssertHistoryCount("History(TimeSpan.FromDays(365))", allHistory, 250, SPY, CME_SP1);
+            AssertHistoryCount("History(TimeSpan.FromDays(365))", allHistory, 250, SPY, IBM);
 
             // request the last days's worth of history at the minute resolution
             allHistory = History(TimeSpan.FromDays(1), Resolution.Minute);
-            AssertHistoryCount("History(TimeSpan.FromDays(1), Resolution.Minute)", allHistory, 391, SPY, CME_SP1);
+            AssertHistoryCount("History(TimeSpan.FromDays(1), Resolution.Minute)", allHistory, 390, SPY, IBM);
 
             // request the last 100 bars for the specified securities at the configured resolution
             allHistory = History(Securities.Keys, 100);
-            AssertHistoryCount("History(Securities.Keys, 100)", allHistory, 100, SPY, CME_SP1);
+            AssertHistoryCount("History(Securities.Keys, 100)", allHistory, 100, SPY, IBM);
 
             // request the last 100 minute bars for the specified securities
             allHistory = History(Securities.Keys, 100, Resolution.Minute);
-            AssertHistoryCount("History(Securities.Keys, 100, Resolution.Minute)", allHistory, 101, SPY, CME_SP1);
+            AssertHistoryCount("History(Securities.Keys, 100, Resolution.Minute)", allHistory, 100, SPY, IBM);
 
             // request the last calendar years worth of history for the specified securities
             allHistory = History(Securities.Keys, TimeSpan.FromDays(365));
-            AssertHistoryCount("History(Securities.Keys, TimeSpan.FromDays(365))", allHistory, 250, SPY, CME_SP1);
+            AssertHistoryCount("History(Securities.Keys, TimeSpan.FromDays(365))", allHistory, 250, SPY, IBM);
             // we can also specify the resolution
             allHistory = History(Securities.Keys, TimeSpan.FromDays(1), Resolution.Minute);
-            AssertHistoryCount("History(Securities.Keys, TimeSpan.FromDays(1), Resolution.Minute)", allHistory, 391, SPY, CME_SP1);
+            AssertHistoryCount("History(Securities.Keys, TimeSpan.FromDays(1), Resolution.Minute)", allHistory, 390, SPY, IBM);
 
             // if we loop over this allHistory, we get Slice objects
             foreach (Slice slice in allHistory)
@@ -215,7 +213,7 @@ namespace QuantConnect.Algorithm.CSharp
 
             if (_count > 5)
             {
-                throw new Exception("Invalid number of bars arrived. Expected exactly 5");
+                throw new Exception($"Invalid number of bars arrived. Expected exactly 5, but received {_count}");
             }
 
             if (!Portfolio.Invested)
@@ -245,9 +243,9 @@ namespace QuantConnect.Algorithm.CSharp
             }
             else if (typeof(T).IsGenericType && typeof(T).GetGenericTypeDefinition() == typeof(DataDictionary<>))
             {
-                if (typeof(T).GetGenericArguments()[0] == typeof(QuandlFuture))
+                if (typeof(T).GetGenericArguments()[0] == typeof(CustomData))
                 {
-                    var dictionaries = (IEnumerable<DataDictionary<QuandlFuture>>) history;
+                    var dictionaries = (IEnumerable<DataDictionary<CustomData>>) history;
                     unexpectedSymbols = dictionaries.SelectMany(dd => dd.Keys)
                         .Distinct()
                         .Where(sym => !expectedSymbols.Contains(sym))
@@ -340,19 +338,5 @@ namespace QuantConnect.Algorithm.CSharp
             {"Rolling Averaged Population Magnitude", "0%"},
             {"OrderListHash", "33d01821923c397f999cfb2e5b5928ad"}
         };
-
-        /// <summary>
-        /// Custom quandl data type for setting customized value column name. Value column is used for the primary trading calculations and charting.
-        /// </summary>
-        public class QuandlFuture : Quandl
-        {
-            /// <summary>
-            /// Initializes a new instance of the <see cref="QuandlFuture"/> class.
-            /// </summary>
-            public QuandlFuture()
-                : base(valueColumnName: "Settle")
-            {
-            }
-        }
     }
-}
+}

Algorithm.CSharp/IndiaDataRegressionAlgorithm.cs

@@ -0,0 +1,202 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using QuantConnect.Interfaces;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Regression algorithm demonstrating use of map files with India data
+    /// </summary>
+    /// <meta name="tag" content="using data" />
+    /// <meta name="tag" content="India data" />
+    /// <meta name="tag" content="regression test" />
+    /// <meta name="tag" content="rename event" />
+    /// <meta name="tag" content="map" />
+    /// <meta name="tag" content="mapping" />
+    /// <meta name="tag" content="map files" />
+    public class IndiaDataRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        private Symbol _mappingSymbol, _splitAndDividendSymbol;
+        private bool _initialMapping;
+        private bool _executionMapping;
+        private bool _receivedWarningEvent;
+        private bool _receivedOccurredEvent;
+
+        /// <summary>
+        /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
+        /// </summary>
+        public override void Initialize()
+        {
+            SetAccountCurrency("INR");  //Set Account Currency 
+            SetStartDate(2004, 5, 20);  //Set Start Date
+            SetEndDate(2016, 7, 26);    //Set End Date
+            _mappingSymbol = AddEquity("3MINDIA", Resolution.Daily, Market.India).Symbol;
+            _splitAndDividendSymbol = AddEquity("CCCL", Resolution.Daily, Market.India).Symbol;
+        }
+
+        /// <summary>
+        /// Raises the data event.
+        /// </summary>
+        /// <param name="data">Data.</param>
+        public void OnData(Dividends data)
+        {
+            if (data.ContainsKey(_splitAndDividendSymbol))
+            {
+                var dividend = data[_splitAndDividendSymbol];
+                if (Time.Date == new DateTime(2010, 06, 15) &&
+                    (dividend.Price != 0.5m || dividend.ReferencePrice != 88.8m || dividend.Distribution != 0.5m))
+                {
+                    throw new Exception("Did not receive expected dividend values");
+                }
+            }
+        }
+
+        /// <summary>
+        /// Raises the data event.
+        /// </summary>
+        /// <param name="data">Data.</param>
+        public void OnData(Splits data)
+        {
+            if (data.ContainsKey(_splitAndDividendSymbol))
+            {
+                var split = data[_splitAndDividendSymbol];
+                if (split.Type == SplitType.Warning)
+                {
+                    _receivedWarningEvent = true;
+                }
+                else if (split.Type == SplitType.SplitOccurred)
+                {
+                    _receivedOccurredEvent = true;
+                    if (split.Price != 421m || split.ReferencePrice != 421m || split.SplitFactor != 0.2m)
+                    {
+                        throw new Exception("Did not receive expected split values");
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Checks the symbol change event
+        /// </summary>
+        public override void OnData(Slice slice)
+        {
+            if (slice.SymbolChangedEvents.ContainsKey(_mappingSymbol))
+            {
+                var mappingEvent = slice.SymbolChangedEvents.Single(x => x.Key.SecurityType == SecurityType.Equity).Value;
+                Log($"{Time} - Ticker changed from: {mappingEvent.OldSymbol} to {mappingEvent.NewSymbol}");
+                if (Time.Date == new DateTime(1999, 01, 01))
+                {
+                    _initialMapping = true;
+                }
+                else if (Time.Date == new DateTime(2004, 06, 15))
+                {
+                    if (mappingEvent.NewSymbol == "3MINDIA"
+                        && mappingEvent.OldSymbol == "BIRLA3M")
+                    {
+                        _executionMapping = true;
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Final step of the algorithm
+        /// </summary>
+        public override void OnEndOfAlgorithm()
+        {
+            if (_initialMapping)
+            {
+                throw new Exception("The ticker generated the initial rename event");
+            }
+            if (!_executionMapping)
+            {
+                throw new Exception("The ticker did not rename throughout the course of its life even though it should have");
+            }
+            if (!_receivedOccurredEvent)
+            {
+                throw new Exception("Did not receive expected split event");
+            }
+            if (!_receivedWarningEvent)
+            {
+                throw new Exception("Did not receive expected split warning event");
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public Language[] Languages { get; } = { Language.CSharp, Language.Python };
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Trades", "0"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "0%"},
+            {"Drawdown", "0%"},
+            {"Expectancy", "0"},
+            {"Net Profit", "0%"},
+            {"Sharpe Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "0%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "0"},
+            {"Tracking Error", "0"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "$0.00"},
+            {"Estimated Strategy Capacity", "$0"},
+            {"Lowest Capacity Asset", ""},
+            {"Fitness Score", "0"},
+            {"Kelly Criterion Estimate", "0"},
+            {"Kelly Criterion Probability Value", "0"},
+            {"Sortino Ratio", "79228162514264337593543950335"},
+            {"Return Over Maximum Drawdown", "79228162514264337593543950335"},
+            {"Portfolio Turnover", "0"},
+            {"Total Insights Generated", "0"},
+            {"Total Insights Closed", "0"},
+            {"Total Insights Analysis Completed", "0"},
+            {"Long Insight Count", "0"},
+            {"Short Insight Count", "0"},
+            {"Long/Short Ratio", "100%"},
+            {"Estimated Monthly Alpha Value", "₹0"},
+            {"Total Accumulated Estimated Alpha Value", "₹0"},
+            {"Mean Population Estimated Insight Value", "₹0"},
+            {"Mean Population Direction", "0%"},
+            {"Mean Population Magnitude", "0%"},
+            {"Rolling Averaged Population Direction", "0%"},
+            {"Rolling Averaged Population Magnitude", "0%"},
+            {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"}
+        };
+    }
+}

Algorithm.CSharp/IndicatorSuiteAlgorithm.cs

@@ -16,7 +16,6 @@
 using System;
 using System.Collections.Generic;
 using QuantConnect.Data;
-using QuantConnect.Data.Custom;
 using QuantConnect.Data.Market;
 using QuantConnect.Indicators;
 using QuantConnect.Interfaces;
@@ -34,7 +33,7 @@ namespace QuantConnect.Algorithm.CSharp
     public class IndicatorSuiteAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
         private string _ticker = "SPY";
-        private string _customTicker = "WIKI/FB";
+        private string _customTicker = "IBM";
 
         private Symbol _symbol;
         private Symbol _customSymbol;
@@ -64,7 +63,7 @@ namespace QuantConnect.Algorithm.CSharp
             _symbol = AddSecurity(SecurityType.Equity, _ticker, Resolution.Daily).Symbol;
 
             //Add the Custom Data:
-            _customSymbol = AddData<Quandl>(_customTicker, Resolution.Daily).Symbol;
+            _customSymbol = AddData<CustomData>(_customTicker, Resolution.Daily).Symbol;
 
             //Set up default Indicators, these indicators are defined on the Value property of incoming data (except ATR and AROON which use the full TradeBar object)
             _indicators = new Indicators
@@ -118,9 +117,9 @@ namespace QuantConnect.Algorithm.CSharp
             // these are indicators that require multiple inputs. the most common of which is a ratio.
             // suppose we seek the ratio of BTC to SPY, we could write the following:
             var spyClose = Identity(_symbol);
-            var fbClose = Identity(_customSymbol);
+            var ibmClose = Identity(_customSymbol);
             // this will create a new indicator whose value is FB/SPY
-            _ratio = fbClose.Over(spyClose);
+            _ratio = ibmClose.Over(spyClose);
             // we can also easily plot our indicators each time they update using th PlotIndicator function
             PlotIndicator("Ratio", _ratio);
         }
@@ -128,8 +127,8 @@ namespace QuantConnect.Algorithm.CSharp
         /// <summary>
         /// Custom data event handler:
         /// </summary>
-        /// <param name="data">Quandl - dictionary Bars of Quandl Data</param>
-        public void OnData(Quandl data)
+        /// <param name="data">CustomData - dictionary Bars of custom data</param>
+        public void OnData(CustomData data)
         {
         }
 

Algorithm.CSharp/ParameterizedAlgorithm.cs

@@ -28,9 +28,9 @@ namespace QuantConnect.Algorithm.CSharp
     /// <meta name="tag" content="using quantconnect" />
     public class ParameterizedAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
-        // we place attributes on top of our fields or properties that should receive
+        // We place attributes on top of our fields or properties that should receive
         // their values from the job. The values 100 and 200 are just default values that
-        // or only used if the parameters do not exist
+        // are only used if the parameters do not exist.
         [Parameter("ema-fast")]
         public int FastPeriod = 100;
 

Algorithm.CSharp/QuandlFuturesDataAlgorithm.cs

@@ -1,73 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using QuantConnect.Data.Custom;
-
-namespace QuantConnect.Algorithm.CSharp
-{
-    /// <summary>
-    /// Futures demonstration algorithm.
-    /// QuantConnect allows importing generic data sources! This example demonstrates importing a futures
-    /// data from the popular open data source Quandl. QuantConnect has a special deal with Quandl giving you access
-    /// to Stevens Continuous Futurs (SCF) for free. If you'd like to download SCF for local backtesting, you can download it through Quandl.com.
-    /// </summary>
-    /// <meta name="tag" content="using data" />
-    /// <meta name="tag" content="quandl" />
-    /// <meta name="tag" content="custom data" />
-    /// <meta name="tag" content="futures" />
-    public class QCUQuandlFutures : QCAlgorithm
-    {
-        private string _crude = "SCF/CME_CL1_ON";
-
-        /// <summary>
-        /// Initialize the data and resolution you require for your strategy
-        /// </summary>
-        public override void Initialize()
-        {
-            SetStartDate(2000, 1, 1);
-            SetEndDate(DateTime.Now.Date.AddDays(-1));
-            SetCash(25000);
-            AddData<QuandlFuture>(_crude, Resolution.Daily);
-        }
-
-        /// <summary>
-        /// Data Event Handler: New data arrives here. "TradeBars" type is a dictionary of strings so you can access it by symbol.
-        /// </summary>
-        /// <param name="data">Data.</param>
-        public void OnData(Quandl data)
-        {
-            if (!Portfolio.HoldStock)
-            {
-                SetHoldings(_crude, 1);
-                Debug(Time.ToStringInvariant("u") + " Purchased Crude Oil: " + _crude);
-            }
-        }
-
-        /// <summary>
-        /// Custom quandl data type for setting customized value column name. Value column is used for the primary trading calculations and charting.
-        /// </summary>
-        public class QuandlFuture : Quandl
-        {
-            /// <summary>
-            /// Initializes a new instance of the <see cref="QuandlFuture"/> class.
-            /// </summary>
-            public QuandlFuture()
-                : base(valueColumnName: "Settle")
-            {
-            }
-        }
-    }
-}

Algorithm.CSharp/QuandlImporterAlgorithm.cs

@@ -1,70 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using QuantConnect.Data.Custom;
-using QuantConnect.Indicators;
-
-namespace QuantConnect.Algorithm.CSharp
-{
-    /// <summary>
-    /// Using the underlying dynamic data class "Quandl" QuantConnect take care of the data
-    /// importing and definition for you. Simply point QuantConnect to the Quandl Short Code.
-    /// The Quandl object has properties which match the spreadsheet headers.
-    /// If you have multiple quandl streams look at data.Symbol to distinguish them.
-    /// </summary>
-    /// <meta name="tag" content="custom data" />
-    /// <meta name="tag" content="using data" />
-    /// <meta name="tag" content="quandl" />
-    public class QuandlImporterAlgorithm : QCAlgorithm
-    {
-        private SimpleMovingAverage _sma;
-        string _quandlCode = "WIKI/IBM";
-
-        /// Initialize the data and resolution you require for your strategy:
-        public override void Initialize()
-        {
-            //Start and End Date range for the backtest:
-            SetStartDate(2013, 1, 1);
-            SetEndDate(DateTime.Now.Date.AddDays(-1));
-
-            //Cash allocation
-            SetCash(25000);
-
-            // Optional argument - personal token necessary for restricted dataset
-            // Quandl.SetAuthCode("your-quandl-token");
-
-            //Add Generic Quandl Data:
-            AddData<Quandl>(_quandlCode, Resolution.Daily);
-
-            _sma = SMA(_quandlCode, 14);
-        }
-
-        /// Data Event Handler: New data arrives here. "TradeBars" type is a dictionary of strings so you can access it by symbol
-        public void OnData(Quandl data)
-        {
-            if (!Portfolio.HoldStock)
-            {
-                //Order function places trades: enter the string symbol and the quantity you want:
-                SetHoldings(_quandlCode, 1);
-
-                //Debug sends messages to the user console: "Time" is the algorithm time keeper object
-                Debug("Purchased " + _quandlCode + " >> " + Time.ToShortDateString());
-            }
-
-            Plot("SPY", _sma);
-        }
-    }
-}

Algorithm.CSharp/QuantConnect.Algorithm.CSharp.csproj

@@ -35,7 +35,7 @@
     <DebugType>portable</DebugType>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Accord" Version="3.6.0" />
     <PackageReference Include="Accord.Fuzzy" Version="3.6.0" />
     <PackageReference Include="Accord.MachineLearning" Version="3.6.0" />

Algorithm.Framework/QuantConnect.Algorithm.Framework.csproj

@@ -30,7 +30,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Accord" Version="3.6.0" />
     <PackageReference Include="Accord.Math" Version="3.6.0" />
     <PackageReference Include="Accord.Statistics" Version="3.6.0" />

Algorithm.Python/BasicTemplateFuturesAlgorithm.py

@@ -43,6 +43,8 @@ class BasicTemplateFuturesAlgorithm(QCAlgorithm):
         benchmark = self.AddEquity("SPY")
         self.SetBenchmark(benchmark.Symbol)
 
+        seeder = FuncSecuritySeeder(self.GetLastKnownPrices)
+        self.SetSecurityInitializer(lambda security: seeder.SeedSecurity(security))
 
     def OnData(self,slice):
         if not self.Portfolio.Invested:
@@ -70,3 +72,8 @@ class BasicTemplateFuturesAlgorithm(QCAlgorithm):
         maintenanceOvernight = buyingPowerModel.MaintenanceOvernightMarginRequirement
         initialIntraday = buyingPowerModel.InitialIntradayMarginRequirement
         maintenanceIntraday = buyingPowerModel.MaintenanceIntradayMarginRequirement
+
+    def OnSecuritiesChanged(self, changes):
+        for addedSecurity in changes.AddedSecurities:
+            if addedSecurity.Symbol.SecurityType == SecurityType.Future and not addedSecurity.Symbol.IsCanonical() and not addedSecurity.HasData:
+                raise Exception(f"Future contracts did not work up as expected: {addedSecurity.Symbol}")

Algorithm.Python/BasicTemplateIndiaAlgorithm.py

@@ -0,0 +1,50 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+### <summary>
+### Basic template framework algorithm uses framework components to define the algorithm.
+### </summary>
+### <meta name="tag" content="using data" />
+### <meta name="tag" content="using quantconnect" />
+### <meta name="tag" content="trading and orders" />
+class BasicTemplateIndiaAlgorithm(QCAlgorithm):
+    '''Basic template framework algorithm uses framework components to define the algorithm.'''
+
+    def Initialize(self):
+        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
+
+        self.SetAccountCurrency("INR")  #Set Account Currency 
+        self.SetStartDate(2019, 1, 23)  #Set Start Date
+        self.SetEndDate(2019, 10, 31)   #Set End Date
+        self.SetCash(100000)            #Set Strategy Cash
+        # Find more symbols here: http://quantconnect.com/data
+        self.AddEquity("YESBANK", Resolution.Minute, Market.India)
+        self.Debug("numpy test >>> print numpy.pi: " + str(np.pi))
+
+        # Set Order Prperties as per the requirements for order placement
+        self.DefaultOrderProperties = IndiaOrderProperties(Exchange.NSE)
+
+    def OnData(self, data):
+        '''OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
+
+        Arguments:
+            data: Slice object keyed by symbol containing the stock data
+        '''
+        if not self.Portfolio.Invested:
+            self.MarketOrder("YESBANK", 1)
+
+    def OnOrderEvent(self, orderEvent):
+        if orderEvent.Status == OrderStatus.Filled:
+            self.Debug("Purchased Stock: {0}".format(orderEvent.Symbol))

Algorithm.Python/BasicTemplateIndiaIndexAlgorithm.py

@@ -0,0 +1,71 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+### <summary>
+### Basic Template India Index Algorithm uses framework components to define the algorithm.
+### </summary>
+### <meta name="tag" content="using data" />
+### <meta name="tag" content="using quantconnect" />
+### <meta name="tag" content="trading and orders" />
+class BasicTemplateIndiaIndexAlgorithm(QCAlgorithm):
+    '''Basic template framework algorithm uses framework components to define the algorithm.'''
+
+    def Initialize(self):
+        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
+
+        self.SetAccountCurrency("INR") #Set Account Currency
+        self.SetStartDate(2019, 1, 1)  #Set Start Date
+        self.SetEndDate(2019, 1, 5)    #Set End Date
+        self.SetCash(1000000)          #Set Strategy Cash
+
+        # Use indicator for signal; but it cannot be traded
+        self.Nifty = self.AddIndex("NIFTY50", Resolution.Minute, Market.India).Symbol
+        # Trade Index based ETF
+        self.NiftyETF = self.AddEquity("JUNIORBEES", Resolution.Minute, Market.India).Symbol
+   
+        # Set Order Prperties as per the requirements for order placement
+        self.DefaultOrderProperties = IndiaOrderProperties(Exchange.NSE)
+
+        # Define indicator
+        self._emaSlow = self.EMA(self.Nifty, 80);
+        self._emaFast = self.EMA(self.Nifty, 200);
+
+        self.Debug("numpy test >>> print numpy.pi: " + str(np.pi))
+
+
+    def OnData(self, data):
+        '''OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
+
+        Arguments:
+            data: Slice object keyed by symbol containing the stock data
+        '''
+
+        if not data.Bars.ContainsKey(self.Nifty) or not data.Bars.ContainsKey(self.NiftyETF):
+            return
+
+        if not self._emaSlow.IsReady:
+            return
+
+        if self._emaFast > self._emaSlow:
+            if not self.Portfolio.Invested:
+                self.marketTicket = self.MarketOrder(self.NiftyETF, 1)
+        else:
+            self.Liquidate()
+
+
+    def OnEndOfAlgorithm(self):
+        if self.Portfolio[self.Nifty].TotalSaleVolume > 0:
+            raise Exception("Index is not tradable.")
+

Algorithm.Python/CustomDataIndicatorExtensionsAlgorithm.py

@@ -12,10 +12,11 @@
 # limitations under the License.
 
 from AlgorithmImports import *
+from HistoryAlgorithm import *
 
 ### <summary>
 ### The algorithm creates new indicator value with the existing indicator method by Indicator Extensions
-### Demonstration of using the external custom datasource Quandl to request the VIX and VXV daily data
+### Demonstration of using the external custom data to request the IBM and SPY daily data
 ### </summary>
 ### <meta name="tag" content="using data" />
 ### <meta name="tag" content="using quantconnect" />
@@ -33,38 +34,30 @@ class CustomDataIndicatorExtensionsAlgorithm(QCAlgorithm):
         self.SetEndDate(2018,1,1)  
         self.SetCash(25000)
         
-        self.vix = 'CBOE/VIX'
-        self.vxv = 'CBOE/VXV'
+        self.ibm = 'IBM'
+        self.spy = 'SPY'
         
         # Define the symbol and "type" of our generic data
-        self.AddData(QuandlVix, self.vix, Resolution.Daily)
-        self.AddData(Quandl, self.vxv, Resolution.Daily)
+        self.AddData(CustomDataEquity, self.ibm, Resolution.Daily)
+        self.AddData(CustomDataEquity, self.spy, Resolution.Daily)
         
         # Set up default Indicators, these are just 'identities' of the closing price
-        self.vix_sma = self.SMA(self.vix, 1, Resolution.Daily)
-        self.vxv_sma = self.SMA(self.vxv, 1, Resolution.Daily)
+        self.ibm_sma = self.SMA(self.ibm, 1, Resolution.Daily)
+        self.spy_sma = self.SMA(self.spy, 1, Resolution.Daily)
         
-        # This will create a new indicator whose value is smaVXV / smaVIX
-        self.ratio = IndicatorExtensions.Over(self.vxv_sma, self.vix_sma)
+        # This will create a new indicator whose value is smaSPY / smaIBM
+        self.ratio = IndicatorExtensions.Over(self.spy_sma, self.ibm_sma)
         
         # Plot indicators each time they update using the PlotIndicator function
         self.PlotIndicator("Ratio", self.ratio)
-        self.PlotIndicator("Data", self.vix_sma, self.vxv_sma)
+        self.PlotIndicator("Data", self.ibm_sma, self.spy_sma)
     
     # OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
     def OnData(self, data):
         
         # Wait for all indicators to fully initialize
-        if not (self.vix_sma.IsReady and self.vxv_sma.IsReady and self.ratio.IsReady): return
+        if not (self.ibm_sma.IsReady and self.spy_sma.IsReady and self.ratio.IsReady): return
         if not self.Portfolio.Invested and self.ratio.Current.Value > 1:
-            self.MarketOrder(self.vix, 100)
+            self.MarketOrder(self.ibm, 100)
         elif self.ratio.Current.Value < 1:
                 self.Liquidate()
-
-# In CBOE/VIX data, there is a "vix close" column instead of "close" which is the 
-# default column namein LEAN Quandl custom data implementation.
-# This class assigns new column name to match the the external datasource setting.
-class QuandlVix(PythonQuandl):
-    
-    def __init__(self):
-        self.ValueColumnName = "VIX Close"

Algorithm.Python/HistoryAlgorithm.py

@@ -30,12 +30,11 @@ class HistoryAlgorithm(QCAlgorithm):
         self.SetCash(100000)           #Set Strategy Cash
         # Find more symbols here: http://quantconnect.com/data
         self.AddEquity("SPY", Resolution.Daily)
-        self.AddData(QuandlFuture,"CHRIS/CME_SP1", Resolution.Daily)
+        self.AddData(CustomDataEquity, "IBM", Resolution.Daily)
         # specifying the exchange will allow the history methods that accept a number of bars to return to work properly
-        self.Securities["CHRIS/CME_SP1"].Exchange = EquityExchange()
 
         # we can get history in initialize to set up indicators and such
-        self.spyDailySma = SimpleMovingAverage(14)
+        self.dailySma = SimpleMovingAverage(14)
 
         # get the last calendar year's worth of SPY data at the configured resolution (daily)
         tradeBarHistory = self.History([self.Securities["SPY"].Symbol], timedelta(365))
@@ -56,56 +55,52 @@ class HistoryAlgorithm(QCAlgorithm):
         # we can loop over the return value from these functions and we get TradeBars
         # we can use these TradeBars to initialize indicators or perform other math
         for index, tradeBar in tradeBarHistory.loc["SPY"].iterrows():
-            self.spyDailySma.Update(index, tradeBar["close"])
+            self.dailySma.Update(index, tradeBar["close"])
 
-        # get the last calendar year's worth of quandl data at the configured resolution (daily)
-        quandlHistory = self.History(QuandlFuture, "CHRIS/CME_SP1", timedelta(365))
-        self.AssertHistoryCount("History(QuandlFuture, \"CHRIS/CME_SP1\", timedelta(365))", quandlHistory, 250)
+        # get the last calendar year's worth of customData data at the configured resolution (daily)
+        customDataHistory = self.History(CustomDataEquity, "IBM", timedelta(365))
+        self.AssertHistoryCount("History(CustomDataEquity, \"IBM\", timedelta(365))", customDataHistory, 10)
 
-        # get the last 14 bars of SPY at the configured resolution (daily)
-        quandlHistory = self.History(QuandlFuture, "CHRIS/CME_SP1", 14)
-        self.AssertHistoryCount("History(QuandlFuture, \"CHRIS/CME_SP1\", 14)", quandlHistory, 14)
+        # get the last 10 bars of IBM at the configured resolution (daily)
+        customDataHistory = self.History(CustomDataEquity, "IBM", 14)
+        self.AssertHistoryCount("History(CustomDataEquity, \"IBM\", 14)", customDataHistory, 10)
 
-        # we can loop over the return values from these functions and we'll get Quandl data
+        # we can loop over the return values from these functions and we'll get Custom data
         # this can be used in much the same way as the tradeBarHistory above
-        self.spyDailySma.Reset()
-        for index, quandl in quandlHistory.loc["CHRIS/CME_SP1"].iterrows():
-            self.spyDailySma.Update(index, quandl["settle"])
+        self.dailySma.Reset()
+        for index, customData in customDataHistory.loc["IBM"].iterrows():
+            self.dailySma.Update(index, customData["value"])
 
-        # get the last year's worth of all configured Quandl data at the configured resolution (daily)
-        #allQuandlData = self.History(QuandlFuture, timedelta(365))
-        #self.AssertHistoryCount("History(QuandlFuture, timedelta(365))", allQuandlData, 250)
+        # get the last 10 bars worth of Custom data for the specified symbols at the configured resolution (daily)
+        allCustomData = self.History(CustomDataEquity, self.Securities.Keys, 14)
+        self.AssertHistoryCount("History(CustomDataEquity, self.Securities.Keys, 14)", allCustomData, 10)
 
-        # get the last 14 bars worth of Quandl data for the specified symbols at the configured resolution (daily)
-        allQuandlData = self.History(QuandlFuture, self.Securities.Keys, 14)
-        self.AssertHistoryCount("History(QuandlFuture, self.Securities.Keys, 14)", allQuandlData, 14)
+        # NOTE: Using different resolutions require that they are properly implemented in your data type. If your
+        #  custom data source has different resolutions, it would need to be implemented in the GetSource and 
+        #  Reader methods properly.
+        #customDataHistory = self.History(CustomDataEquity, "IBM", timedelta(7), Resolution.Minute)
+        #customDataHistory = self.History(CustomDataEquity, "IBM", 14, Resolution.Minute)
+        #allCustomData = self.History(CustomDataEquity, timedelta(365), Resolution.Minute)
+        #allCustomData = self.History(CustomDataEquity, self.Securities.Keys, 14, Resolution.Minute)
+        #allCustomData = self.History(CustomDataEquity, self.Securities.Keys, timedelta(1), Resolution.Minute)
+        #allCustomData = self.History(CustomDataEquity, self.Securities.Keys, 14, Resolution.Minute)
 
-        # NOTE: using different resolutions require that they are properly implemented in your data type, since
-        #  Quandl doesn't support minute data, this won't actually work, but if your custom data source has
-        #  different resolutions, it would need to be implemented in the GetSource and Reader methods properly
-        #quandlHistory = self.History(QuandlFuture, "CHRIS/CME_SP1", timedelta(7), Resolution.Minute)
-        #quandlHistory = self.History(QuandlFuture, "CHRIS/CME_SP1", 14, Resolution.Minute)
-        #allQuandlData = self.History(QuandlFuture, timedelta(365), Resolution.Minute)
-        #allQuandlData = self.History(QuandlFuture, self.Securities.Keys, 14, Resolution.Minute)
-        #allQuandlData = self.History(QuandlFuture, self.Securities.Keys, timedelta(1), Resolution.Minute)
-        #allQuandlData = self.History(QuandlFuture, self.Securities.Keys, 14, Resolution.Minute)
-
-        # get the last calendar year's worth of all quandl data
-        allQuandlData = self.History(QuandlFuture, self.Securities.Keys, timedelta(365))
-        self.AssertHistoryCount("History(QuandlFuture, self.Securities.Keys, timedelta(365))", allQuandlData, 250)
+        # get the last calendar year's worth of all customData data
+        allCustomData = self.History(CustomDataEquity, self.Securities.Keys, timedelta(365))
+        self.AssertHistoryCount("History(CustomDataEquity, self.Securities.Keys, timedelta(365))", allCustomData, 10)
 
         # we can also access the return value from the multiple symbol functions to request a single
         # symbol and then loop over it
-        singleSymbolQuandl = allQuandlData.loc["CHRIS/CME_SP1"]
-        self.AssertHistoryCount("allQuandlData.loc[\"CHRIS/CME_SP1\"]", singleSymbolQuandl, 250)
-        for  quandl in singleSymbolQuandl:
-            # do something with 'CHRIS/CME_SP1.QuandlFuture' quandl data
+        singleSymbolCustom = allCustomData.loc["IBM"]
+        self.AssertHistoryCount("allCustomData.loc[\"IBM\"]", singleSymbolCustom, 10)
+        for  customData in singleSymbolCustom:
+            # do something with 'IBM.CustomDataEquity' customData data
             pass
 
-        quandlSpyLows = allQuandlData.loc["CHRIS/CME_SP1"]["low"]
-        self.AssertHistoryCount("allQuandlData.loc[\"CHRIS/CME_SP1\"][\"low\"]", quandlSpyLows, 250)
-        for  low in quandlSpyLows:
-            # do something with 'CHRIS/CME_SP1.QuandlFuture' quandl data
+        customDataSpyValues = allCustomData.loc["IBM"]["value"]
+        self.AssertHistoryCount("allCustomData.loc[\"IBM\"][\"value\"]", customDataSpyValues, 10)
+        for value in customDataSpyValues:
+            # do something with 'IBM.CustomDataEquity' value data
             pass
 
 
@@ -124,10 +119,20 @@ class HistoryAlgorithm(QCAlgorithm):
             raise Exception("{} expected {}, but received {}".format(methodCall, expected, count))
 
 
-class QuandlFuture(PythonQuandl):
-    '''Custom quandl data type for setting customized value column name. Value column is used for the primary trading calculations and charting.'''
-    def __init__(self):
-        # Define ValueColumnName: cannot be None, Empty or non-existant column name
-        # If ValueColumnName is "Close", do not use PythonQuandl, use Quandl:
-        # self.AddData[QuandlFuture](self.crude, Resolution.Daily)
-        self.ValueColumnName = "Settle"
+class CustomDataEquity(PythonData):
+    def GetSource(self, config, date, isLive):
+        source = "https://www.dl.dropboxusercontent.com/s/o6ili2svndzn556/custom_data.csv?dl=0"
+        return SubscriptionDataSource(source, SubscriptionTransportMedium.RemoteFile)
+
+    def Reader(self, config, line, date, isLive):
+        if line == None:
+            return None
+
+        customData = CustomDataEquity()
+        customData.Symbol = config.Symbol
+
+        csv = line.split(",")
+        customData.Time = datetime.strptime(csv[0], '%Y%m%d %H:%M')
+        customData.EndTime = customData.Time + timedelta(days=1)
+        customData.Value = float(csv[1])
+        return customData

Algorithm.Python/IndiaDataRegressionAlgorithm.py

@@ -0,0 +1,83 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+### <summary>
+### Basic template framework algorithm uses framework components to define the algorithm.
+### </summary>
+### <meta name="tag" content="using data" />
+### <meta name="tag" content="using quantconnect" />
+### <meta name="tag" content="trading and orders" />
+class IndiaDataRegressionAlgorithm(QCAlgorithm):
+    '''Basic template framework algorithm uses framework components to define the algorithm.'''
+
+    def Initialize(self):
+        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
+        
+        self.SetAccountCurrency("INR")
+        self.SetStartDate(2004, 5, 20)
+        self.SetEndDate(2016, 7, 26) 
+        self._mappingSymbol = self.AddEquity("3MINDIA", Resolution.Daily, Market.India).Symbol
+        self._splitAndDividendSymbol = self.AddEquity("CCCL", Resolution.Daily, Market.India).Symbol
+        self._receivedWarningEvent = False
+        self._receivedOccurredEvent = False
+        self._initialMapping = False
+        self._executionMapping = False
+        self.Debug("numpy test >>> print numpy.pi: " + str(np.pi))
+
+    def OnData(self, data):
+        '''OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
+
+        Arguments:
+            data: Slice object keyed by symbol containing the stock data
+        '''
+
+        # dividend
+        if data.Dividends.ContainsKey(self._splitAndDividendSymbol):
+            dividend = data.Dividends[self._splitAndDividendSymbol]
+            if ((self.Time.year == 2010 and self.Time.month == 6 and self.Time.day == 15) and
+                    (dividend.Price != 0.5 or dividend.ReferencePrice != 88.8 or dividend.Distribution != 0.5)):
+                raise Exception("Did not receive expected dividend values")
+
+        # split
+        if data.Splits.ContainsKey(self._splitAndDividendSymbol):
+            split = data.Splits[self._splitAndDividendSymbol]
+            if split.Type == SplitType.Warning:
+                self._receivedWarningEvent = True
+            elif split.Type == SplitType.SplitOccurred:
+                self._receivedOccurredEvent = True
+                if split.Price != 421.0 or split.ReferencePrice != 421.0 or split.SplitFactor != 0.2:
+                    raise Exception("Did not receive expected price values")
+
+        # mapping
+        if data.SymbolChangedEvents.ContainsKey(self._mappingSymbol):
+                mappingEvent = [x.Value for x in data.SymbolChangedEvents if x.Key.SecurityType == 1][0]
+                if self.Time.year == 1999 and self.Time.month == 1 and self.Time.day == 1:
+                    self._initialMapping = True
+                elif self.Time.year == 2004 and self.Time.month == 6 and self.Time.day == 15:
+                    if mappingEvent.NewSymbol == "3MINDIA" and mappingEvent.OldSymbol == "BIRLA3M":
+                        self._executionMapping = True
+    
+    def OnEndOfAlgorithm(self):
+        if self._initialMapping:
+            raise Exception("The ticker generated the initial rename event")
+
+        if not self._executionMapping:
+            raise Exception("The ticker did not rename throughout the course of its life even though it should have")
+        
+        if not self._receivedOccurredEvent:
+            raise Exception("Did not receive expected split event")
+
+        if not self._receivedWarningEvent:
+            raise Exception("Did not receive expected split warning event")

Algorithm.Python/IndicatorSuiteAlgorithm.py

@@ -27,7 +27,7 @@ class IndicatorSuiteAlgorithm(QCAlgorithm):
         '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
 
         self.symbol = "SPY"
-        self.customSymbol = "WIKI/FB"
+        self.customSymbol = "IBM"
         self.price = None
 
         self.SetStartDate(2013, 1, 1)  #Set Start Date
@@ -36,7 +36,7 @@ class IndicatorSuiteAlgorithm(QCAlgorithm):
         # Find more symbols here: http://quantconnect.com/data
 
         self.AddEquity(self.symbol, Resolution.Daily)
-        self.AddData(Quandl, self.customSymbol, Resolution.Daily)
+        self.AddData(CustomData, self.customSymbol, Resolution.Daily)
 
         # Set up default Indicators, these indicators are defined on the Value property of incoming data (except ATR and AROON which use the full TradeBar object)
         self.indicators = {
@@ -88,10 +88,10 @@ class IndicatorSuiteAlgorithm(QCAlgorithm):
         # these are indicators that require multiple inputs. the most common of which is a ratio.
         # suppose we seek the ratio of BTC to SPY, we could write the following:
         spyClose = Identity(self.symbol)
-        fbClose = Identity(self.customSymbol)
+        ibmClose = Identity(self.customSymbol)
 
-        # this will create a new indicator whose value is FB/SPY
-        self.ratio = IndicatorExtensions.Over(fbClose, spyClose)
+        # this will create a new indicator whose value is IBM/SPY
+        self.ratio = IndicatorExtensions.Over(ibmClose, spyClose)
 
         # we can also easily plot our indicators each time they update using th PlotIndicator function
         self.PlotIndicator("Ratio", self.ratio)

Algorithm.Python/QuandlFuturesDataAlgorithm.py

@@ -1,53 +0,0 @@
-# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
-# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from AlgorithmImports import *
-
-### <summary>
-### Futures demonstration algorithm.
-### QuantConnect allows importing generic data sources! This example demonstrates importing a futures
-### data from the popular open data source Quandl. QuantConnect has a special deal with Quandl giving you access
-### to Stevens Continuous Futurs (SCF) for free. If you'd like to download SCF for local backtesting, you can download it through Quandl.com.
-### </summary>
-### <meta name="tag" content="using data" />
-### <meta name="tag" content="quandl" />
-### <meta name="tag" content="custom data" />
-### <meta name="tag" content="futures" />
-class QuandlFuturesDataAlgorithm(QCAlgorithm):
-
-    def Initialize(self):
-        ''' Initialize the data and resolution you require for your strategy '''
-        self.SetStartDate(2000, 1, 1)
-        self.SetEndDate(datetime.now().date() - timedelta(1))
-        self.SetCash(25000)
-
-        # Symbol corresponding to the quandl code
-        self.crude = "SCF/CME_CL1_ON"
-        self.AddData(QuandlFuture, self.crude, Resolution.Daily)
-
-
-    def OnData(self, data):
-        '''Data Event Handler: New data arrives here. "TradeBars" type is a dictionary of strings so you can access it by symbol.'''
-        if self.Portfolio.HoldStock: return
-
-        self.SetHoldings(self.crude, 1)
-        self.Debug(str(self.Time) + str(" Purchased Crude Oil: ") + self.crude)
-
-
-class QuandlFuture(PythonQuandl):
-    '''Custom quandl data type for setting customized value column name. Value column is used for the primary trading calculations and charting.'''
-    def __init__(self):
-        # Define ValueColumnName: cannot be None, Empty or non-existant column name
-        # If ValueColumnName is "Close", do not use PythonQuandl, use Quandl:
-        # self.AddData[QuandlFuture](self.crude, Resolution.Daily)
-        self.ValueColumnName = "Settle"

Algorithm.Python/QuandlImporterAlgorithm.py

@@ -1,51 +0,0 @@
-# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
-# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from AlgorithmImports import *
-
-### <summary>
-### Using the underlying dynamic data class "Quandl" QuantConnect take care of the data
-### importing and definition for you. Simply point QuantConnect to the Quandl Short Code.
-### The Quandl object has properties which match the spreadsheet headers.
-### If you have multiple quandl streams look at data.Symbol to distinguish them.
-### </summary>
-### <meta name="tag" content="custom data" />
-### <meta name="tag" content="using data" />
-### <meta name="tag" content="quandl" />
-class QuandlImporterAlgorithm(QCAlgorithm):
-
-    def Initialize(self):
-        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
-        self.quandlCode = "WIKI/IBM"
-        ## Optional argument - personal token necessary for restricted dataset
-        # Quandl.SetAuthCode("your-quandl-token")
-        self.SetStartDate(2014,4,1)                                 #Set Start Date
-        self.SetEndDate(datetime.today() - timedelta(1))            #Set End Date
-        self.SetCash(25000)                                         #Set Strategy Cash
-        self.AddData(QuandlCustomColumns, self.quandlCode, Resolution.Daily, TimeZones.NewYork)
-        self.sma = self.SMA(self.quandlCode, 14)
-
-    def OnData(self, data):
-        '''OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.'''
-        if not self.Portfolio.HoldStock:
-            self.SetHoldings(self.quandlCode, 1)
-            self.Debug("Purchased {0} >> {1}".format(self.quandlCode, self.Time))
-
-        self.Plot(self.quandlCode, "PriceSMA", self.sma.Current.Value)
-        
-# Quandl often doesn't use close columns so need to tell LEAN which is the "value" column.
-class QuandlCustomColumns(PythonQuandl):
-    '''Custom quandl data type for setting customized value column name. Value column is used for the primary trading calculations and charting.'''
-    def __init__(self):
-        # Define ValueColumnName: cannot be None, Empty or non-existant column name
-        self.ValueColumnName = "adj. close"

Algorithm.Python/QuantConnect.Algorithm.Python.csproj

@@ -37,7 +37,7 @@
     <Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
@@ -222,8 +222,6 @@
     <None Include="OptionSplitRegressionAlgorithm.py" />
     <None Include="OrderTicketDemoAlgorithm.py" />
     <None Include="ParameterizedAlgorithm.py" />
-    <None Include="QuandlFuturesDataAlgorithm.py" />
-    <None Include="QuandlImporterAlgorithm.py" />
     <None Include="readme.md" />
     <None Include="RawPricesCoarseUniverseAlgorithm.py" />
     <None Include="RegressionAlgorithm.py" />

Algorithm/QCAlgorithm.History.cs

@@ -513,16 +513,18 @@ namespace QuantConnect.Algorithm
             }
 
             var result = new Dictionary<TickType, BaseData>();
-            // For speed and memory usage, use Resolution.Minute as the minimum resolution
-            var resolution = (Resolution)Math.Max((int)Resolution.Minute,
-                (int)SubscriptionManager.SubscriptionDataConfigService.GetSubscriptionDataConfigs(symbol).GetHighestResolution());
+            Resolution? resolution = null;
             Func<int, bool> requestData = period =>
             {
-                var historyRequests = CreateBarCountHistoryRequests(new[] { symbol }, period, resolution)
+                var historyRequests = CreateBarCountHistoryRequests(new[] { symbol }, period)
                     .Select(request =>
                     {
+                        // For speed and memory usage, use Resolution.Minute as the minimum resolution
+                        request.Resolution = (Resolution)Math.Max((int)Resolution.Minute, (int)request.Resolution);
                         // force no fill forward behavior
                         request.FillForwardResolution = null;
+
+                        resolution = request.Resolution;
                         return request;
                     })
                     // request only those tick types we didn't get the data we wanted
@@ -547,12 +549,21 @@ namespace QuantConnect.Algorithm
 
             if (!requestData(5))
             {
-                // If the first attempt to get the last know price returns null, it maybe the case of an illiquid security.
-                // We increase the look-back period for this case accordingly to the resolution to cover 3 trading days
-                var periods =
-                    resolution == Resolution.Daily ? 3 :
-                    resolution == Resolution.Hour ? 24 : 1440;
-                requestData(periods);
+                if (resolution.HasValue)
+                {
+                    // If the first attempt to get the last know price returns null, it maybe the case of an illiquid security.
+                    // We increase the look-back period for this case accordingly to the resolution to cover 3 trading days
+                    var periods =
+                        resolution.Value == Resolution.Daily ? 3 :
+                        resolution.Value == Resolution.Hour ? 24 : 1440;
+                    requestData(periods);
+                }
+                else
+                {
+                    // this shouldn't happen but just in case
+                    QuantConnect.Logging.Log.Error(
+                        $"QCAlgorithm.GetLastKnownPrices(): no history request was created for symbol {symbol} at {Time}");
+                }
             }
             // return the data ordered by time ascending
             return result.Values.OrderBy(data => data.Time);
@@ -804,9 +815,30 @@ namespace QuantConnect.Algorithm
             Security security;
             if (Securities.TryGetValue(symbol, out security))
             {
-                return resolution ?? SubscriptionManager.SubscriptionDataConfigService
-                    .GetSubscriptionDataConfigs(symbol)
-                    .GetHighestResolution();
+                if (resolution != null)
+                {
+                    return resolution.Value;
+                }
+
+                Resolution? result = null;
+                var hasNonInternal = false;
+                foreach (var config in SubscriptionManager.SubscriptionDataConfigService
+                    .GetSubscriptionDataConfigs(symbol, includeInternalConfigs: true)
+                    // we process non internal configs first
+                    .OrderBy(config => config.IsInternalFeed ? 1 : 0))
+                {
+                    if (!config.IsInternalFeed || !hasNonInternal)
+                    {
+                        // once we find a non internal config we ignore internals
+                        hasNonInternal |= !config.IsInternalFeed;
+                        if (!result.HasValue || config.Resolution < result)
+                        {
+                            result = config.Resolution;
+                        }
+                    }
+                }
+
+                return result ?? UniverseSettings.Resolution;
             }
             else
             {

Algorithm/QCAlgorithm.Universe.cs

@@ -657,27 +657,9 @@ namespace QuantConnect.Algorithm
                 bar.Symbol = security.Symbol;
                 
                 var maxSupportedResolution = bar.SupportedResolutions().Max();
-
                 var updateFrequency = maxSupportedResolution.ToTimeSpan();
-                int periods;
-                switch (maxSupportedResolution)
-                {
-                    case Resolution.Tick:
-                    case Resolution.Second:
-                        periods = 600;
-                        break;
-                    case Resolution.Minute:
-                        periods = 60 * 24;
-                        break;
-                    case Resolution.Hour:
-                        periods = 24 * 30;
-                        break;
-                    default:
-                        periods = 30;
-                        break;
-                }
                 
-                security.VolatilityModel = new StandardDeviationOfReturnsVolatilityModel(periods, maxSupportedResolution, updateFrequency);
+                security.VolatilityModel = new StandardDeviationOfReturnsVolatilityModel(maxSupportedResolution, updateFrequency);
             }
         }
 

Algorithm/QCAlgorithm.cs

@@ -1680,6 +1680,12 @@ namespace QuantConnect.Algorithm
                             contractDepthOffset: contractOffset
                         );
 
+                        // let's add a MHDB entry for the continuous symbol using the associated security
+                        var continuousContractSymbol = ContinuousContractUniverse.CreateSymbol(security.Symbol);
+                        MarketHoursDatabase.SetEntry(continuousContractSymbol.ID.Market,
+                            continuousContractSymbol.ID.Symbol,
+                            continuousContractSymbol.ID.SecurityType,
+                            security.Exchange.Hours);
                         AddUniverse(new ContinuousContractUniverse(security, new UniverseSettings(settings)
                             {
                                 DataMappingMode = continuousConfigs.First().DataMappingMode,
@@ -1687,7 +1693,7 @@ namespace QuantConnect.Algorithm
                                 ContractDepthOffset = (int)continuousConfigs.First().ContractDepthOffset,
                                 SubscriptionDataTypes = dataTypes
                             }, LiveMode,
-                            new SubscriptionDataConfig(canonicalConfig, symbol: ContinuousContractUniverse.CreateSymbol(security.Symbol))));
+                            new SubscriptionDataConfig(canonicalConfig, symbol: continuousContractSymbol)));
 
                         universe = new FuturesChainUniverse((Future)security, settings);
                     }

Algorithm/QuantConnect.Algorithm.csproj

@@ -30,7 +30,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>

AlgorithmFactory/DebuggerHelper.cs

@@ -48,11 +48,18 @@ namespace QuantConnect.AlgorithmFactory
 
             /// <summary>
             ///  Python Tool for Visual Studio Debugger for remote python debugging.
-            /// <see cref="Language.Python"/> will use 'Python Extension in VS Code' 
-            ///or 'Python Tools in Visual Studio'
+            /// <see cref="Language.Python"/>. Deprecated, routes to DebugPy which
+            /// is it's replacement. Used in the same way.
             /// </summary>
             PTVSD,
 
+            /// <summary>
+            ///  DebugPy - a debugger for Python.
+            /// <see cref="Language.Python"/> can use  `Python Extension` in VS Code
+            /// or attach to Python in Visual Studio
+            /// </summary>
+            DebugPy,
+
             /// <summary>
             ///  PyCharm PyDev Debugger for remote python debugging.
             /// <see cref="Language.Python"/> will use 'Python Debug Server' in PyCharm
@@ -68,7 +75,7 @@ namespace QuantConnect.AlgorithmFactory
             if (language == Language.Python)
             {
                 DebuggingMethod debuggingType;
-                Enum.TryParse(Config.Get("debugging-method", DebuggingMethod.LocalCmdline.ToString()), out debuggingType);
+                Enum.TryParse(Config.Get("debugging-method", DebuggingMethod.LocalCmdline.ToString()), true, out debuggingType);
 
                 Log.Trace("DebuggerHelper.Initialize(): initializing python...");
                 PythonInitializer.Initialize();
@@ -91,8 +98,9 @@ while not sys.gettrace():
                             break;
 
                         case DebuggingMethod.PTVSD:
-                            Log.Trace("DebuggerHelper.Initialize(): waiting for PTVSD debugger to attach at localhost:5678...");
-                            PythonEngine.RunSimpleString("import ptvsd; ptvsd.enable_attach(); ptvsd.wait_for_attach()");
+                        case DebuggingMethod.DebugPy:
+                            Log.Trace("DebuggerHelper.Initialize(): debugpy waiting for attach at port 5678...");
+                            PythonEngine.RunSimpleString("import debugpy; debugpy.listen(('0.0.0.0', 5678)); debugpy.wait_for_client()");
                             break;
 
                         case DebuggingMethod.PyCharm:

AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs

@@ -83,7 +83,7 @@ namespace QuantConnect.AlgorithmFactory.Python.Wrappers
                         var repr = attr.Repr().GetStringBetweenChars('\'', '\'');
 
                         if (repr.StartsWith(moduleName) &&                // Must be defined in the module
-                            attr.TryConvert(out type) &&                  // Must be a Type
+                            attr.TryConvert(out type, true) &&                  // Must be a Type
                             typeof(QCAlgorithm).IsAssignableFrom(type))   // Must inherit from QCAlgorithm
                         {
                             Logging.Log.Trace("AlgorithmPythonWrapper(): Creating IAlgorithm instance.");

AlgorithmFactory/QuantConnect.AlgorithmFactory.csproj

@@ -29,7 +29,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

Brokerages/Binance/BinanceBrokerage.cs

@@ -320,7 +320,7 @@ namespace QuantConnect.Brokerages.Binance
             var apiKey = job.BrokerageData["binance-api-key"];
             var apiSecret = job.BrokerageData["binance-api-secret"];
             var aggregator = Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(
-                Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"));
+                Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false);
 
             Initialize(
                 wssUrl: webSocketBaseUrl,

Brokerages/Binance/BinanceBrokerageFactory.cs

@@ -88,7 +88,7 @@ namespace QuantConnect.Brokerages.Binance
                 job.BrokerageData["binance-api-url"],
                 job.BrokerageData["binance-websocket-url"],
                 algorithm,
-                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager")),
+                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false),
                 job);
             Composer.Instance.AddPart<IDataQueueHandler>(brokerage);
 

Brokerages/Bitfinex/BitfinexBrokerage.cs

@@ -443,7 +443,7 @@ namespace QuantConnect.Brokerages.Bitfinex
             var apiKey = job.BrokerageData["bitfinex-api-key"];
             var apiSecret = job.BrokerageData["bitfinex-api-secret"];
             var aggregator = Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(
-                Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"));
+                Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false);
 
             Initialize(
                 wssUrl: WebSocketUrl,

Brokerages/Bitfinex/BitfinexBrokerageFactory.cs

@@ -84,7 +84,7 @@ namespace QuantConnect.Brokerages.Bitfinex
                 job.BrokerageData["bitfinex-api-secret"],
                 algorithm,
                 priceProvider,
-                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager")),
+                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false),
                 job);
             Composer.Instance.AddPart<IDataQueueHandler>(brokerage);
 

Brokerages/GDAX/GDAXBrokerageFactory.cs

@@ -91,7 +91,7 @@ namespace QuantConnect.Brokerages.GDAX
             var restClient = new RestClient(restApi);
             var webSocketClient = new WebSocketClientWrapper();
             var priceProvider = new ApiPriceProvider(job.UserId, job.UserToken);
-            var aggregator = Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"));
+            var aggregator = Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false);
 
             IBrokerage brokerage;
             if (job.DataQueueHandler.Contains("GDAXDataQueueHandler"))

Brokerages/GDAX/GDAXDataQueueHandler.cs

@@ -97,7 +97,7 @@ namespace QuantConnect.Brokerages.GDAX
             var apiSecret = job.BrokerageData["gdax-api-secret"];
             var priceProvider = new ApiPriceProvider(job.UserId, job.UserToken);
             var aggregator = Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(
-                Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"));
+                Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false);
 
             Initialize(
                 wssUrl: wssUrl,

Brokerages/InteractiveBrokers/InteractiveBrokersBrokerage.cs

@@ -55,7 +55,7 @@ namespace QuantConnect.Brokerages.InteractiveBrokers
         /// <summary>
         /// The default gateway version to use
         /// </summary>
-        public static string DefaultVersion { get; } = "985";
+        public static string DefaultVersion { get; } = "1012";
 
         private IBAutomater.IBAutomater _ibAutomater;
 
@@ -2614,7 +2614,7 @@ namespace QuantConnect.Brokerages.InteractiveBrokers
             Initialize(null,
                 null,
                 null,
-                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager")),
+                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false),
                 Composer.Instance.GetExportedValueByTypeName<IMapFileProvider>(Config.Get("map-file-provider", "QuantConnect.Data.Auxiliary.LocalDiskMapFileProvider")),
                 account,
                 host,

Brokerages/InteractiveBrokers/InteractiveBrokersBrokerageFactory.cs

@@ -102,7 +102,7 @@ namespace QuantConnect.Brokerages.InteractiveBrokers
                 algorithm,
                 algorithm.Transactions,
                 algorithm.Portfolio,
-                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager")),
+                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false),
                 Composer.Instance.GetExportedValueByTypeName<IMapFileProvider>(Config.Get("map-file-provider", "QuantConnect.Data.Auxiliary.LocalDiskMapFileProvider")),
                 account,
                 host,

Brokerages/Oanda/OandaBrokerage.cs

@@ -261,7 +261,7 @@ namespace QuantConnect.Brokerages.Oanda
             Initialize(
                 null,
                 null,
-                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager")),
+                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false),
                 environment,
                 accessToken,
                 accountId,

Brokerages/Oanda/OandaBrokerageFactory.cs

@@ -96,7 +96,7 @@ namespace QuantConnect.Brokerages.Oanda
             var brokerage = new OandaBrokerage(
                 algorithm.Transactions, 
                 algorithm.Portfolio,
-                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager")),
+                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false),
                 environment, 
                 accessToken, 
                 accountId, 

Brokerages/QuantConnect.Brokerages.csproj

@@ -37,7 +37,6 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
-    <PackageReference Include="NodaTime" Version="3.0.5" />
     <PackageReference Include="QuantConnect.IBAutomater" Version="2.0.64" />
     <PackageReference Include="RestSharp" Version="106.12.0" />
   </ItemGroup>

Brokerages/Tradier/TradierBrokerage.DataQueueHandler.cs

@@ -52,7 +52,7 @@ namespace QuantConnect.Brokerages.Tradier
             var useSandbox = bool.Parse(job.BrokerageData["tradier-use-sandbox"]);
             var accountId = job.BrokerageData["tradier-account-id"];
             var accessToken = job.BrokerageData["tradier-access-token"];
-            var aggregator = Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"));
+            var aggregator = Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false);
 
             Initialize(
                 wssUrl: WebSocketUrl,

Brokerages/Tradier/TradierBrokerage.cs

@@ -468,7 +468,8 @@ namespace QuantConnect.Brokerages.Tradier
             request.AddParameter("symbols", csvSymbols, ParameterType.QueryString);
 
             var dataContainer = Execute<TradierQuoteContainer>(request, TradierApiRequestType.Data, "quotes");
-            return dataContainer.Quotes;
+            // can return null quotes and not really be failing for cases where the provided symbols do not match
+            return dataContainer.Quotes ?? new List<TradierQuote>();
         }
 
         /// <summary>

Brokerages/Tradier/TradierBrokerageFactory.cs

@@ -101,7 +101,7 @@ namespace QuantConnect.Brokerages.Tradier
                 algorithm,
                 algorithm.Transactions,
                 algorithm.Portfolio,
-                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager")),
+                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false),
                 useSandbox,
                 accountId,
                 accessToken);

Brokerages/Zerodha/Constants.cs

@@ -1,120 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-
-namespace QuantConnect.Brokerages.Zerodha
-{
-#pragma warning disable 1591
-    /// <summary>
-    /// Types of product supported by Kite
-    /// </summary>
-    public enum KiteProductType
-    {
-        MIS,
-        CNC,
-        NRML
-    }
-
-    /// <summary>
-    /// Types of order supported by Kite
-    /// </summary>
-    public enum KiteOrderType
-    {
-        MARKET,
-        LIMIT,
-        SLM,
-        SL
-    }
-
-    public class Constants
-    {
-
-        // Products
-        public const string PRODUCT_MIS = "MIS";
-        public const string PRODUCT_CNC = "CNC";
-        public const string PRODUCT_NRML = "NRML";
-
-        // Order types
-        public const string ORDER_TYPE_MARKET = "MARKET";
-        public const string ORDER_TYPE_LIMIT = "LIMIT";
-        public const string ORDER_TYPE_SLM = "SL-M";
-        public const string ORDER_TYPE_SL = "SL";
-
-        // Order status
-        public const string ORDER_STATUS_COMPLETE = "COMPLETE";
-        public const string ORDER_STATUS_CANCELLED = "CANCELLED";
-        public const string ORDER_STATUS_REJECTED = "REJECTED";
-
-        // Varities
-        public const string VARIETY_REGULAR = "regular";
-        public const string VARIETY_BO = "bo";
-        public const string VARIETY_CO = "co";
-        public const string VARIETY_AMO = "amo";
-
-        // Transaction type
-        public const string TRANSACTION_TYPE_BUY = "BUY";
-        public const string TRANSACTION_TYPE_SELL = "SELL";
-
-        // Validity
-        public const string VALIDITY_DAY = "DAY";
-        public const string VALIDITY_IOC = "IOC";
-
-        // Exchanges
-        public const string EXCHANGE_NSE = "NSE";
-        public const string EXCHANGE_BSE = "BSE";
-        public const string EXCHANGE_NFO = "NFO";
-        public const string EXCHANGE_CDS = "CDS";
-        public const string EXCHANGE_BFO = "BFO";
-        public const string EXCHANGE_MCX = "MCX";
-
-        // Margins segments
-        public const string MARGIN_EQUITY = "equity";
-        public const string MARGIN_COMMODITY = "commodity";
-
-        // Ticker modes
-        public const string MODE_FULL = "full";
-        public const string MODE_QUOTE = "quote";
-        public const string MODE_LTP = "ltp";
-
-        // Positions
-        public const string POSITION_DAY = "day";
-        public const string POSITION_OVERNIGHT = "overnight";
-
-        // Historical intervals
-        public const string INTERVAL_MINUTE = "minute";
-        public const string INTERVAL_3MINUTE = "3minute";
-        public const string INTERVAL_5MINUTE = "5minute";
-        public const string INTERVAL_10MINUTE = "10minute";
-        public const string INTERVAL_15MINUTE = "15minute";
-        public const string INTERVAL_30MINUTE = "30minute";
-        public const string INTERVAL_60MINUTE = "60minute";
-        public const string INTERVAL_DAY = "day";
-
-        // GTT status
-        public const string GTT_ACTIVE = "active";
-        public const string GTT_TRIGGERED = "triggered";
-        public const string GTT_DISABLED = "disabled";
-        public const string GTT_EXPIRED = "expired";
-        public const string GTT_CANCELLED = "cancelled";
-        public const string GTT_REJECTED = "rejected";
-        public const string GTT_DELETED = "deleted";
-
-
-        // GTT trigger type
-        public const string GTT_TRIGGER_OCO = "two-leg";
-        public const string GTT_TRIGGER_SINGLE = "single";
-    }
-#pragma warning restore 1591
-}

Brokerages/Zerodha/ExceptionMessages.cs

@@ -1,60 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-
-namespace QuantConnect.Brokerages.Zerodha
-{
-    internal static partial class ExceptionExtensions
-    {
-
-        /// <summary>
-        /// Returns a list of all the exception messages from the top-level
-        /// exception down through all the inner exceptions. Useful for making
-        /// logs and error pages easier to read when dealing with exceptions.
-        /// Usage: Exception.Messages()
-        /// </summary>
-        public static IEnumerable<string> Messages(this Exception ex)
-        {
-            // return an empty sequence if the provided exception is null
-            if (ex == null) { yield break; }
-            // first return THIS exception's message at the beginning of the list
-            yield return ex.Message;
-            // then get all the lower-level exception messages recursively (if any)
-            IEnumerable<Exception> innerExceptions = Enumerable.Empty<Exception>();
-
-            if (ex is AggregateException && (ex as AggregateException).InnerExceptions.Any())
-            {
-                innerExceptions = (ex as AggregateException).InnerExceptions;
-            }
-            else if (ex.InnerException != null)
-            {
-                innerExceptions = new Exception[] { ex.InnerException };
-            }
-
-            foreach (var innerEx in innerExceptions)
-            {
-                foreach (string msg in innerEx.Messages())
-                {
-                    yield return msg;
-                }
-            }
-        }
-
-    }
-}

Brokerages/Zerodha/Exceptions.cs

@@ -1,87 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using System.Net;
-
-namespace QuantConnect.Brokerages.Zerodha
-{
-#pragma warning disable 1591
-    /// <summary>
-    /// KiteAPI Exceptions
-    /// </summary>
-    public class KiteException : Exception
-    {
-        HttpStatusCode status;
-        public KiteException(string message, HttpStatusCode httpStatus, Exception innerException = null) : base(message, innerException) { status = httpStatus; }
-    }
-
-    /// <summary>
-    /// General Exceptions
-    /// </summary>
-    public class GeneralException : KiteException
-    {
-        public GeneralException(string message, HttpStatusCode httpStatus = HttpStatusCode.InternalServerError, Exception innerException = null) : base(message, httpStatus, innerException) { }
-    }
-
-    /// <summary>
-    /// Token Exceptions
-    /// </summary>
-    public class TokenException : KiteException
-    {
-        public TokenException(string message, HttpStatusCode httpStatus = HttpStatusCode.Forbidden, Exception innerException = null) : base(message, httpStatus, innerException) { }
-    }
-
-    /// <summary>
-    /// Permission Exceptions
-    /// </summary>
-    public class PermissionException : KiteException
-    {
-        public PermissionException(string message, HttpStatusCode httpStatus = HttpStatusCode.Forbidden, Exception innerException = null) : base(message, httpStatus, innerException) { }
-    }
-
-    /// <summary>
-    /// Order Exceptions
-    /// </summary>
-    public class OrderException : KiteException
-    {
-        public OrderException(string message, HttpStatusCode httpStatus = HttpStatusCode.BadRequest, Exception innerException = null) : base(message, httpStatus, innerException) { }
-    }
-
-    /// <summary>
-    /// InputExceptions
-    /// </summary>
-    public class InputException : KiteException
-    {
-        public InputException(string message, HttpStatusCode httpStatus = HttpStatusCode.BadRequest, Exception innerException = null) : base(message, httpStatus, innerException) { }
-    }
-
-    /// <summary>
-    /// DataExceptions 
-    /// </summary>
-    public class DataException : KiteException
-    {
-        public DataException(string message, HttpStatusCode httpStatus = HttpStatusCode.BadGateway, Exception innerException = null) : base(message, httpStatus, innerException) { }
-    }
-
-    /// <summary>
-    /// Network Exceptions
-    /// </summary>
-    public class NetworkException : KiteException
-    {
-        public NetworkException(string message, HttpStatusCode httpStatus = HttpStatusCode.ServiceUnavailable, Exception innerException = null) : base(message, httpStatus, innerException) { }
-    }
-#pragma warning restore 1591
-}

Brokerages/Zerodha/Utils.cs

@@ -1,189 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-
-using System.IO;
-using System.Web;
-using System.Text.RegularExpressions;
-using Newtonsoft.Json;
-using System.Globalization;
-using Newtonsoft.Json.Linq;
-
-namespace QuantConnect.Brokerages.Zerodha
-{
-    /// <summary>
-    /// Zerodha utility class
-    /// </summary>
-    public class Utils
-    {
-        /// <summary>
-        /// Convert string to Date object
-        /// </summary>
-        /// <param name="dateString">Date string.</param>
-        /// <returns>Date object/</returns>
-        public static DateTime? StringToDate(string dateString)
-        {
-            if (dateString == null)
-            {
-                return null;
-            }
-
-            try
-            {
-                return DateTime.ParseExact(dateString, dateString.Length == 10 ? "yyyy-MM-dd" : "yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture);
-            }
-            catch (Exception)
-            {
-                return null;
-            }
-        }
-
-        /// <summary>
-        /// Serialize C# object to JSON string.
-        /// </summary>
-        /// <param name="obj">C# object to serialize.</param>
-        /// <returns>JSON string/</returns>
-        public static string JsonSerialize(object obj)
-        {
-            string json = JsonConvert.SerializeObject(obj);
-            MatchCollection mc = Regex.Matches(json, @"\\/Date\((\d*?)\)\\/");
-            foreach (Match m in mc)
-            {
-                var unix = Convert.ToInt64(m.Groups[1].Value,CultureInfo.InvariantCulture) / 1000;
-                json = json.Replace(m.Groups[0].Value, UnixToDateTime(unix).ToStringInvariant());
-            }
-            return json;
-        }
-
-        /// <summary>
-        /// Deserialize Json string to nested string dictionary.
-        /// </summary>
-        /// <param name="Json">Json string to deserialize.</param>
-        /// <returns>Json in the form of nested string dictionary.</returns>
-        public static JObject JsonDeserialize(string Json)
-        {
-            JObject jObject = JObject.Parse(Json);
-            return jObject;
-        }
-
-        /// <summary>
-        /// Recursively traverses an object and converts double fields to decimal.
-        /// This is used in Json deserialization. JavaScriptSerializer converts floats
-        /// in exponential notation to double and everthing else to double. This function
-        /// makes everything decimal. Currently supports only Dictionary and Array as input.
-        /// </summary>
-        /// <param name="obj">Input object.</param>
-        /// <returns>Object with decimals instead of doubles</returns>
-        public static dynamic DoubleToDecimal(dynamic obj)
-        {
-            if (obj is double)
-            {
-                obj = Convert.ToDecimal(obj);
-            }
-            else if (obj is IDictionary)
-            {
-                var keys = new List<string>(obj.Keys);
-                for (int i = 0; i < keys.Count; i++)
-                {
-                    obj[keys[i]] = DoubleToDecimal(obj[keys[i]]);
-                }
-            }
-            else if (obj is ICollection)
-            {
-                obj = new ArrayList(obj);
-                for (int i = 0; i < obj.Count; i++)
-                {
-                    obj[i] = DoubleToDecimal(obj[i]);
-                }
-            }
-            return obj;
-        }
-
-        /// <summary>
-        /// Wraps a string inside a stream
-        /// </summary>
-        /// <param name="value">string data</param>
-        /// <returns>Stream that reads input string</returns>
-        public static MemoryStream StreamFromString(string value)
-        {
-            return new MemoryStream(Encoding.UTF8.GetBytes(value ?? ""));
-        }
-
-        /// <summary>
-        /// Helper function to add parameter to the request only if it is not null or empty
-        /// </summary>
-        /// <param name="Params">Dictionary to add the key-value pair</param>
-        /// <param name="Key">Key of the parameter</param>
-        /// <param name="Value">Value of the parameter</param>
-        public static void AddIfNotNull(Dictionary<string, dynamic> Params, string Key, string Value)
-        {
-            if (!String.IsNullOrEmpty(Value))
-                Params.Add(Key, Value);
-        }
-
-
-        /// <summary>
-        /// Creates key=value with url encoded value
-        /// </summary>
-        /// <param name="Key">Key</param>
-        /// <param name="Value">Value</param>
-        /// <returns>Combined string</returns>
-        public static string BuildParam(string Key, dynamic Value)
-        {
-            if (Value is string)
-            {
-                return HttpUtility.UrlEncode(Key) + "=" + HttpUtility.UrlEncode((string)Value);
-            }
-            else
-            {
-                string[] values = (string[])Value;
-                return String.Join("&", values.Select(x => HttpUtility.UrlEncode(Key) + "=" + HttpUtility.UrlEncode(x)));
-            }
-        }
-
-        /// <summary>
-        /// Convert Unix TimeStamp to DateTime
-        /// </summary>
-        /// <param name="unixTimeStamp">Timestamp to convert</param>
-        /// <returns><see cref="DateTime"/> object representing the timestamp</returns>
-        public static DateTime UnixToDateTime(long unixTimeStamp)
-        {
-            // Unix timestamp is seconds past epoch
-            DateTime dateTime = new DateTime(1970, 1, 1, 5, 30, 0, 0, DateTimeKind.Unspecified);
-            dateTime = dateTime.AddSeconds(unixTimeStamp);
-            return dateTime;
-        }
-
-        /// <summary>
-        /// Convert ArrayList to list of <see cref="decimal"/>
-        /// </summary>
-        /// <param name="arrayList"><see cref="ArrayList"/> to convert</param>
-        /// <returns>List of <see cref="decimal"/>s</returns>
-        public static List<decimal> ToDecimalList(ArrayList arrayList)
-        {
-            var res = new List<decimal>();
-            foreach(var i in arrayList)
-            {
-                res.Add(Convert.ToDecimal(i,CultureInfo.InvariantCulture));
-            }
-            return res;
-        }
-    }
-}

Brokerages/Zerodha/ZerodhaBrokerage.DataQueueHandler.cs

@@ -1,102 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using QuantConnect.Configuration;
-using QuantConnect.Data;
-using QuantConnect.Packets;
-using QuantConnect.Util;
-using System;
-using System.Collections.Generic;
-
-namespace QuantConnect.Brokerages.Zerodha
-{
-    /// <summary>
-    /// ZerodhaBrokerage: IDataQueueHandler implementation
-    /// </summary>
-    public partial class ZerodhaBrokerage
-    {
-        #region IDataQueueHandler implementation
-
-        /// <summary>
-        /// Sets the job we're subscribing for
-        /// </summary>
-        /// <param name="job">Job we're subscribing for</param>
-        public void SetJob(LiveNodePacket job)
-        {
-            Initialize(
-                job.BrokerageData["zerodha-trading-segment"],
-                job.BrokerageData["zerodha-product-type"],
-                job.BrokerageData["zerodha-api-key"],
-                job.BrokerageData["zerodha-access-token"],
-                null,
-                null,
-                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"))
-            );
-
-            if (!IsConnected)
-            {
-                Connect();
-            }
-        }
-
-        /// <summary>
-        /// Subscribe to the specified configuration
-        /// </summary>
-        /// <param name="dataConfig">defines the parameters to subscribe to a data feed</param>
-        /// <param name="newDataAvailableHandler">handler to be fired on new data available</param>
-        /// <returns>The new enumerator for this subscription request</returns>
-        public IEnumerator<BaseData> Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler)
-        {
-            var symbol = dataConfig.Symbol;
-            if (!CanSubscribe(symbol))
-            {
-                return null;
-            }
-
-            var enumerator = _aggregator.Add(dataConfig, newDataAvailableHandler);
-            SubscriptionManager.Subscribe(dataConfig);
-
-            return enumerator;
-        }
-
-        /// <summary>
-        /// UnSubscribe to the specified configuration
-        /// </summary>
-        /// <param name="dataConfig">defines the parameters to subscribe to a data feed</param>
-        public void Unsubscribe(SubscriptionDataConfig dataConfig)
-        {
-            SubscriptionManager.Unsubscribe(dataConfig);
-            _aggregator.Remove(dataConfig);
-        }
-
-        /// <summary>
-        /// Returns true if this data provide can handle the specified symbol
-        /// </summary>
-        /// <param name="symbol">The symbol to be handled</param>
-        /// <returns>True if this data provider can get data for the symbol, false otherwise</returns>
-        private static bool CanSubscribe(Symbol symbol)
-        {
-            var market = symbol.ID.Market;
-            var securityType = symbol.ID.SecurityType;
-            if (symbol.Value.IndexOfInvariant("universe", true) != -1) return false;
-            // Include future options as a special case with no matching market, otherwise
-            // our subscriptions are removed without any sort of notice.
-            return
-                (securityType == SecurityType.Equity) && (market == Market.India);
-        }
-
-        #endregion IDataQueueHandler implementation
-    }
-}

Brokerages/Zerodha/ZerodhaBrokerageFactory.cs

@@ -1,92 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using System.Collections.Generic;
-using QuantConnect.Configuration;
-using QuantConnect.Data;
-using QuantConnect.Interfaces;
-using QuantConnect.Securities;
-using QuantConnect.Util;
-
-namespace QuantConnect.Brokerages.Zerodha
-{
-    /// <summary>
-    /// Factory method to create Zerodha Websockets brokerage
-    /// </summary>
-    public class ZerodhaBrokerageFactory : BrokerageFactory
-    {
-        /// <summary>
-        /// Factory constructor
-        /// </summary>
-        public ZerodhaBrokerageFactory() : base(typeof(ZerodhaBrokerage))
-        {
-        }
-
-        /// <summary>
-        /// Not required
-        /// </summary>
-        public override void Dispose()
-        {
-        }
-
-        /// <summary>
-        /// provides brokerage connection data
-        /// </summary>
-        public override Dictionary<string, string> BrokerageData => new Dictionary<string, string>
-        {
-            { "zerodha-api-key", Config.Get("zerodha-api-key")},
-            { "zerodha-access-token", Config.Get("zerodha-access-token")},
-            { "zerodha-trading-segment", Config.Get("zerodha-trading-segment")},
-            { "zerodha-product-type", Config.Get("zerodha-product-type")},
-        };
-
-        /// <summary>
-        /// The brokerage model
-        /// </summary>
-        /// <param name="orderProvider">The order provider</param>
-        public override IBrokerageModel GetBrokerageModel(IOrderProvider orderProvider) => new ZerodhaBrokerageModel();
-
-        /// <summary>
-        /// Create the Brokerage instance
-        /// </summary>
-        /// <param name="job"></param>
-        /// <param name="algorithm"></param>
-        /// <returns></returns>
-        public override IBrokerage CreateBrokerage(Packets.LiveNodePacket job, IAlgorithm algorithm)
-        {
-            var required = new[] { "zerodha-api-key", "zerodha-access-token", "zerodha-trading-segment"};
-
-            foreach (var item in required)
-            {
-                if (string.IsNullOrEmpty(job.BrokerageData[item]))
-                    throw new Exception($"ZerodhaBrokerageFactory.CreateBrokerage: Missing {item} in config.json");
-            }
-
-            var brokerage = new ZerodhaBrokerage(
-                job.BrokerageData["zerodha-trading-segment"],
-                job.BrokerageData["zerodha-product-type"],
-                job.BrokerageData["zerodha-api-key"],
-                job.BrokerageData["zerodha-access-token"],
-                algorithm,
-                algorithm.Portfolio,
-                Composer.Instance.GetExportedValueByTypeName<IDataAggregator>(Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"))
-               );
-            //Add the brokerage to the composer to ensure its accessible to the live data feed.
-            Composer.Instance.AddPart<IDataQueueHandler>(brokerage);
-            return brokerage;
-        }
-    }
-}

Brokerages/Zerodha/ZerodhaSymbolMapper.cs

@@ -1,393 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using System.Collections.Generic;
-using System.Globalization;
-using System.Linq;
-using QuantConnect.Brokerages.Zerodha.Messages;
-using QuantConnect.Util;
-
-namespace QuantConnect.Brokerages.Zerodha
-{
-    /// <summary>
-    /// Provides the mapping between Lean symbols and Zerodha symbols.
-    /// </summary>
-    public class ZerodhaSymbolMapper : ISymbolMapper
-    {
-
-        /// <summary>
-        /// Symbols that are Tradable
-        /// </summary>
-        public List<Symbol> KnownSymbols
-        {
-            get
-            {
-                return KnownSymbolsList;
-            }
-        }
-
-        /// <summary>
-        /// Custom class to store information about symbols
-        /// </summary>
-        private class SymbolData 
-        { 
-            /// <summary>
-            /// Stores exchange name for the tradingSymbol
-            /// </summary>
-            public string Exchange { get; set;}
-
-            /// <summary>
-            /// Stores instrumentToken name for the tradingSymbol
-            /// </summary>
-            public uint InstrumentToken {get; set;}
-
-            /// <summary>
-            /// Initalize values to the class attributes
-            /// </summary>
-            public SymbolData(uint token, string exchangeName)
-            {
-                Exchange = exchangeName;
-                InstrumentToken = token;
-            }
-        }
-
-        /// <summary>
-        /// The list of known Zerodha symbols.
-        /// </summary>
-        private List<Symbol> KnownSymbolsList = new List<Symbol>();
-
-        /// <summary>
-        /// Mapping between brokerageSymbol and a list of all available SymbolData objects for the brokerageSymbol.
-        /// </summary>
-        private Dictionary<string, List<SymbolData>> ZerodhaInstrumentsList = new Dictionary<string, List<SymbolData>>();
-
-        /// <summary>
-        /// Mapping between instrumentToken and it's market segment ( E.g: 408065-> nse)
-        /// </summary>
-        private Dictionary<uint,string> ZerodhaInstrumentsExchangeMapping = new Dictionary<uint,string>();
-
-        /// <summary>
-        /// Constructs default instance of the Zerodha Sybol Mapper
-        /// </summary>
-        public ZerodhaSymbolMapper(Kite kite, string exchange = "")
-        {
-            KnownSymbolsList = GetTradableInstrumentsList(kite, exchange);
-        }
-        /// <summary>
-        /// Get list of tradable symbol
-        /// </summary>
-        /// <param name="kite">Kite</param>
-        /// <param name="exchange">Exchange</param>
-        /// <returns></returns>
-        private List<Symbol> GetTradableInstrumentsList(Kite kite, string exchange = "")
-        {
-
-            var tradableInstruments = kite.GetInstruments(exchange);
-            var symbols = new List<Symbol>();
-            var zerodhaInstrumentsMapping = new Dictionary<string, List<SymbolData>>();
-            var zerodhaTokenExchangeDict = new Dictionary<uint,string>();
-
-            foreach (var tp in tradableInstruments)
-            {
-                var securityType = SecurityType.Equity;
-                var market = Market.India;
-                zerodhaTokenExchangeDict[tp.InstrumentToken] = tp.Exchange.ToLowerInvariant();
-                OptionRight optionRight = 0;
-
-                switch (tp.InstrumentType)
-                {
-                    //Equities
-                    case "EQ":
-                        securityType = SecurityType.Equity;
-                        break;
-                    //Call Options
-                    case "CE":
-                        securityType = SecurityType.Option;
-                        optionRight = OptionRight.Call;
-                        break;
-                    //Put Options
-                    case "PE":
-                        securityType = SecurityType.Option;
-                        optionRight = OptionRight.Put;
-                        break;
-                    //Stock Futures
-                    case "FUT":
-                        securityType = SecurityType.Future;
-                        break;
-                    default:
-                        securityType = SecurityType.Base;
-                        break;
-                }
-                
-                if (securityType == SecurityType.Option)
-                {
-                    var strikePrice = tp.Strike;
-                    var expiryDate = tp.Expiry;
-                    //TODO: Handle parsing of BCDOPT strike price
-                    if(tp.Segment!= "BCD-OPT")
-                    {
-                        var symbol = GetLeanSymbol(tp.Name.Trim().Replace(" ", ""), securityType, market, (DateTime)expiryDate, GetStrikePrice(tp), optionRight);
-                        symbols.Add(symbol);
-                        var cleanSymbol = tp.TradingSymbol.Trim().Replace(" ", "");
-                        if (!zerodhaInstrumentsMapping.ContainsKey(cleanSymbol))
-                        {
-                            zerodhaInstrumentsMapping[cleanSymbol] = new List<SymbolData>();
-                        }
-                        zerodhaInstrumentsMapping[cleanSymbol].Add(new SymbolData(tp.InstrumentToken,market));
-                    }                    
-                }
-                if (securityType == SecurityType.Future)
-                {
-                    var expiryDate = tp.Expiry;
-                    var cleanSymbol = tp.TradingSymbol.Trim().Replace(" ", "");
-                    var symbol = GetLeanSymbol(cleanSymbol, securityType, market, (DateTime)expiryDate);
-                    symbols.Add(symbol);
-                    if (!zerodhaInstrumentsMapping.ContainsKey(cleanSymbol))
-                    {
-                        zerodhaInstrumentsMapping[cleanSymbol] = new List<SymbolData>();
-                    }
-                    zerodhaInstrumentsMapping[cleanSymbol].Add(new SymbolData(tp.InstrumentToken,market));
-                }
-                if (securityType == SecurityType.Equity)
-                {
-                    var cleanSymbol = tp.TradingSymbol.Trim().Replace(" ", "");
-                    var symbol = GetLeanSymbol(cleanSymbol, securityType, market);
-                    symbols.Add(symbol);
-                    if (!zerodhaInstrumentsMapping.ContainsKey(cleanSymbol))
-                    {
-                        zerodhaInstrumentsMapping[cleanSymbol] = new List<SymbolData>();
-                    }
-                    zerodhaInstrumentsMapping[cleanSymbol].Add(new SymbolData(tp.InstrumentToken,market));
-                }
-            }
-            ZerodhaInstrumentsList = zerodhaInstrumentsMapping;
-            ZerodhaInstrumentsExchangeMapping = zerodhaTokenExchangeDict;
-            return symbols;
-        }
-
-        private decimal GetStrikePrice(CsvInstrument scrip)
-        {
-            var strikePrice = scrip.TradingSymbol.Trim().Replace(" ", "").Replace(scrip.Name, "");
-            var strikePriceTemp = strikePrice.Substring(5, strikePrice.Length - 5);
-            var strikePriceResult = strikePriceTemp.Substring(0, strikePriceTemp.Length - 2);
-
-            return Convert.ToDecimal(strikePriceResult, CultureInfo.InvariantCulture);
-        }
-
-        /// <summary>
-        /// Converts a Lean symbol instance to an Zerodha symbol
-        /// </summary>
-        /// <param name="symbol">A Lean symbol instance</param>
-        /// <returns>The Zerodha symbol</returns>
-        public string GetBrokerageSymbol(Symbol symbol)
-        {
-            if (symbol == null || string.IsNullOrWhiteSpace(symbol.Value))
-            {
-                throw new ArgumentException("Invalid symbol: " + (symbol == null ? "null" : symbol.ToString()));
-            }
-
-            if (symbol.ID.SecurityType != SecurityType.Equity && symbol.ID.SecurityType != SecurityType.Future && symbol.ID.SecurityType != SecurityType.Option)
-            {
-                throw new ArgumentException("Invalid security type: " + symbol.ID.SecurityType);
-            }
-
-            var brokerageSymbol = ConvertLeanSymbolToZerodhaSymbol(symbol.Value);
-
-            return brokerageSymbol;
-        }
-
-
-        /// <summary>
-        /// Converts an Zerodha symbol to a Lean symbol instance
-        /// </summary>
-        /// <param name="brokerageSymbol">The Zerodha symbol</param>
-        /// <param name="securityType">The security type</param>
-        /// <param name="market">The market</param>
-        /// <param name="expirationDate">Expiration date of the security(if applicable)</param>
-        /// <param name="strike">The strike of the security (if applicable)</param>
-        /// <param name="optionRight">The option right of the security (if applicable)</param>
-        /// <returns>A new Lean Symbol instance</returns>
-        public Symbol GetLeanSymbol(string brokerageSymbol, SecurityType securityType, string market, DateTime expirationDate = default(DateTime), decimal strike = 0, OptionRight optionRight = OptionRight.Call)
-        {
-            if (string.IsNullOrWhiteSpace(brokerageSymbol))
-            {
-                throw new ArgumentException($"Invalid Zerodha symbol: {brokerageSymbol}");
-            }
-
-            if (securityType == SecurityType.Forex || securityType == SecurityType.Cfd || securityType == SecurityType.Commodity || securityType == SecurityType.Crypto)
-            {
-                throw new ArgumentException($"Invalid security type: {securityType}");
-            }
-
-            if (!Market.Encode(market).HasValue)
-            {
-                throw new ArgumentException($"Invalid market: {market}");
-            }
-            var cleanSymbol = brokerageSymbol.Replace(" ", "").Trim();
-
-            switch (securityType)
-            {
-                case SecurityType.Option:
-                    OptionStyle optionStyle = OptionStyle.European;
-                    return Symbol.CreateOption(cleanSymbol, market, optionStyle, optionRight, strike, expirationDate);
-                case SecurityType.Future:
-                    return Symbol.CreateFuture(cleanSymbol, market, expirationDate);
-                default:
-                    return Symbol.Create(cleanSymbol, securityType, market);
-            }
-
-        }
-
-
-        /// <summary>
-        /// Converts an Zerodha symbol to a Lean symbol instance
-        /// </summary>
-        /// <param name="brokerageSymbol">The Zerodha symbol</param>
-        /// <returns>A new Lean Symbol instance</returns>
-        public Symbol GetLeanSymbol(string brokerageSymbol)
-        {
-            if (string.IsNullOrWhiteSpace(brokerageSymbol))
-            {
-                throw new ArgumentException($"Invalid Zerodha symbol: {brokerageSymbol}");
-            }
-
-            var cleanSymbol = brokerageSymbol.Replace(" ", "").Trim();
-
-            if (IsKnownBrokerageSymbol(cleanSymbol))
-            {
-                throw new ArgumentException($"Symbol not present : {cleanSymbol}");
-            }
-
-            var symbol = KnownSymbols.FirstOrDefault(s => s.Value == cleanSymbol);
-            var exchange = GetZerodhaDefaultExchange(cleanSymbol);
-            return GetLeanSymbol(cleanSymbol, symbol.SecurityType, exchange);
-        }
-
-        /// <summary>
-        /// Fetches the trading segment inside India Market, E.g: NSE, BSE for the given Instrument Token
-        /// </summary>
-        /// <param name="Token">The Zerodha Instrument Token</param>
-        /// <returns>An exchange value for the given token</returns>
-        public string GetZerodhaExchangeFromToken(uint Token)
-        {   
-            string exchange = string.Empty;
-            if (ZerodhaInstrumentsExchangeMapping.ContainsKey(Token))
-            {
-                ZerodhaInstrumentsExchangeMapping.TryGetValue(Token, out exchange);
-            }
-            return exchange;
-        }
-        
-
-        /// <summary>
-        /// Fetches the first available Exchage value for the given symbol from list of possible exchanges
-        /// </summary>
-        /// <param name="brokerageSymbol">The Zerodha symbol</param>
-        /// <returns>A default exchange value for the given ticker</returns>
-        private string GetZerodhaDefaultExchange(string brokerageSymbol)
-        {   
-            if (string.IsNullOrWhiteSpace(brokerageSymbol))
-            {
-                throw new ArgumentException($"Invalid Zerodha symbol: {brokerageSymbol}");
-            }
-
-            var cleanSymbol = brokerageSymbol.Replace(" ", "").Trim();
-
-            List<SymbolData> tempSymbolDataList;
-            if (ZerodhaInstrumentsList.TryGetValue(cleanSymbol, out tempSymbolDataList))
-            {   
-                return tempSymbolDataList[0].Exchange;
-            }
-            return string.Empty;
-        }
-
-        /// <summary>
-        /// Converts Lean symbol to a List of Zerodha Instrument Tokens available from various exchange
-        /// </summary>
-        /// <param name="brokerageSymbol">The Zerodha symbol</param>
-        /// <returns>A list of Zerodha Instrument Tokens</returns>
-        public List<uint> GetZerodhaInstrumentTokenList(string brokerageSymbol)
-        {
-            if (string.IsNullOrWhiteSpace(brokerageSymbol))
-            {
-                throw new ArgumentException($"Invalid Zerodha symbol: {brokerageSymbol}");
-            }
-
-            var cleanSymbol = brokerageSymbol.Replace(" ", "").Trim();
-
-            List<uint> tokenList = new List<uint>();
-            List<SymbolData> tempSymbolDataList;
-            if (ZerodhaInstrumentsList.TryGetValue(cleanSymbol, out tempSymbolDataList))
-            {
-                foreach (var sd in tempSymbolDataList)
-                {
-                    tokenList.Add(sd.InstrumentToken);
-                }
-            }
-            return tokenList;
-        }
-
-        /// <summary>
-        /// Checks if the symbol is supported by Zerodha
-        /// </summary>
-        /// <param name="brokerageSymbol">The Zerodha symbol</param>
-        /// <returns>True if Zerodha supports the symbol</returns>
-        private bool IsKnownBrokerageSymbol(string brokerageSymbol)
-        {
-            if (string.IsNullOrWhiteSpace(brokerageSymbol))
-            {
-                return false;
-            }
-
-            return KnownSymbolsList.Where(x => x.Value.Contains(brokerageSymbol)).IsNullOrEmpty();
-        }
-
-        /// <summary>
-        /// Converts an Zerodha symbol to a Lean symbol string
-        /// </summary>
-        public Symbol ConvertZerodhaSymbolToLeanSymbol(uint ZerodhaSymbol)
-        {
-            var _symbol = string.Empty;
-            foreach (var item in ZerodhaInstrumentsList)
-            {
-                foreach( var sd in item.Value) 
-                { 
-                    if (sd.InstrumentToken == ZerodhaSymbol) 
-                    {
-                        _symbol = item.Key;
-                        break;
-                    }
-                }
-            }
-            // return as it is due to Zerodha has similar Symbol format
-            return KnownSymbolsList.Where(s => s.Value == _symbol).FirstOrDefault();
-        }
-
-        /// <summary>
-        /// Converts a Lean symbol string to an Zerodha symbol
-        /// </summary>
-        private static string ConvertLeanSymbolToZerodhaSymbol(string leanSymbol)
-        {
-            if (string.IsNullOrWhiteSpace(leanSymbol))
-            {
-                throw new ArgumentException($"Invalid Lean symbol: {leanSymbol}");
-            }
-
-            // return as it is due to Zerodha has similar Symbol format
-            return leanSymbol.ToUpperInvariant();
-        }
-    }
-}

Common/AlgorithmImports.py

@@ -40,6 +40,7 @@ from QuantConnect.Python import *
 from QuantConnect.Storage import *
 from QuantConnect.Research import *
 from QuantConnect.Algorithm import *
+from QuantConnect.Statistics import *
 from QuantConnect.Parameters import *
 from QuantConnect.Benchmarks import *
 from QuantConnect.Brokerages import *
@@ -60,6 +61,7 @@ from QuantConnect.Data.Shortable import *
 from QuantConnect.Orders.Slippage import *
 from QuantConnect.Securities.Forex import *
 from QuantConnect.Data.Fundamental import *
+from QuantConnect.Algorithm.CSharp import *
 from QuantConnect.Securities.Option import *
 from QuantConnect.Securities.Equity import *
 from QuantConnect.Securities.Future import *
@@ -69,6 +71,7 @@ from QuantConnect.Algorithm.Framework import *
 from QuantConnect.Securities.Volatility import *
 from QuantConnect.Securities.Interfaces import *
 from QuantConnect.Data.UniverseSelection import *
+from QuantConnect.Data.Custom.IconicTypes import *
 from QuantConnect.Data.Custom.AlphaStreams import *
 from QuantConnect.Algorithm.Framework.Risk import *
 from QuantConnect.Algorithm.Framework.Alphas import *

Common/Brokerages/BrokerageName.cs

@@ -104,6 +104,11 @@ namespace QuantConnect.Brokerages
         /// <summary>
         /// Transaction and submit/execution rules will use ftx models
         /// </summary>
-        FTX
+        FTX,
+
+        /// <summary>
+        /// Transaction and submit/execution rules will use ftx us models
+        /// </summary>
+        FTXUS
     }
 }

Common/Brokerages/FTXBrokerageModel.cs

@@ -29,6 +29,16 @@ namespace QuantConnect.Brokerages
     {
         private const decimal _defaultLeverage = 3m;
 
+        /// <summary>
+        /// market name
+        /// </summary>
+        protected virtual string MarketName => Market.FTX;
+
+        /// <summary>
+        /// Gets a map of the default markets to be used for each security type
+        /// </summary>
+        public override IReadOnlyDictionary<SecurityType, string> DefaultMarkets { get; } = GetDefaultMarkets(Market.FTX);
+
         /// <summary>
         /// Creates an instance of <see cref="FTXBrokerageModel"/> class
         /// </summary>
@@ -36,13 +46,7 @@ namespace QuantConnect.Brokerages
         public FTXBrokerageModel(AccountType accountType = AccountType.Margin) : base(accountType)
         {
         }
-
-        /// <summary>
-        /// Gets a map of the default markets to be used for each security type
-        /// </summary>
-        public override IReadOnlyDictionary<SecurityType, string> DefaultMarkets { get; } = GetDefaultMarkets();
-
-
+        
         /// <summary>
         /// Gets a new buying power model for the security, returning the default model with the security's configured leverage.
         /// For cash accounts, leverage = 1 is used.
@@ -87,7 +91,7 @@ namespace QuantConnect.Brokerages
         /// <returns>The benchmark for this brokerage</returns>
         public override IBenchmark GetBenchmark(SecurityManager securities)
         {
-            var symbol = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.FTX);
+            var symbol = Symbol.Create("BTCUSD", SecurityType.Crypto, MarketName);
             return SecurityBenchmark.CreateInstance(securities, symbol);
         }
 
@@ -158,7 +162,7 @@ namespace QuantConnect.Brokerages
             if (security.Type != SecurityType.Crypto)
             {
                 message = new BrokerageMessageEvent(BrokerageMessageType.Warning, "NotSupported",
-                    StringExtensions.Invariant($"The {nameof(FTXBrokerageModel)} does not support {security.Type} security type.")
+                    StringExtensions.Invariant($"The {this.GetType().Name} does not support {security.Type} security type.")
                 );
 
                 return false;
@@ -186,10 +190,10 @@ namespace QuantConnect.Brokerages
             return false;
         }
 
-        private static IReadOnlyDictionary<SecurityType, string> GetDefaultMarkets()
+        protected static IReadOnlyDictionary<SecurityType, string> GetDefaultMarkets(string market)
         {
             var map = DefaultMarketMap.ToDictionary();
-            map[SecurityType.Crypto] = Market.FTX;
+            map[SecurityType.Crypto] = market;
             return map.ToReadOnlyDictionary();
         }
     }

Common/Brokerages/FTXUSBrokerageModel.cs

@@ -0,0 +1,53 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.Collections.Generic;
+using QuantConnect.Orders.Fees;
+using QuantConnect.Securities;
+
+namespace QuantConnect.Brokerages
+{
+    /// <summary>
+    /// FTX.US Brokerage model
+    /// </summary>
+    public class FTXUSBrokerageModel : FTXBrokerageModel
+    {
+        /// <summary>
+        /// Market name
+        /// </summary>
+        protected override string MarketName => Market.FTXUS;
+
+        /// <summary>
+        /// Gets a map of the default markets to be used for each security type
+        /// </summary>
+        public override IReadOnlyDictionary<SecurityType, string> DefaultMarkets { get; } = GetDefaultMarkets(Market.FTXUS);
+
+        /// <summary>
+        /// Creates an instance of <see cref="FTXUSBrokerageModel"/> class
+        /// </summary>
+        /// <param name="accountType">Cash or Margin</param>
+        public FTXUSBrokerageModel(AccountType accountType = AccountType.Margin) : base(accountType)
+        {
+        }
+
+        /// <summary>
+        /// Provides FTX.US fee model
+        /// </summary>
+        /// <param name="security">The security to get a fee model for</param>
+        /// <returns>The new fee model for this brokerage</returns>
+        public override IFeeModel GetFeeModel(Security security)
+            => new FTXUSFeeModel();
+    }
+}

Common/Brokerages/IBrokerageModel.cs

@@ -231,6 +231,9 @@ namespace QuantConnect.Brokerages
                 case BrokerageName.FTX:
                     return new FTXBrokerageModel(accountType);
 
+                case BrokerageName.FTXUS:
+                    return new FTXUSBrokerageModel(accountType);
+
                 default:
                     throw new ArgumentOutOfRangeException(nameof(brokerage), brokerage, null);
             }

Common/Brokerages/SamcoBrokerageModel.cs

@@ -167,7 +167,7 @@ namespace QuantConnect.Brokerages
                 return 1m;
             }
 
-            if (security.Type == SecurityType.Equity || security.Type == SecurityType.Future || security.Type == SecurityType.Option)
+            if (security.Type == SecurityType.Equity || security.Type == SecurityType.Future || security.Type == SecurityType.Option || security.Type == SecurityType.Index)
             {
                 return _maxLeverage;
             }

Common/Brokerages/ZerodhaBrokerageModel.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -164,7 +164,7 @@ namespace QuantConnect.Brokerages
                 return 1m;
             }
 
-            if (security.Type == SecurityType.Equity || security.Type == SecurityType.Future || security.Type == SecurityType.Option)
+            if (security.Type == SecurityType.Equity || security.Type == SecurityType.Future || security.Type == SecurityType.Option || security.Type == SecurityType.Index)
             {
                 return _maxLeverage;
             }

Common/Currencies.cs

@@ -23,24 +23,34 @@ namespace QuantConnect
     public static class Currencies
     {
         /// <summary>
-        /// USD currency string
+        /// USD (United States Dollar) currency string
         /// </summary>
-        public static string USD = "USD";
+        public const string USD = "USD";
 
         /// <summary>
-        /// EUR currency string
+        /// EUR (Euro) currency string
         /// </summary>
-        public static string EUR = "EUR";
+        public const string EUR = "EUR";
 
         /// <summary>
-        /// GBP currency string
+        /// GBP (British pound sterling) currency string
         /// </summary>
-        public static string GBP = "GBP";
+        public const string GBP = "GBP";
 
         /// <summary>
-        /// INR currency string
+        /// INR (Indian rupee) currency string
         /// </summary>
-        public static string INR = "INR";
+        public const string INR = "INR";
+
+        /// <summary>
+        /// CNH (Chinese Yuan Renminbi) currency string
+        /// </summary>
+        public const string CNH = "CNH";
+
+        /// <summary>
+        /// HKD (Hong Kong dollar) currency string
+        /// </summary>
+        public const string HKD = "HKD";
 
         /// <summary>
         /// Null currency used when a real one is not required
@@ -55,19 +65,19 @@ namespace QuantConnect
         /// </remarks>
         public static readonly IReadOnlyDictionary<string, string> CurrencySymbols = new Dictionary<string, string>
         {
-            {"USD", "$"},
-            {"GBP", "₤"},
+            {USD, "$"},
+            {GBP, "₤"},
             {"JPY", "¥"},
-            {"EUR", "€"},
+            {EUR, "€"},
             {"NZD", "$"},
             {"AUD", "$"},
             {"CAD", "$"},
             {"CHF", "Fr"},
-            {"HKD", "$"},
+            {HKD, "$"},
             {"SGD", "$"},
             {"XAG", "Ag"},
             {"XAU", "Au"},
-            {"CNH", "¥"},
+            {CNH, "¥"},
             {"CNY", "¥"},
             {"CZK", "Kč"},
             {"DKK", "kr"},

Common/Data/Custom/Quandl.cs

@@ -1,156 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using System.Collections.Generic;
-using System.Globalization;
-using System.Net;
-
-namespace QuantConnect.Data.Custom
-{
-    /// <summary>
-    /// Quandl Data Type - Import generic data from quandl, without needing to define Reader methods.
-    /// This reads the headers of the data imported, and dynamically creates properties for the imported data.
-    /// </summary>
-    public class Quandl : DynamicData
-    {
-        private bool _isInitialized;
-        private readonly List<string> _propertyNames = new List<string>();
-        private readonly string _valueColumn;
-        private static string _authCode = "";
-
-        /// <summary>
-        /// Static constructor for the <see cref="Quandl"/> class
-        /// </summary>
-        static Quandl()
-        {
-            // The Quandl API now requires TLS 1.2 for API requests (since 9/18/2018).
-            // NET 4.5.2 and below does not enable this more secure protocol by default, so we add it in here
-            ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
-        }
-
-        /// <summary>
-        /// Flag indicating whether or not the Quanl auth code has been set yet
-        /// </summary>
-        public static bool IsAuthCodeSet
-        {
-            get;
-            private set;
-        }
-
-        /// <summary>
-        /// The end time of this data. Some data covers spans (trade bars) and as such we want
-        /// to know the entire time span covered
-        /// </summary>
-        public override DateTime EndTime
-        {
-            get { return Time + Period; }
-            set { Time = value - Period; }
-        }
-
-        /// <summary>
-        /// Gets a time span of one day
-        /// </summary>
-        public TimeSpan Period
-        {
-            get { return QuantConnect.Time.OneDay; }
-        }
-
-        /// <summary>
-        /// Default quandl constructor uses Close as its value column
-        /// </summary>
-        public Quandl() : this("Close")
-        {
-        }
-
-        /// <summary>
-        /// Constructor for creating customized quandl instance which doesn't use "Close" as its value item.
-        /// </summary>
-        /// <param name="valueColumnName"></param>
-        protected Quandl(string valueColumnName)
-        {
-            _valueColumn = valueColumnName;
-        }
-
-        /// <summary>
-        /// Generic Reader Implementation for Quandl Data.
-        /// </summary>
-        /// <param name="config">Subscription configuration</param>
-        /// <param name="line">CSV line of data from the souce</param>
-        /// <param name="date">Date of the requested line</param>
-        /// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
-        /// <returns></returns>
-        public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
-        {
-            // be sure to instantiate the correct type
-            var data = (Quandl) Activator.CreateInstance(GetType());
-            data.Symbol = config.Symbol;
-            var csv = line.Split(',');
-
-            if (!_isInitialized)
-            {
-                _isInitialized = true;
-                foreach (var propertyName in csv)
-                {
-                    var property = propertyName.Trim();
-                    // should we remove property names like Time?
-                    // do we need to alias the Time??
-                    data.SetProperty(property, 0m);
-                    _propertyNames.Add(property);
-                }
-                // Returns null at this point where we are only reading the properties names
-                return null;
-            }
-
-            data.Time = DateTime.ParseExact(csv[0], "yyyy-MM-dd", CultureInfo.InvariantCulture);
-
-            for (var i = 1; i < csv.Length; i++)
-            {
-                var value = csv[i].ToDecimal();
-                data.SetProperty(_propertyNames[i], value);
-            }
-
-            // we know that there is a close property, we want to set that to 'Value'
-            data.Value = (decimal)data.GetProperty(_valueColumn);
-
-            return data;
-        }
-
-        /// <summary>
-        /// Quandl Source Locator: Using the Quandl V1 API automatically set the URL for the dataset.
-        /// </summary>
-        /// <param name="config">Subscription configuration object</param>
-        /// <param name="date">Date of the data file we're looking for</param>
-        /// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
-        /// <returns>STRING API Url for Quandl.</returns>
-        public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
-        {
-            var source = $"https://www.quandl.com/api/v3/datasets/{config.Symbol.Value}.csv?order=asc&api_key={_authCode}";
-            return new SubscriptionDataSource(source, SubscriptionTransportMedium.RemoteFile);
-        }
-
-        /// <summary>
-        /// Set the auth code for the quandl set to the QuantConnect auth code.
-        /// </summary>
-        /// <param name="authCode"></param>
-        public static void SetAuthCode(string authCode)
-        {
-            if (string.IsNullOrWhiteSpace(authCode)) return;
-
-            _authCode = authCode;
-            IsAuthCodeSet = true;
-        }
-    }
-}

Common/Data/DiskDataCacheProvider.cs

@@ -16,9 +16,11 @@
 
 using System.IO;
 using Ionic.Zip;
-using QuantConnect.Interfaces;
-using QuantConnect.Logging;
+using System.Linq;
 using QuantConnect.Util;
+using QuantConnect.Logging;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
 
 namespace QuantConnect.Data
 {
@@ -93,6 +95,15 @@ namespace QuantConnect.Data
             Compression.ZipCreateAppendData(filePath, entryName, data, true);
         }
 
+        /// <summary>
+        /// Returns a list of zip entries in a provided zip file
+        /// </summary>
+        public List<string> GetZipEntries(string zipFile)
+        {
+            using var stream = new FileStream(zipFile, FileMode.Open, FileAccess.Read);
+            return Compression.GetZipEntryFileNames(stream).ToList();
+        }
+
         /// <summary>
         /// Dispose for this class
         /// </summary>

Common/Data/HistoryRequestFactory.cs

@@ -17,6 +17,7 @@ using System;
 using NodaTime;
 using QuantConnect.Interfaces;
 using QuantConnect.Securities;
+using QuantConnect.Util;
 
 namespace QuantConnect.Data
 {
@@ -53,12 +54,21 @@ namespace QuantConnect.Data
         {
             resolution ??= subscription.Resolution;
 
+            var dataType = subscription.Type;
+
+            // if we change resolution the data type can change, for example subscription being Tick type and resolution daily
+            // data type here won't be Tick anymore, but TradeBar/QuoteBar
+            if (resolution.Value != subscription.Resolution && LeanData.IsCommonLeanDataType(dataType))
+            {
+                dataType = LeanData.GetDataType(resolution.Value, subscription.TickType);
+            }
+
             var request = new HistoryRequest(subscription,
                 exchangeHours,
                 startAlgoTz.ConvertToUtc(_algorithm.TimeZone),
                 endAlgoTz.ConvertToUtc(_algorithm.TimeZone))
             {
-                DataType = subscription.Type,
+                DataType = dataType,
                 Resolution = resolution.Value,
                 FillForwardResolution = subscription.FillDataForward ? resolution : null,
                 TickType = subscription.TickType

Common/Data/Market/OptionContract.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -13,8 +13,10 @@
  * limitations under the License.
 */
 
-using System;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
 using QuantConnect.Securities.Option;
+using System;
 
 namespace QuantConnect.Data.Market
 {
@@ -23,8 +25,7 @@ namespace QuantConnect.Data.Market
     /// </summary>
     public class OptionContract
     {
-        private Lazy<OptionPriceModelResult> _optionPriceModelResult = new Lazy<OptionPriceModelResult>(() => 
-                                                                            new OptionPriceModelResult(0m, new Greeks())); 
+        private Lazy<OptionPriceModelResult> _optionPriceModelResult = new(() => OptionPriceModelResult.None);
 
         /// <summary>
         /// Gets the option contract's symbol
@@ -176,5 +177,40 @@ namespace QuantConnect.Data.Market
         /// A string that represents the current object.
         /// </returns>
         public override string ToString() => Symbol.Value;
+
+        /// <summary>
+        /// Creates a <see cref="OptionContract"/> 
+        /// </summary>
+        /// <param name="baseData"></param>
+        /// <param name="security">provides price properties for a <see cref="Security"/></param>
+        /// <param name="underlyingLastPrice">last price the underlying security traded at</param>
+        /// <returns>Option contract</returns>
+        public static OptionContract Create(BaseData baseData, ISecurityPrice security, decimal underlyingLastPrice)
+            => Create(baseData.Symbol, baseData.Symbol.Underlying, baseData.EndTime, security, underlyingLastPrice);
+
+        /// <summary>
+        /// Creates a <see cref="OptionContract"/>
+        /// </summary>
+        /// <param name="symbol">The option contract symbol</param>
+        /// <param name="underlyingSymbol">The symbol of the underlying security</param>
+        /// <param name="endTime">local date time this contract's data was last updated</param>
+        /// <param name="security">provides price properties for a <see cref="Security"/></param>
+        /// <param name="underlyingLastPrice">last price the underlying security traded at</param>
+        /// <returns>Option contract</returns>
+        public static OptionContract Create(Symbol symbol, Symbol underlyingSymbol, DateTime endTime, ISecurityPrice security, decimal underlyingLastPrice)
+        {
+            return new OptionContract(symbol, underlyingSymbol)
+            {
+                Time = endTime,
+                LastPrice = security.Close,
+                Volume = (long)security.Volume,
+                BidPrice = security.BidPrice,
+                BidSize = (long)security.BidSize,
+                AskPrice = security.AskPrice,
+                AskSize = (long)security.AskSize,
+                OpenInterest = security.OpenInterest,
+                UnderlyingLastPrice = underlyingLastPrice
+            };
+        }
     }
 }

Common/Data/Slice.cs

@@ -18,7 +18,7 @@ using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using System.Reflection;
-using QuantConnect.Data.Custom;
+using QuantConnect.Data.Custom.IconicTypes;
 using QuantConnect.Data.Market;
 using QuantConnect.Python;
 
@@ -44,7 +44,7 @@ namespace QuantConnect.Data
         // string -> data   for non-tick data
         // string -> list{data} for tick data
         private readonly Lazy<DataDictionary<SymbolData>> _data;
-        // Quandl -> DataDictonary<Quandl>
+        // UnlinkedData -> DataDictonary<UnlinkedData>
         private Dictionary<Type, object> _dataByType;
 
         /// <summary>
@@ -284,7 +284,7 @@ namespace QuantConnect.Data
         /// Gets the data corresponding to the specified symbol. If the requested data
         /// is of <see cref="MarketDataType.Tick"/>, then a <see cref="List{Tick}"/> will
         /// be returned, otherwise, it will be the subscribed type, for example, <see cref="TradeBar"/>
-        /// or event <see cref="Quandl"/> for custom data.
+        /// or event <see cref="UnlinkedData"/> for custom data.
         /// </summary>
         /// <param name="symbol">The data's symbols</param>
         /// <returns>The data for the specified symbol</returns>
@@ -304,7 +304,7 @@ namespace QuantConnect.Data
         /// <summary>
         /// Gets the <see cref="DataDictionary{T}"/> for all data of the specified type
         /// </summary>
-        /// <typeparam name="T">The type of data we want, for example, <see cref="TradeBar"/> or <see cref="Quandl"/>, ect...</typeparam>
+        /// <typeparam name="T">The type of data we want, for example, <see cref="TradeBar"/> or <see cref="UnlinkedData"/>, ect...</typeparam>
         /// <returns>The <see cref="DataDictionary{T}"/> containing the data of the specified type</returns>
         public DataDictionary<T> Get<T>()
             where T : IBaseData

Common/Data/SubscriptionDataConfigExtensions.cs

@@ -127,15 +127,16 @@ namespace QuantConnect.Data
         /// <param name="config">The subscription data configuration we are processing</param>
         /// <remarks>One of the objectives of this method is to normalize the 'use price scale'
         /// check and void code duplication and related issues</remarks>
+        /// <param name="liveMode">True, is this is a live mode data stream</param>
         /// <returns>True if ticker prices should be scaled</returns>
-        public static bool PricesShouldBeScaled(this SubscriptionDataConfig config)
+        public static bool PricesShouldBeScaled(this SubscriptionDataConfig config, bool liveMode = false)
         {
             if (config.IsCustomData || config.Symbol.Value.Contains("UNIVERSE"))
             {
                 return false;
             }
 
-            if(config.SecurityType == SecurityType.Equity)
+            if(config.SecurityType == SecurityType.Equity && !liveMode)
             {
                 return true;
             }

Common/Extensions.cs

@@ -2505,11 +2505,25 @@ namespace QuantConnect
             {
                 try
                 {
+                    // We must first check if allowPythonDerivative is true to then only return true
+                    // when the PyObject is assignable from Type or IEnumerable and is a C# type
+                    // wrapped in PyObject
+                    if (allowPythonDerivative)
+                    {
+                        result = (T)pyObject.AsManagedObject(type);
+                        return true;
+                    }
+
                     // Special case: Type
                     if (typeof(Type).IsAssignableFrom(type))
                     {
                         result = (T)pyObject.AsManagedObject(type);
-                        return true;
+                        // pyObject is a C# object wrapped in PyObject, in this case return true
+                        // Otherwise, pyObject is a python object that subclass a C# class, only return true if 'allowPythonDerivative'
+                        var castedResult = (Type)pyObject.AsManagedObject(type);
+                        var pythonName = pyObject.GetAttr("__name__").GetAndDispose<string>();
+
+                        return pythonName == castedResult.Name;
                     }
 
                     // Special case: IEnumerable
@@ -2535,7 +2549,7 @@ namespace QuantConnect
                     // Otherwise, pyObject is a python object that subclass a C# class, only return true if 'allowPythonDerivative'
                     var name = (((dynamic) pythonType).__name__ as PyObject).GetAndDispose<string>();
                     pythonType.Dispose();
-                    return allowPythonDerivative || name == result.GetType().Name;
+                    return name == result.GetType().Name;
                 }
                 catch
                 {
@@ -2835,9 +2849,7 @@ namespace QuantConnect
         public static Type CreateType(this PyObject pyObject)
         {
             Type type;
-            if (pyObject.TryConvert(out type) &&
-                type != typeof(PythonQuandl) &&
-                type != typeof(PythonData))
+            if (pyObject.TryConvert(out type))
             {
                 return type;
             }

Common/Interfaces/IDataCacheProvider.cs

@@ -16,6 +16,7 @@
 
 using System;
 using System.IO;
+using System.Collections.Generic;
 
 namespace QuantConnect.Interfaces
 {
@@ -42,5 +43,10 @@ namespace QuantConnect.Interfaces
         /// <param name="key">The source of the data, used as a key to retrieve data in the cache</param>
         /// <param name="data">The data to cache as a byte array</param>
         void Store(string key, byte[] data);
+
+        /// <summary>
+        /// Returns a list of zip entries in a provided zip file
+        /// </summary>
+        List<string> GetZipEntries(string zipFile);
     }
 }

Common/Market.cs

@@ -62,7 +62,8 @@ namespace QuantConnect
             Tuple.Create(HKFE, 25),
 
             Tuple.Create(CFE, 33),
-            Tuple.Create(FTX, 34)
+            Tuple.Create(FTX, 34),
+            Tuple.Create(FTXUS, 35)
         };
 
         static Market()
@@ -212,6 +213,11 @@ namespace QuantConnect
         /// </summary>
         public const string FTX = "ftx";
 
+        /// <summary>
+        /// FTX.US
+        /// </summary>
+        public const string FTXUS = "ftxus";
+
 
         /// <summary>
         /// Adds the specified market to the map of available markets with the specified identifier.
@@ -279,5 +285,13 @@ namespace QuantConnect
         {
             return !ReverseMarkets.TryGetValue(code, out var market) ? null : market;
         }
+
+        /// <summary>
+        /// Returns a list of the supported markets
+        /// </summary>
+        public static List<string> SupportedMarkets()
+        {
+            return Markets.Keys.ToList();
+        }
     }
 }

Common/Orders/Fees/FTXFeeModel.cs

@@ -14,7 +14,6 @@
 */
 
 using QuantConnect.Securities;
-using QuantConnect.Securities.Crypto;
 
 namespace QuantConnect.Orders.Fees
 {
@@ -27,11 +26,12 @@ namespace QuantConnect.Orders.Fees
         /// <summary>
         /// Tier 1 maker fees
         /// </summary>
-        public const decimal MakerFee = 0.0002m;
+        public virtual decimal MakerFee => 0.0002m;
+        
         /// <summary>
         /// Tier 1 taker fees
         /// </summary>
-        public const decimal TakerFee = 0.0007m;
+        public virtual decimal TakerFee => 0.0007m;
 
         /// <summary>
         /// Get the fee for this order in quote currency

Common/Orders/Fees/FTXUSFeeModel.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -13,33 +13,22 @@
  * limitations under the License.
 */
 
-using QuantConnect.Data;
-using QuantConnect.Data.Custom;
-using System;
-using System.Collections.Generic;
-
-namespace QuantConnect.Python
+namespace QuantConnect.Orders.Fees
 {
     /// <summary>
-    /// Dynamic data class for Python algorithms.
+    /// Provides an implementation of <see cref="FeeModel"/> that models FTX order fees
+    /// https://help.ftx.us/hc/en-us/articles/360043579273-Fees
     /// </summary>
-    public class PythonQuandl : Quandl
+    public class FTXUSFeeModel : FTXFeeModel
     {
         /// <summary>
-        /// Constructor for initialising the PythonQuandl class
+        /// Tier 1 maker fees
         /// </summary>
-        public PythonQuandl() : base("Close")
-        {
-            //Empty constructor required for fast-reflection initialization
-        }
-
+        public override decimal MakerFee => 0.001m;
+        
         /// <summary>
-        /// Constructor for creating customized quandl instance which doesn't use "Close" as its value item.
+        /// Tier 1 taker fees
         /// </summary>
-        /// <param name="valueColumnName"></param>
-        public PythonQuandl(string valueColumnName) : base(valueColumnName)
-        {
-            //
-        }
+        public override decimal TakerFee => 0.004m;
     }
-}
+}

Common/Orders/Fees/InteractiveBrokersFeeModel.cs

@@ -33,9 +33,16 @@ namespace QuantConnect.Orders.Fees
         private readonly Dictionary<string, Func<decimal, decimal, CashAmount>> _optionFee =
             new Dictionary<string, Func<decimal, decimal, CashAmount>>();
 
-        private readonly Dictionary<string, CashAmount> _futureFee =
+        /// <summary>
+        /// Reference at https://www.interactivebrokers.com/en/index.php?f=commission&p=futures1
+        /// </summary>
+        private readonly Dictionary<string, Func<Security, CashAmount>> _futureFee =
             //                                                               IB fee + exchange fee
-            new Dictionary<string, CashAmount> { { Market.USA, new CashAmount(0.85m + 1, "USD") } };
+            new()
+            {
+                { Market.USA, UnitedStatesFutureFees },
+                { Market.HKFE, HongKongFutureFees }
+            };
 
         /// <summary>
         /// Initializes a new instance of the <see cref="ImmediateFillModel"/>
@@ -109,17 +116,18 @@ namespace QuantConnect.Orders.Fees
                     if (market == Market.Globex || market == Market.NYMEX
                         || market == Market.CBOT || market == Market.ICE
                         || market == Market.CFE || market == Market.COMEX
-                        || market == Market.CME || market == Market.HKFE)
+                        || market == Market.CME)
                     {
                         // just in case...
                         market = Market.USA;
                     }
 
-                    CashAmount feeRatePerContract;
-                    if (!_futureFee.TryGetValue(market, out feeRatePerContract))
+                    if (!_futureFee.TryGetValue(market, out var feeRatePerContractFunc))
                     {
                         throw new KeyNotFoundException($"InteractiveBrokersFeeModel(): unexpected future Market {market}");
                     }
+
+                    var feeRatePerContract = feeRatePerContractFunc(security);
                     feeResult = order.AbsoluteQuantity * feeRatePerContract.Amount;
                     feeCurrency = feeRatePerContract.Currency;
                     break;
@@ -129,7 +137,10 @@ namespace QuantConnect.Orders.Fees
                     switch (market)
                     {
                         case Market.USA:
-                            equityFee = new EquityFee("USD", feePerShare: 0.005m, minimumFee: 1, maximumFeeRate: 0.005m);
+                            equityFee = new EquityFee(Currencies.USD, feePerShare: 0.005m, minimumFee: 1, maximumFeeRate: 0.005m);
+                            break;
+                        case Market.India:
+                            equityFee = new EquityFee(Currencies.INR, feePerShare: 0.01m, minimumFee: 6, maximumFeeRate: 20);
                             break;
                         default:
                             throw new KeyNotFoundException($"InteractiveBrokersFeeModel(): unexpected equity Market {market}");
@@ -235,6 +246,42 @@ namespace QuantConnect.Orders.Fees
             }
         }
 
+        private static CashAmount UnitedStatesFutureFees(Security security)
+        {
+            return new CashAmount(0.85m + 1, Currencies.USD);
+        }
+
+        /// <summary>
+        /// See https://www.hkex.com.hk/Services/Rules-and-Forms-and-Fees/Fees/Listed-Derivatives/Trading/Transaction?sc_lang=en
+        /// </summary>
+        private static CashAmount HongKongFutureFees(Security security)
+        {
+            if (security.Symbol.ID.Symbol.Equals("HSI", StringComparison.InvariantCultureIgnoreCase))
+            {
+                // IB fee + exchange fee
+                return new CashAmount(30 + 10, Currencies.HKD);
+            }
+
+            decimal ibFeePerContract;
+            switch (security.QuoteCurrency.Symbol)
+            {
+                case Currencies.CNH:
+                    ibFeePerContract = 13;
+                    break;
+                case Currencies.HKD:
+                    ibFeePerContract = 20;
+                    break;
+                case Currencies.USD:
+                    ibFeePerContract = 2.40m;
+                    break;
+                default:
+                    throw new ArgumentException($"Unexpected quote currency {security.QuoteCurrency.Symbol} for Hong Kong futures exchange");
+            }
+
+            // let's add a 50% extra charge for exchange fees
+            return new CashAmount(ibFeePerContract * 1.5m, security.QuoteCurrency.Symbol);
+        }
+
         /// <summary>
         /// Helper class to handle IB Equity fees
         /// </summary>

Common/Python/PythonActivator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -42,40 +42,13 @@ namespace QuantConnect.Python
         {
             Type = type;
 
-            var isPythonQuandl = false;
-
-            using (Py.GIL())
+            Factory = x =>
             {
-                var pythonType = value.Invoke().GetPythonType();
-                isPythonQuandl = pythonType.As<Type>() == typeof(PythonQuandl);
-                pythonType.Dispose();
-            }
-
-            if (isPythonQuandl)
-            {
-                Factory = x =>
+                using (Py.GIL())
                 {
-                    using (Py.GIL())
-                    {
-                        var instance = value.Invoke();
-                        var pyValueColumnName = instance.GetAttr("ValueColumnName");
-                        var valueColumnName = pyValueColumnName.ToString();
-                        instance.Dispose();
-                        pyValueColumnName.Dispose();
-                        return new PythonQuandl(valueColumnName);
-                    }
-                };
-            }
-            else
-            {
-                Factory = x =>
-                {
-                    using (Py.GIL())
-                    {
-                        var instance = value.Invoke();
-                        return new PythonData(instance);
-                    }
-                };
+                    var instance = value.Invoke();
+                    return new PythonData(instance);
+                }
             };
         }
     }

Common/Python/PythonSlice.cs

@@ -15,7 +15,7 @@
 
 using Python.Runtime;
 using QuantConnect.Data;
-using QuantConnect.Data.Custom;
+using QuantConnect.Data.Custom.IconicTypes;
 using QuantConnect.Data.Market;
 using System.Collections.Generic;
 
@@ -111,7 +111,7 @@ namespace QuantConnect.Python
         /// Gets the data corresponding to the specified symbol. If the requested data
         /// is of <see cref="MarketDataType.Tick"/>, then a <see cref="List{Tick}"/> will
         /// be returned, otherwise, it will be the subscribed type, for example, <see cref="TradeBar"/>
-        /// or event <see cref="Quandl"/> for custom data.
+        /// or event <see cref="UnlinkedData"/> for custom data.
         /// </summary>
         /// <param name="symbol">The data's symbols</param>
         /// <returns>The data for the specified symbol</returns>

Common/QuantConnect.csproj

@@ -35,7 +35,7 @@
     <Message Text="SelectedOptimization $(SelectedOptimization)" Importance="high" />
   </Target>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="CloneExtensions" Version="1.3.0" />
     <PackageReference Include="fasterflect" Version="3.0.0" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />

Common/Securities/Option/ConstantQLDividendYieldEstimator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -13,22 +13,15 @@
  * limitations under the License.
 */
 
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
-using QLNet;
-using QuantConnect.Util;
 
 namespace QuantConnect.Securities.Option
 {
     /// <summary>
     /// Class implements default flat dividend yield curve estimator, implementing <see cref="IQLDividendYieldEstimator"/>.  
     /// </summary>
-    class ConstantQLDividendYieldEstimator : IQLDividendYieldEstimator
+    public class ConstantQLDividendYieldEstimator : IQLDividendYieldEstimator
     {
         private readonly double _dividendYield;
         /// <summary>

Common/Securities/Option/ConstantQLRiskFreeRateEstimator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -14,14 +14,8 @@
 */
 
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
-using QuantConnect.Util;
-using QLNet;
 
 namespace QuantConnect.Securities.Option
 {
@@ -30,12 +24,12 @@ namespace QuantConnect.Securities.Option
     /// </summary>
     public class ConstantQLRiskFreeRateEstimator : IQLRiskFreeRateEstimator
     {
-        private readonly double _riskFreeRate;
+        private readonly decimal _riskFreeRate;
         /// <summary>
         /// Constructor initializes class with risk free rate constant
         /// </summary>
         /// <param name="riskFreeRate"></param>
-        public ConstantQLRiskFreeRateEstimator(double riskFreeRate = 0.01)
+        public ConstantQLRiskFreeRateEstimator(decimal riskFreeRate = 0.01m)
         {
             _riskFreeRate = riskFreeRate;
         }
@@ -49,8 +43,6 @@ namespace QuantConnect.Securities.Option
         /// <param name="contract">The option contract to evaluate</param>
         /// <returns>The estimate</returns>
         public double Estimate(Security security, Slice slice, OptionContract contract)
-        {
-            return _riskFreeRate;
-        }
+            => Convert.ToDouble(_riskFreeRate);
     }
 }

Common/Securities/Option/ConstantQLUnderlyingVolatilityEstimator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -13,15 +13,8 @@
  * limitations under the License.
 */
 
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
-using QuantConnect.Util;
-using QLNet;
 
 namespace QuantConnect.Securities.Option
 {
@@ -29,8 +22,13 @@ namespace QuantConnect.Securities.Option
     /// Class implements default underlying constant volatility estimator (<see cref="IQLUnderlyingVolatilityEstimator"/>.), that projects the underlying own volatility 
     /// model into corresponding option pricing model.
     /// </summary>
-    class ConstantQLUnderlyingVolatilityEstimator : IQLUnderlyingVolatilityEstimator
+    public class ConstantQLUnderlyingVolatilityEstimator : IQLUnderlyingVolatilityEstimator
     {
+        /// <summary>
+        /// Indicates whether volatility model has been warmed ot not
+        /// </summary>
+        public bool IsReady { get; private set; }
+
         /// <summary>
         /// Returns current estimate of the underlying volatility
         /// </summary>
@@ -48,6 +46,7 @@ namespace QuantConnect.Securities.Option
                 option.Underlying.VolatilityModel != null &&
                 option.Underlying.VolatilityModel.Volatility > 0m)
             {
+                IsReady = true;
                 return (double)option.Underlying.VolatilityModel.Volatility;
             }
 

Common/Securities/Option/IQLDividendYieldEstimator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -15,12 +15,6 @@
 
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
-using QLNet;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace QuantConnect.Securities.Option
 {
@@ -28,7 +22,7 @@ namespace QuantConnect.Securities.Option
     /// Defines QuantLib dividend yield estimator for option pricing model. User may define his own estimators, 
     /// including those forward and backward looking ones.
     /// </summary>
-    interface IQLDividendYieldEstimator
+    public interface IQLDividendYieldEstimator
     {
         /// <summary>
         /// Returns current estimate of the stock dividend yield

Common/Securities/Option/IQLRiskFreeRateEstimator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -15,19 +15,13 @@
 
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
-using QLNet;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace QuantConnect.Securities.Option
 {
     /// <summary>
     /// Defines QuantLib risk free rate estimator for option pricing model. 
     /// </summary>
-    interface IQLRiskFreeRateEstimator
+    public interface IQLRiskFreeRateEstimator
     {
         /// <summary>
         /// Returns current estimate of the risk free rate

Common/Securities/Option/IQLUnderlyingVolatilityEstimator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -15,12 +15,6 @@
 
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
-using QLNet;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace QuantConnect.Securities.Option
 {
@@ -28,7 +22,7 @@ namespace QuantConnect.Securities.Option
     /// Defines QuantLib underlying volatility estimator for option pricing model. User may define his own estimators, 
     /// including those forward and backward looking ones.
     /// </summary>
-    interface IQLUnderlyingVolatilityEstimator
+    public interface IQLUnderlyingVolatilityEstimator
     {
         /// <summary>
         /// Returns current estimate of the underlying volatility
@@ -39,5 +33,10 @@ namespace QuantConnect.Securities.Option
         /// <param name="contract">The option contract to evaluate</param>
         /// <returns>Volatility</returns>
         double Estimate(Security security, Slice slice, OptionContract contract);
+
+        /// <summary>
+        /// Indicates whether volatility model is warmed up or no
+        /// </summary>
+        bool IsReady { get; }
     }
 }

Common/Securities/Option/OptionPriceModelResult.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -23,6 +23,11 @@ namespace QuantConnect.Securities.Option
     /// </summary>
     public class OptionPriceModelResult
     {
+        /// <summary>
+        /// Represents the zero option price and greeks.
+        /// </summary>
+        public static OptionPriceModelResult None { get; } = new(0, new Greeks());
+
         private readonly Lazy<Greeks> _greeks;
         private readonly Lazy<decimal> _impliedVolatility;
 
@@ -81,4 +86,4 @@ namespace QuantConnect.Securities.Option
             _greeks = new Lazy<Greeks>(greeks);
         }
     }
-}
+}

Common/Securities/Option/OptionPriceModels.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -13,12 +13,11 @@
  * limitations under the License.
 */
 
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using QLNet;
+using System;
+using System.Globalization;
+using System.Linq;
+using Fasterflect;
 
 namespace QuantConnect.Securities.Option
 {
@@ -43,7 +42,27 @@ namespace QuantConnect.Securities.Option
         private const int _timeStepsFD = 100;
 
         /// <summary>
-        /// Pricing engine for European vanilla options using analytical formulae. 
+        /// Creates pricing engine by engine type name. 
+        /// </summary>
+        /// <param name="priceEngineName">QL price engine name</param>
+        /// <param name="riskFree">The risk free rate</param>
+        /// <returns>New option price model instance of specific engine</returns>
+        public static IOptionPriceModel Create(string priceEngineName, decimal riskFree)
+        {
+            var type = AppDomain.CurrentDomain.GetAssemblies()
+                .Where(a => !a.IsDynamic)
+                .SelectMany(a => a.GetTypes())
+                .Where(s => s.Implements(typeof(IPricingEngine)))
+                .FirstOrDefault(t => t.FullName?.EndsWith(priceEngineName, StringComparison.InvariantCulture) == true);
+
+            return new QLOptionPriceModel(process => (IPricingEngine)Activator.CreateInstance(type, process),
+                _underlyingVolEstimator,
+                new ConstantQLRiskFreeRateEstimator(riskFree),
+                _dividendYieldEstimator);
+        }
+
+        /// <summary>
+        /// Pricing engine for European vanilla options using analytical formula. 
         /// QuantLib reference: http://quantlib.org/reference/class_quant_lib_1_1_analytic_european_engine.html
         /// </summary>
         /// <returns>New option price model instance</returns>
@@ -103,7 +122,7 @@ namespace QuantConnect.Securities.Option
         {
             PricingEngineFuncEx pricingEngineFunc = (symbol, process) =>
                             symbol.ID.OptionStyle == OptionStyle.American ?
-                            new FDAmericanEngine(process, _timeStepsFD, _timeStepsFD - 1) as IPricingEngine:
+                            new FDAmericanEngine(process, _timeStepsFD, _timeStepsFD - 1) as IPricingEngine :
                             new FDEuropeanEngine(process, _timeStepsFD, _timeStepsFD - 1) as IPricingEngine;
 
             return new QLOptionPriceModel(pricingEngineFunc,

Common/Securities/Option/QLOptionPriceModel.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -28,7 +28,7 @@ namespace QuantConnect.Securities.Option
     /// <summary>
     /// Provides QuantLib(QL) implementation of <see cref="IOptionPriceModel"/> to support major option pricing models, available in QL.
     /// </summary>
-    class QLOptionPriceModel : IOptionPriceModel
+    public class QLOptionPriceModel : IOptionPriceModel
     {
         private readonly IQLUnderlyingVolatilityEstimator _underlyingVolEstimator;
         private readonly IQLRiskFreeRateEstimator _riskFreeRateEstimator;
@@ -41,6 +41,11 @@ namespace QuantConnect.Securities.Option
         /// </summary>
         public bool EnableGreekApproximation { get; set; } = true;
 
+        /// <summary>
+        /// True if volatility model is warmed up, i.e. has generated volatility value different from zero, otherwise false.
+        /// </summary>
+        public bool VolatilityEstimatorWarmedUp => _underlyingVolEstimator.IsReady;
+
         /// <summary>
         /// Method constructs QuantLib option price model with necessary estimators of underlying volatility, risk free rate, and underlying dividend yield
         /// </summary>
@@ -83,6 +88,12 @@ namespace QuantConnect.Securities.Option
         {
             try
             {
+                // expired options has no price
+                if (contract.Time > contract.Expiry)
+                {
+                    return OptionPriceModelResult.None;
+                }
+
                 // setting up option pricing parameters
                 var calendar = new UnitedStates();
                 var dayCounter = new Actual365Fixed();
@@ -101,6 +112,11 @@ namespace QuantConnect.Securities.Option
                 var underlyingVolValue = new SimpleQuote(_underlyingVolEstimator.Estimate(security, slice, contract));
                 var underlyingVol = new Handle<BlackVolTermStructure>(new BlackConstantVol(0, calendar, new Handle<Quote>(underlyingVolValue), dayCounter));
 
+                if (!_underlyingVolEstimator.IsReady)
+                {
+                    return OptionPriceModelResult.None;
+                }
+
                 // preparing stochastic process and payoff functions
                 var stochasticProcess = new BlackScholesMertonProcess(new Handle<Quote>(underlyingQuoteValue), dividendYield, riskFreeRate, underlyingVol);
                 var payoff = new PlainVanillaPayoff(contract.Right == OptionRight.Call ? QLNet.Option.Type.Call : QLNet.Option.Type.Put, (double)contract.Strike);
@@ -116,7 +132,8 @@ namespace QuantConnect.Securities.Option
                 option.setPricingEngine(_pricingEngineFunc(contract.Symbol, stochasticProcess));
 
                 // running calculations
-                var npv = EvaluateOption(option);
+                // can return negative value in neighbourhood of 0
+                var npv = Math.Max(0, EvaluateOption(option));
 
                 // function extracts QL greeks catching exception if greek is not generated by the pricing engine and reevaluates option to get numerical estimate of the seisitivity
                 Func<Func<double>, Func<double>, decimal> tryGetGreekOrReevaluate = (greek, reevalFunc) =>
@@ -217,10 +234,10 @@ namespace QuantConnect.Securities.Option
                                             () => tryGetGreekOrReevaluate(() => option.rho(), reevalRho),
                                             () => tryGetGreek(() => option.elasticity())));
             }
-            catch(Exception err)
+            catch (Exception err)
             {
                 Log.Debug($"QLOptionPriceModel.Evaluate() error: {err.Message}");
-                return new OptionPriceModelResult(0m, new Greeks());
+                return OptionPriceModelResult.None;
             }
         }
 

Common/Securities/Volatility/StandardDeviationOfReturnsVolatilityModel.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -35,10 +35,10 @@ namespace QuantConnect.Securities
         private decimal _volatility;
         private DateTime _lastUpdate = DateTime.MinValue;
         private decimal _lastPrice;
-        private readonly Resolution? _resolution;
-        private readonly TimeSpan _periodSpan;
+        private Resolution? _resolution;
+        private TimeSpan _periodSpan;
         private readonly object _sync = new object();
-        private readonly RollingWindow<double> _window;
+        private RollingWindow<double> _window;
 
         /// <summary>
         /// Gets the volatility of the security as a percentage
@@ -92,7 +92,7 @@ namespace QuantConnect.Securities
         {
             if (periods < 2)
             {
-                throw new ArgumentOutOfRangeException("periods", "'periods' must be greater than or equal to 2.");
+                throw new ArgumentOutOfRangeException(nameof(periods), "'periods' must be greater than or equal to 2.");
             }
 
             _window = new RollingWindow<double>(periods);
@@ -100,6 +100,30 @@ namespace QuantConnect.Securities
             _periodSpan = updateFrequency ?? resolution?.ToTimeSpan() ?? TimeSpan.FromDays(1);
         }
 
+        /// <summary>
+        /// Initializes a new instance of the <see cref="StandardDeviationOfReturnsVolatilityModel"/> class
+        /// </summary>
+        /// <param name="resolution">
+        /// Resolution of the price data inserted into the rolling window series to calculate standard deviation.
+        /// Will be used as the default value for update frequency if a value is not provided for <paramref name="updateFrequency"/>.
+        /// This only has a material effect in live mode. For backtesting, this value does not cause any behavioral changes.
+        /// </param>
+        /// <param name="updateFrequency">Frequency at which we insert new values into the rolling window for the standard deviation calculation</param>
+        /// <remarks>
+        /// The volatility model will be updated with the most granular/highest resolution data that was added to your algorithm.
+        /// That means that if I added <see cref="Resolution.Tick"/> data for my Futures strategy, that this model will be
+        /// updated using <see cref="Resolution.Tick"/> data as the algorithm progresses in time.
+        ///
+        /// Keep this in mind when setting the period and update frequency. The Resolution parameter is only used for live mode, or for
+        /// the default value of the <paramref name="updateFrequency"/> if no value is provided.
+        /// </remarks>
+        public StandardDeviationOfReturnsVolatilityModel(
+            Resolution resolution,
+            TimeSpan? updateFrequency = null
+            ) : this(PeriodsInResolution(resolution), resolution, updateFrequency)
+        {
+        }
+
         /// <summary>
         /// Updates this model using the new price information in
         /// the specified security instance
@@ -139,5 +163,28 @@ namespace QuantConnect.Securities
                 _resolution,
                 _window.Size + 1);
         }
+
+        private static int PeriodsInResolution(Resolution resolution)
+        {
+            int periods;
+            switch (resolution)
+            {
+                case Resolution.Tick:
+                case Resolution.Second:
+                    periods = 600;
+                    break;
+                case Resolution.Minute:
+                    periods = 60 * 24;
+                    break;
+                case Resolution.Hour:
+                    periods = 24 * 30;
+                    break;
+                default:
+                    periods = 30;
+                    break;
+            }
+
+            return periods;
+        }
     }
-}
+}

Common/Util/Composer.cs

@@ -27,6 +27,7 @@ using System.Reflection;
 using System.Threading;
 using System.Threading.Tasks;
 using QuantConnect.Configuration;
+using QuantConnect.Data;
 using QuantConnect.Logging;
 
 namespace QuantConnect.Util
@@ -241,8 +242,10 @@ namespace QuantConnect.Util
         /// </summary>
         /// <typeparam name="T">The type of the export</typeparam>
         /// <param name="typeName">The name of the type to find. This can be an assembly qualified name, a full name, or just the type's name</param>
+        /// <param name="forceTypeNameOnExisting">When false, if any existing instance of type T is found, it will be returned even if type name doesn't match.
+        /// This is useful in cases where a single global instance is desired, like for <see cref="IDataAggregator"/></param>
         /// <returns>The export instance</returns>
-        public T GetExportedValueByTypeName<T>(string typeName)
+        public T GetExportedValueByTypeName<T>(string typeName, bool forceTypeNameOnExisting = true)
             where T : class
         {
             try
@@ -254,8 +257,8 @@ namespace QuantConnect.Util
                     var type = typeof(T);
                     if (_exportedValues.TryGetValue(type, out values))
                     {
-                        // if we've alread loaded this part, then just return the same one
-                        instance = values.OfType<T>().FirstOrDefault(x => x.GetType().MatchesTypeName(typeName));
+                        // if we've already loaded this part, then just return the same one
+                        instance = values.OfType<T>().FirstOrDefault(x => !forceTypeNameOnExisting || x.GetType().MatchesTypeName(typeName));
                         if (instance != null)
                         {
                             return instance;

Common/Util/LeanData.cs

@@ -38,8 +38,8 @@ namespace QuantConnect.Util
         /// The different <see cref="SecurityType"/> used for data paths
         /// </summary>
         /// <remarks>This includes 'alternative'</remarks>
-        public static IReadOnlyList<string> SecurityTypeAsDataPath => Enum.GetNames(typeof(SecurityType))
-            .Select(x => x.ToLowerInvariant()).Union(new[] { "alternative" }).ToList();
+        public static HashSet<string> SecurityTypeAsDataPath => Enum.GetNames(typeof(SecurityType))
+            .Select(x => x.ToLowerInvariant()).Union(new[] { "alternative" }).ToHashSet();
 
         /// <summary>
         /// Converts the specified base data instance into a lean data file csv line.
@@ -472,7 +472,8 @@ namespace QuantConnect.Util
         ///  <see cref="QuoteBar"/> or <see cref="OpenInterest"/></returns>
         public static bool IsCommonLeanDataType(Type baseDataType)
         {
-            if (baseDataType == typeof(TradeBar) ||
+            if (baseDataType == typeof(Tick) ||
+                baseDataType == typeof(TradeBar) ||
                 baseDataType == typeof(QuoteBar) ||
                 baseDataType == typeof(OpenInterest))
             {
@@ -965,9 +966,11 @@ namespace QuantConnect.Util
         /// </summary>
         /// <param name="fileName">File name to be parsed</param>
         /// <param name="securityType">The securityType as parsed from the fileName</param>
-        public static bool TryParseSecurityType(string fileName, out SecurityType securityType)
+        /// <param name="market">The market as parsed from the fileName</param>
+        public static bool TryParseSecurityType(string fileName, out SecurityType securityType, out string market)
         {
             securityType = SecurityType.Base;
+            market = string.Empty;
 
             try
             {
@@ -976,6 +979,13 @@ namespace QuantConnect.Util
                 // find the securityType and parse it
                 var typeString = info.Find(x => SecurityTypeAsDataPath.Contains(x.ToLowerInvariant()));
                 securityType = ParseDataSecurityType(typeString);
+
+                var existingMarkets = Market.SupportedMarkets();
+                var foundMarket = info.Find(x => existingMarkets.Contains(x.ToLowerInvariant()));
+                if (foundMarket != null)
+                {
+                    market = foundMarket;
+                }
             }
             catch (Exception e)
             {

Configuration/Config.cs

@@ -298,8 +298,8 @@ namespace QuantConnect.Configuration
         /// </summary>
         /// <typeparam name="T">The desired output type</typeparam>
         /// <param name="key">The configuration key</param>
-        /// <param name="value">The output value</param>
-        /// <returns>True on successful parse, false when output value is default(T)</returns>
+        /// <param name="value">The output value. If the key is found and parsed successfully, it will be the parsed value, else default(T).</param>
+        /// <returns>True on successful parse or if they key is not found. False only when key is found but fails to parse.</returns>
         public static bool TryGetValue<T>(string key, out T value)
         {
             return TryGetValue(key, default(T), out value);
@@ -312,8 +312,8 @@ namespace QuantConnect.Configuration
         /// <typeparam name="T">The desired output type</typeparam>
         /// <param name="key">The configuration key</param>
         /// <param name="defaultValue">The default value to use on key not found or unsuccessful parse</param>
-        /// <param name="value">The output value</param>
-        /// <returns>True on successful parse, false when output value is defaultValue</returns>
+        /// <param name="value">The output value. If the key is found and parsed successfully, it will be the parsed value, else defaultValue.</param>
+        /// <returns>True on successful parse or if they key is not found and using defaultValue. False only when key is found but fails to parse.</returns>
         public static bool TryGetValue<T>(string key, T defaultValue, out T value)
         {
             try

Configuration/ToolboxArgumentParser.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -77,6 +77,9 @@ namespace QuantConnect.Configuration
                 new CommandLineOption("splits-percentage", CommandOptionType.SingleValue, "[OPTIONAL for RandomDataGenerator. Sets the probability each equity generated will have a stock split event. Note that this is not the total probability for all symbols generated. Only used for Equity. Defaults to 15.0: Example: --splits-percentage=10.0 ]"),
                 new CommandLineOption("dividends-percentage", CommandOptionType.SingleValue, "[OPTIONAL for RandomDataGenerator. Sets the probability each equity generated will have dividends. Note that this is not the probability for all symbols genearted. Only used for Equity. Defaults to 60.0: Example: --dividends-percentage=25.5 ]"),
                 new CommandLineOption("dividend-every-quarter-percentage", CommandOptionType.SingleValue, "[OPTIONAL for RandomDataGenerator. Sets the probability each equity generated will have a dividend event every quarter. Note that this is not the total probability for all symbols generated. Only used for Equity. Defaults to 30.0: Example: --dividend-every-quarter-percentage=15.0 ]"),
+                new CommandLineOption("option-price-engine", CommandOptionType.SingleValue, "[OPTIONAL for RandomDataGenerator. Sets the stochastic process, and returns new pricing engine to run calculations for that option. Defaults to BaroneAdesiWhaleyApproximationEngine: Example: --option-price-engine=BaroneAdesiWhaleyApproximationEngine ]"),
+                new CommandLineOption("volatility-model-resolution", CommandOptionType.SingleValue, "[OPTIONAL for RandomDataGenerator. Sets the volatility model period span. Defaults to Daily: Example: --volatility-model-resolution=Daily ]"),
+                new CommandLineOption("chain-symbol-count", CommandOptionType.SingleValue, "[OPTIONAL for RandomDataGenerator. Sets the size of the option chain. Defaults to 1 put and 1 call: Example: --chain-symbol-count=2 ]")
             };
 
         /// <summary>

Data/equity/india/factor_files/cccl.csv

@@ -0,0 +1,5 @@
+20100104,0.9800619,0.2,413
+20100209,0.9800619,0.2,421
+20100615,0.9800619,1,88.8
+20110615,0.9856115,1,34.75
+20501231,1,1,0

Data/equity/india/map_files/3mindia.csv

@@ -0,0 +1,3 @@
+19990101,birla3m
+20040615,birla3m
+20501231,3mindia