Deprecate PTVSD for DebugPy (#6153)

* Deprecate PTVSD for DebugPy

* Replace all references to PTVSD with debugpy

* Address review

.vscode/launch.json

@@ -8,8 +8,8 @@
         marketplace.
 
         Attach to Python:
-        Will attempt to attach to LEAN running locally using PTVSD. Requires that the process is
-        actively running and config is set: "debugging": true, "debugging-method": "PTVSD",
+        Will attempt to attach to LEAN running locally using DebugPy. Requires that the process is
+        actively running and config is set: "debugging": true, "debugging-method": "DebugPy",
         Requires Python extension from the marketplace. Currently only works with algorithms in
         Algorithm.Python directory. This is because we map that directory to our build directory
         that contains the py file at runtime. If using another location change "localRoot" value 

.vscode/readme.md

@@ -95,7 +95,7 @@ Python algorithms require a little extra work in order to be able to debug them.
 First in order to debug a Python algorithm in VS Code we must make the following change to our configuration (Launcher\config.json) under the comment debugging configuration:
 
     "debugging": true,
-    "debugging-method": "PTVSD",
+    "debugging-method": "DebugPy,
 
 In setting this we are telling Lean to expect a debugger connection using ‘Python Tools for Visual Studio Debugger’. Once this is set Lean will stop upon initialization and await a connection to the debugger via port 5678.
 

Algorithm.CSharp/AddRemoveOptionUniverseRegressionAlgorithm.cs

@@ -40,8 +40,8 @@ namespace QuantConnect.Algorithm.CSharp
         private int _expectedContractIndex;
         private readonly List<Symbol> _expectedContracts = new List<Symbol>
         {
-            SymbolRepresentation.ParseOptionTickerOSI("GOOG  151224P00750000"),
             SymbolRepresentation.ParseOptionTickerOSI("GOOG  151224P00747500"),
+            SymbolRepresentation.ParseOptionTickerOSI("GOOG  151224P00750000"),
             SymbolRepresentation.ParseOptionTickerOSI("GOOG  151224P00752500")
         };
 
@@ -109,6 +109,11 @@ namespace QuantConnect.Algorithm.CSharp
                 var googOptionChain = AddOption(UnderlyingTicker);
                 googOptionChain.SetFilter(u =>
                 {
+                    // we added the universe at 10, the universe selection data should not be from before
+                    if (u.Underlying.EndTime.Hour < 10)
+                    {
+                        throw new Exception($"Unexpected underlying data point {u.Underlying.EndTime} {u.Underlying}");
+                    }
                     // find first put above market price
                     return u.IncludeWeeklys()
                         .Strikes(+1, +1)
@@ -231,7 +236,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Treynor Ratio", "0"},
             {"Total Fees", "$6.00"},
             {"Estimated Strategy Capacity", "$2000.00"},
-            {"Lowest Capacity Asset", "GOOCV 305RBQ2BZBZT2|GOOCV VP83T1ZUHROL"},
+            {"Lowest Capacity Asset", "GOOCV 305RBR0BSWIX2|GOOCV VP83T1ZUHROL"},
             {"Fitness Score", "0"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
@@ -251,7 +256,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "1e7b3e90918777b9dbf46353a96f3329"}
+            {"OrderListHash", "550a99c482106defd8ba15f48183768e"}
         };
     }
 }

Algorithm.CSharp/Alphas/MortgageRateVolatilityAlpha.cs

@@ -1,162 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-using System;
-using System.Linq;
-using QuantConnect.Data;
-using QuantConnect.Indicators;
-using QuantConnect.Orders.Fees;
-using QuantConnect.Data.Custom;
-using System.Collections.Generic;
-using QuantConnect.Algorithm.Framework.Alphas;
-using QuantConnect.Algorithm.Framework.Execution;
-using QuantConnect.Algorithm.Framework.Portfolio;
-using QuantConnect.Algorithm.Framework.Risk;
-using QuantConnect.Algorithm.Framework.Selection;
-
-namespace QuantConnect.Algorithm.CSharp.Alphas
-{
-    ///<summary>
-    /// This Alpha Model uses Wells Fargo 30-year Fixed Rate Mortgage data from Quandl to
-    /// generate Insights about the movement of Real Estate ETFs. Mortgage rates can provide information
-    /// regarding the general price trend of real estate, and ETFs provide good continuous-time instruments
-    /// to measure the impact against. Volatility in mortgage rates tends to put downward pressure on real
-    /// estate prices, whereas stable mortgage rates, regardless of true rate, lead to stable or higher real
-    /// estate prices. This Alpha model seeks to take advantage of this correlation by emitting insights
-    /// based on volatility and rate deviation from its historic mean.
-    ///
-    /// This alpha is part of the Benchmark Alpha Series created by QuantConnect which are open
-    /// sourced so the community and client funds can see an example of an alpha.
-    ///</summary>
-    public class MortgageRateVolatilityAlgorithm : QCAlgorithm
-    {
-        public override void Initialize()
-        {
-            SetStartDate(2017, 1, 1);  //Set Start Date
-            SetCash(100000);             //Set Strategy Cash
-
-            UniverseSettings.Resolution = Resolution.Daily;
-            SetSecurityInitializer(security => security.FeeModel = new ConstantFeeModel(0));
-
-            // Basket of 6 liquid real estate ETFs
-            Func<string, Symbol> toSymbol = x => QuantConnect.Symbol.Create(x, SecurityType.Equity, Market.USA);
-            var realEstateETFs = new[] { "VNQ", "REET", "TAO", "FREL", "SRET", "HIPS" }.Select(toSymbol).ToArray();
-            SetUniverseSelection(new ManualUniverseSelectionModel(realEstateETFs));
-
-            SetAlpha(new MortgageRateVolatilityAlphaModel(this));
-
-            SetPortfolioConstruction(new EqualWeightingPortfolioConstructionModel());
-
-            SetExecution(new ImmediateExecutionModel());
-
-            SetRiskManagement(new NullRiskManagementModel());
-
-        }
-
-        private class MortgageRateVolatilityAlphaModel : AlphaModel
-        {
-            private readonly int _indicatorPeriod;
-            private readonly Resolution _resolution;
-            private readonly TimeSpan _insightDuration;
-            private readonly int _deviations;
-            private readonly double _insightMagnitude;
-            private readonly Symbol _mortgageRate;
-            private readonly SimpleMovingAverage _mortgageRateSma;
-            private readonly StandardDeviation _mortgageRateStd;
-
-            public MortgageRateVolatilityAlphaModel(
-                QCAlgorithm algorithm,
-                int indicatorPeriod = 15,
-                double insightMagnitude = 0.0005,
-                int deviations = 2,
-                Resolution resolution = Resolution.Daily
-                )
-            {
-            	// Add Quandl data for a Well's Fargo 30-year Fixed Rate mortgage
-                _mortgageRate = algorithm.AddData<QuandlMortgagePriceColumns>("WFC/PR_GOV_30YFIXEDVA_APR").Symbol;
-                _indicatorPeriod = indicatorPeriod;
-                _resolution = resolution;
-                _insightDuration = resolution.ToTimeSpan().Multiply(indicatorPeriod);
-                _insightMagnitude = insightMagnitude;
-                _deviations = deviations;
-                
-                // Add indicators for the mortgage rate -- Standard Deviation and Simple Moving Average
-                _mortgageRateStd = algorithm.STD(_mortgageRate, _indicatorPeriod, resolution);
-                _mortgageRateSma = algorithm.SMA(_mortgageRate, _indicatorPeriod, resolution);
-                
-                // Use a history call to warm-up the indicators
-                WarmUpIndicators(algorithm);
-            }
-            
-            public override IEnumerable<Insight> Update(QCAlgorithm algorithm, Slice data)
-            {
-                var insights = new List<Insight>();
-                
-                // Return empty list if data slice doesn't contain monrtgage rate data
-                if (!data.Keys.Contains(_mortgageRate))
-                {
-                    return insights;
-                }
-				// Extract current mortgage rate, the current STD indicator value, and current SMA value
-                var rate = data[_mortgageRate].Value;
-                var deviation = _deviations * _mortgageRateStd;
-                var sma = _mortgageRateSma;
-
-				// Loop through all Active Securities to emit insights
-                foreach (var security in algorithm.ActiveSecurities.Keys)
-                {
-                	// Mortgage rate Symbol will be in the collection, so skip it
-                    if (security == _mortgageRate)
-                    {
-                        return insights;
-                    }
-
-					// If volatility in mortgage rates is high, then we emit an Insight to sell
-                    if ((rate < sma - deviation) || (rate > sma + deviation))
-                    {
-                        insights.Add(Insight.Price(security, _insightDuration, InsightDirection.Down, _insightMagnitude));
-                    }
-                    
-                    // If volatility in mortgage rates is low, then we emit an Insight to buy
-                    if ((rate < sma - (decimal)deviation/2) || (rate > sma + (decimal)deviation/2))
-                    {
-                        insights.Add(Insight.Price(security, _insightDuration, InsightDirection.Up, _insightMagnitude));
-                    }
-                }
-
-                return insights;
-            }
-
-            private void WarmUpIndicators(QCAlgorithm algorithm)
-            {
-            	// Make a history call and update the indicators
-                algorithm.History(new[] { _mortgageRate }, _indicatorPeriod, _resolution).PushThrough(bar =>
-                {
-                    _mortgageRateSma.Update(bar.EndTime, bar.Value);
-                    _mortgageRateStd.Update(bar.EndTime, bar.Value);
-                });
-            }
-        }
-        public class QuandlMortgagePriceColumns : Quandl
-        {
-            public QuandlMortgagePriceColumns()
-            
-            // Rename the Quandl object column to the data we want, which is the 'Value' column
-	        // of the CSV that our API call returns
-                : base(valueColumnName: "Value")
-            {
-            }
-        }
-    }
-}

Algorithm.CSharp/BasicTemplateIndiaAlgorithm.cs

@@ -33,13 +33,13 @@ namespace QuantConnect.Algorithm.CSharp
         /// </summary>
         public override void Initialize()
         {
-            SetStartDate(2003, 10, 07);  //Set Start Date
-            SetEndDate(2003, 10, 11);    //Set End Date
+            SetStartDate(2019, 1, 23);  //Set Start Date
+            SetEndDate(2019, 10, 31);    //Set End Date
             SetCash(100000);             //Set Strategy Cash
 
             // Find more symbols here: http://quantconnect.com/data
             // Equities Resolutions: Tick, Second, Minute, Hour, Daily.
-            AddEquity("UNIONBANK", Resolution.Second, Market.India);
+            AddEquity("YESBANK", Resolution.Minute, Market.India);
 
             //Set Order Prperties as per the requirements for order placement
             DefaultOrderProperties = new IndiaOrderProperties(exchange: Exchange.NSE);
@@ -58,7 +58,7 @@ namespace QuantConnect.Algorithm.CSharp
         {
             if (!Portfolio.Invested)
             {
-                var marketTicket = MarketOrder("UNIONBANK", 1);
+                var marketTicket = MarketOrder("YESBANK", 1);
             }
         }
 
@@ -73,12 +73,12 @@ namespace QuantConnect.Algorithm.CSharp
         /// <summary>
         /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
         /// </summary>
-        public bool CanRunLocally { get; } = false;
+        public bool CanRunLocally { get; } = true;
 
         /// <summary>
         /// This is used by the regression test system to indicate which languages this algorithm is written in.
         /// </summary>
-        public Language[] Languages { get; } = { Language.CSharp };
+        public Language[] Languages { get; } = { Language.CSharp, Language.Python };
 
         /// <summary>
         /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm

Algorithm.CSharp/CustomDataIndicatorExtensionsAlgorithm.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -14,16 +14,17 @@
 */
 
 using System;
+using System.IO;
+using System.Globalization;
 using QuantConnect.Data;
-using QuantConnect.Data.Custom;
-using QuantConnect.Data.Market;
+using QuantConnect.Util;
 using QuantConnect.Indicators;
 
 namespace QuantConnect.Algorithm.CSharp
 {
     /// <summary>
     /// The algorithm creates new indicator value with the existing indicator method by Indicator Extensions
-    /// Demonstration of using the external custom datasource Quandl to request the VIX and VXV daily data
+    /// Demonstration of using local custom datasource CustomData to request the IBM and SPY daily data
     /// </summary>
     /// <meta name="tag" content="using data" />
     /// <meta name="tag" content="using quantconnect" />
@@ -34,10 +35,10 @@ namespace QuantConnect.Algorithm.CSharp
     /// <meta name="tag" content="charting" />
     public class CustomDataIndicatorExtensionsAlgorithm : QCAlgorithm
     {
-        private const string _vix = "CBOE/VIX";
-        private const string _vxv = "CBOE/VXV";
-        private SimpleMovingAverage _smaVIX;
-        private SimpleMovingAverage _smaVXV;
+        private const string _ibm = "IBM";
+        private const string _spy = "SPY";
+        private SimpleMovingAverage _smaIBM;
+        private SimpleMovingAverage _smaSPY;
         private IndicatorBase<IndicatorDataPoint> _ratio;
 
         /// <summary>
@@ -50,46 +51,82 @@ namespace QuantConnect.Algorithm.CSharp
             SetCash(25000);
 
             // Define the symbol and "type" of our generic data
-            AddData<QuandlVix>(_vix, Resolution.Daily);
-            AddData<Quandl>(_vxv, Resolution.Daily);
+            AddData<CustomData>(_ibm, Resolution.Daily);
+            AddData<CustomData>(_spy, Resolution.Daily);
             // Set up default Indicators, these are just 'identities' of the closing price
-            _smaVIX = SMA(_vix, 1);
-            _smaVXV = SMA(_vxv, 1);
-            // This will create a new indicator whose value is smaVXV / smaVIX
-            _ratio = _smaVXV.Over(_smaVIX);
+            _smaIBM = SMA(_ibm, 1);
+            _smaSPY = SMA(_spy, 1);
+            // This will create a new indicator whose value is smaSPY / smaIBM
+            _ratio = _smaSPY.Over(_smaIBM);
         }
 
         /// <summary>
         /// Custom data event handler:
         /// </summary>
-        /// <param name="data">Quandl - dictionary Bars of Quandl Data</param>
-        public void OnData(Quandl data)
+        /// <param name="data">CustomData - dictionary Bars of custom data</param>
+        public void OnData(CustomData data)
         {
             // Wait for all indicators to fully initialize
-            if (_smaVIX.IsReady && _smaVXV.IsReady && _ratio.IsReady)
+            if (_smaIBM.IsReady && _smaSPY.IsReady && _ratio.IsReady)
             {
                 if (!Portfolio.Invested && _ratio > 1)
                 {
-                    MarketOrder(_vix, 100);
+                    MarketOrder(_ibm, 100);
                 }
                 else if (_ratio < 1)
                 {
                     Liquidate();
                 }
                 // plot all indicators
-                PlotIndicator("SMA", _smaVIX, _smaVXV);
+                PlotIndicator("SMA", _smaIBM, _smaSPY);
                 PlotIndicator("Ratio", _ratio);
             }
         }
     }
 
     /// <summary>
-    /// In CBOE/VIX data, there is a "vix close" column instead of "close" which is the 
-    /// default column namein LEAN Quandl custom data implementation.
-    /// This class assigns new column name to match the the external datasource setting.
+    /// Custom data from local LEAN data
     /// </summary>
-    public class QuandlVix : Quandl
+    public class CustomData : BaseData
     {
-        public QuandlVix() : base(valueColumnName: "vix close") { }
+        public decimal Open;
+        public decimal High;
+        public decimal Low;
+        public decimal Close;
+
+        public override DateTime EndTime
+        {
+            get { return Time + Period; }
+            set { Time = value - Period; }
+        }
+
+        public TimeSpan Period
+        {
+            get { return QuantConnect.Time.OneDay; }
+        }
+
+        public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
+        {
+            var source = Path.Combine(Globals.DataFolder, "equity", "usa", config.Resolution.ToString().ToLower(), LeanData.GenerateZipFileName(config.Symbol, date, config.Resolution, config.TickType));
+            return new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.Csv);
+        }
+
+        public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
+        {
+            var csv = line.ToCsv(6);
+            var _scaleFactor = 1 / 10000m;
+
+            var custom = new CustomData
+            {
+                Symbol = config.Symbol,
+                Time = DateTime.ParseExact(csv[0], DateFormat.TwelveCharacter, CultureInfo.InvariantCulture),
+                Open = csv[1].ToDecimal() * _scaleFactor,
+                High = csv[2].ToDecimal() * _scaleFactor,
+                Low = csv[3].ToDecimal() * _scaleFactor,
+                Close = csv[4].ToDecimal() * _scaleFactor,
+                Value = csv[4].ToDecimal() * _scaleFactor
+            };
+            return custom;
+        }
     }
-}
+}

Algorithm.CSharp/HistoryAlgorithm.cs

@@ -14,10 +14,10 @@
 */
 
 using System;
+using QuantConnect.Util;
 using System.Collections.Generic;
 using System.Linq;
 using QuantConnect.Data;
-using QuantConnect.Data.Custom;
 using QuantConnect.Data.Market;
 using QuantConnect.Indicators;
 using QuantConnect.Securities.Equity;
@@ -36,7 +36,7 @@ namespace QuantConnect.Algorithm.CSharp
     public class HistoryAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
         private int _count;
-        private SimpleMovingAverage _spyDailySma;
+        private SimpleMovingAverage _dailySma;
 
         /// <summary>
         /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
@@ -49,12 +49,12 @@ namespace QuantConnect.Algorithm.CSharp
 
             // Find more symbols here: http://quantconnect.com/data
             var SPY = AddSecurity(SecurityType.Equity, "SPY", Resolution.Daily).Symbol;
-            var CME_SP1 = AddData<QuandlFuture>("CHRIS/CME_SP1", Resolution.Daily).Symbol;
+            var IBM = AddData<CustomData>("IBM", Resolution.Daily).Symbol;
             // specifying the exchange will allow the history methods that accept a number of bars to return to work properly
-            Securities["CHRIS/CME_SP1"].Exchange = new EquityExchange();
+            Securities["IBM"].Exchange = new EquityExchange();
 
             // we can get history in initialize to set up indicators and such
-            _spyDailySma = new SimpleMovingAverage(14);
+            _dailySma = new SimpleMovingAverage(14);
 
             // get the last calendar year's worth of SPY data at the configured resolution (daily)
             var tradeBarHistory = History<TradeBar>("SPY", TimeSpan.FromDays(365));
@@ -76,99 +76,97 @@ namespace QuantConnect.Algorithm.CSharp
             // we can use these TradeBars to initialize indicators or perform other math
             foreach (TradeBar tradeBar in tradeBarHistory)
             {
-                _spyDailySma.Update(tradeBar.EndTime, tradeBar.Close);
+                _dailySma.Update(tradeBar.EndTime, tradeBar.Close);
             }
 
-            // get the last calendar year's worth of quandl data at the configured resolution (daily)
-            var quandlHistory = History<QuandlFuture>("CHRIS/CME_SP1", TimeSpan.FromDays(365));
-            AssertHistoryCount("History<Quandl>(\"CHRIS/CME_SP1\", TimeSpan.FromDays(365))", quandlHistory, 250, CME_SP1);
+            // get the last calendar year's worth of IBM data at the configured resolution (daily)
+            var customDataHistory = History<CustomData>("IBM", TimeSpan.FromDays(365));
+            AssertHistoryCount("History<CustomData>(\"IBM\", TimeSpan.FromDays(365))", customDataHistory, 250, IBM);
 
-            // get the last 14 bars of SPY at the configured resolution (daily)
-            quandlHistory = History<QuandlFuture>("CHRIS/CME_SP1", 14);
-            AssertHistoryCount("History<Quandl>(\"CHRIS/CME_SP1\", 14)", quandlHistory, 14, CME_SP1);
+            // get the last 14 bars of IBM at the configured resolution (daily)
+            customDataHistory = History<CustomData>("IBM", 14);
+            AssertHistoryCount("History<CustomData>(\"IBM\", 14)", customDataHistory, 14, IBM);
 
-            // get the last 14 minute bars of SPY
-
-            // we can loop over the return values from these functions and we'll get Quandl data
+            // we can loop over the return values from these functions and we'll get custom data
             // this can be used in much the same way as the tradeBarHistory above
-            _spyDailySma.Reset();
-            foreach (QuandlFuture quandl in quandlHistory)
+            _dailySma.Reset();
+            foreach (CustomData customData in customDataHistory)
             {
-                _spyDailySma.Update(quandl.EndTime, quandl.Value);
+                _dailySma.Update(customData.EndTime, customData.Value);
             }
 
-            // get the last year's worth of all configured Quandl data at the configured resolution (daily)
-            var allQuandlData = History<QuandlFuture>(TimeSpan.FromDays(365));
-            AssertHistoryCount("History<QuandlFuture>(TimeSpan.FromDays(365))", allQuandlData, 250, CME_SP1);
+            // get the last year's worth of all configured custom data at the configured resolution (daily)
+            var allCustomData = History<CustomData>(TimeSpan.FromDays(365));
+            AssertHistoryCount("History<CustomData>(TimeSpan.FromDays(365))", allCustomData, 250, IBM);
 
-            // get the last 14 bars worth of Quandl data for the specified symbols at the configured resolution (daily)
-            allQuandlData = History<QuandlFuture>(Securities.Keys, 14);
-            AssertHistoryCount("History<QuandlFuture>(Securities.Keys, 14)", allQuandlData, 14, CME_SP1);
+            // get the last 14 bars worth of custom data for the specified symbols at the configured resolution (daily)
+            allCustomData = History<CustomData>(Securities.Keys, 14);
+            AssertHistoryCount("History<CustomData>(Securities.Keys, 14)", allCustomData, 14, IBM);
 
-            // NOTE: using different resolutions require that they are properly implemented in your data type, since
-            //  Quandl doesn't support minute data, this won't actually work, but if your custom data source has
-            //  different resolutions, it would need to be implemented in the GetSource and Reader methods properly
-            //quandlHistory = History<QuandlFuture>("CHRIS/CME_SP1", TimeSpan.FromDays(7), Resolution.Minute);
-            //quandlHistory = History<QuandlFuture>("CHRIS/CME_SP1", 14, Resolution.Minute);
-            //allQuandlData = History<QuandlFuture>(TimeSpan.FromDays(365), Resolution.Minute);
-            //allQuandlData = History<QuandlFuture>(Securities.Keys, 14, Resolution.Minute);
-            //allQuandlData = History<QuandlFuture>(Securities.Keys, TimeSpan.FromDays(1), Resolution.Minute);
-            //allQuandlData = History<QuandlFuture>(Securities.Keys, 14, Resolution.Minute);
+            // NOTE: Using different resolutions require that they are properly implemented in your data type. If your
+            // custom data source has different resolutions, it would need to be implemented in the GetSource and Reader
+            // methods properly.
+            //customDataHistory = History<CustomData>("IBM", TimeSpan.FromDays(7), Resolution.Minute);
+            //customDataHistory = History<CustomData>("IBM", 14, Resolution.Minute);
+            //allCustomData = History<CustomData>(TimeSpan.FromDays(365), Resolution.Minute);
+            //allCustomData = History<CustomData>(Securities.Keys, 14, Resolution.Minute);
+            //allCustomData = History<CustomData>(Securities.Keys, TimeSpan.FromDays(1), Resolution.Minute);
+            //allCustomData = History<CustomData>(Securities.Keys, 14, Resolution.Minute);
 
-            // get the last calendar year's worth of all quandl data
-            allQuandlData = History<QuandlFuture>(Securities.Keys, TimeSpan.FromDays(365));
-            AssertHistoryCount("History<QuandlFuture>(Securities.Keys, TimeSpan.FromDays(365))", allQuandlData, 250, CME_SP1);
+            // get the last calendar year's worth of all custom data
+            allCustomData = History<CustomData>(Securities.Keys, TimeSpan.FromDays(365));
+            AssertHistoryCount("History<CustomData>(Securities.Keys, TimeSpan.FromDays(365))", allCustomData, 250, IBM);
 
-            // the return is a series of dictionaries containing all quandl data at each time
+            // the return is a series of dictionaries containing all custom data at each time
             // we can loop over it to get the individual dictionaries
-            foreach (DataDictionary<QuandlFuture> quandlsDataDictionary in allQuandlData)
+            foreach (DataDictionary<CustomData> customDataDictionary in allCustomData)
             {
-                // we can access the dictionary to get the quandl data we want
-                var quandl = quandlsDataDictionary["CHRIS/CME_SP1"];
+                // we can access the dictionary to get the custom data we want
+                var customData = customDataDictionary["IBM"];
             }
 
             // we can also access the return value from the multiple symbol functions to request a single
             // symbol and then loop over it
-            var singleSymbolQuandl = allQuandlData.Get("CHRIS/CME_SP1");
-            AssertHistoryCount("allQuandlData.Get(\"CHRIS/CME_SP1\")", singleSymbolQuandl, 250, CME_SP1);
-            foreach (QuandlFuture quandl in singleSymbolQuandl)
+            var singleSymbolCustomData = allCustomData.Get("IBM");
+            AssertHistoryCount("allCustomData.Get(\"IBM\")", singleSymbolCustomData, 250, IBM);
+            foreach (CustomData customData in singleSymbolCustomData)
             {
-                // do something with 'CHRIS/CME_SP1' quandl data
+                // do something with 'IBM' custom data
             }
 
             // we can also access individual properties on our data, this will
-            // get the 'CHRIS/CME_SP1' quandls like above, but then only return the Low properties
-            var quandlSpyLows = allQuandlData.Get("CHRIS/CME_SP1", "Low");
-            AssertHistoryCount("allQuandlData.Get(\"CHRIS/CME_SP1\", \"Low\")", quandlSpyLows, 250);
-            foreach (decimal low in quandlSpyLows)
+            // get the 'IBM' CustomData objects like above, but then only return the Value properties
+            var customDataIbmValues = allCustomData.Get("IBM", "Value");
+            AssertHistoryCount("allCustomData.Get(\"IBM\", \"Value\")", customDataIbmValues, 250);
+            foreach (decimal value in customDataIbmValues)
             {
-                // do something with each low value
+                // do something with each value
             }
 
             // sometimes it's necessary to get the history for many configured symbols
 
             // request the last year's worth of history for all configured symbols at their configured resolutions
             var allHistory = History(TimeSpan.FromDays(365));
-            AssertHistoryCount("History(TimeSpan.FromDays(365))", allHistory, 250, SPY, CME_SP1);
+            AssertHistoryCount("History(TimeSpan.FromDays(365))", allHistory, 250, SPY, IBM);
 
             // request the last days's worth of history at the minute resolution
             allHistory = History(TimeSpan.FromDays(1), Resolution.Minute);
-            AssertHistoryCount("History(TimeSpan.FromDays(1), Resolution.Minute)", allHistory, 391, SPY, CME_SP1);
+            AssertHistoryCount("History(TimeSpan.FromDays(1), Resolution.Minute)", allHistory, 390, SPY, IBM);
 
             // request the last 100 bars for the specified securities at the configured resolution
             allHistory = History(Securities.Keys, 100);
-            AssertHistoryCount("History(Securities.Keys, 100)", allHistory, 100, SPY, CME_SP1);
+            AssertHistoryCount("History(Securities.Keys, 100)", allHistory, 100, SPY, IBM);
 
             // request the last 100 minute bars for the specified securities
             allHistory = History(Securities.Keys, 100, Resolution.Minute);
-            AssertHistoryCount("History(Securities.Keys, 100, Resolution.Minute)", allHistory, 101, SPY, CME_SP1);
+            AssertHistoryCount("History(Securities.Keys, 100, Resolution.Minute)", allHistory, 100, SPY, IBM);
 
             // request the last calendar years worth of history for the specified securities
             allHistory = History(Securities.Keys, TimeSpan.FromDays(365));
-            AssertHistoryCount("History(Securities.Keys, TimeSpan.FromDays(365))", allHistory, 250, SPY, CME_SP1);
+            AssertHistoryCount("History(Securities.Keys, TimeSpan.FromDays(365))", allHistory, 250, SPY, IBM);
             // we can also specify the resolution
             allHistory = History(Securities.Keys, TimeSpan.FromDays(1), Resolution.Minute);
-            AssertHistoryCount("History(Securities.Keys, TimeSpan.FromDays(1), Resolution.Minute)", allHistory, 391, SPY, CME_SP1);
+            AssertHistoryCount("History(Securities.Keys, TimeSpan.FromDays(1), Resolution.Minute)", allHistory, 390, SPY, IBM);
 
             // if we loop over this allHistory, we get Slice objects
             foreach (Slice slice in allHistory)
@@ -215,7 +213,7 @@ namespace QuantConnect.Algorithm.CSharp
 
             if (_count > 5)
             {
-                throw new Exception("Invalid number of bars arrived. Expected exactly 5");
+                throw new Exception($"Invalid number of bars arrived. Expected exactly 5, but received {_count}");
             }
 
             if (!Portfolio.Invested)
@@ -245,9 +243,9 @@ namespace QuantConnect.Algorithm.CSharp
             }
             else if (typeof(T).IsGenericType && typeof(T).GetGenericTypeDefinition() == typeof(DataDictionary<>))
             {
-                if (typeof(T).GetGenericArguments()[0] == typeof(QuandlFuture))
+                if (typeof(T).GetGenericArguments()[0] == typeof(CustomData))
                 {
-                    var dictionaries = (IEnumerable<DataDictionary<QuandlFuture>>) history;
+                    var dictionaries = (IEnumerable<DataDictionary<CustomData>>) history;
                     unexpectedSymbols = dictionaries.SelectMany(dd => dd.Keys)
                         .Distinct()
                         .Where(sym => !expectedSymbols.Contains(sym))
@@ -340,19 +338,5 @@ namespace QuantConnect.Algorithm.CSharp
             {"Rolling Averaged Population Magnitude", "0%"},
             {"OrderListHash", "33d01821923c397f999cfb2e5b5928ad"}
         };
-
-        /// <summary>
-        /// Custom quandl data type for setting customized value column name. Value column is used for the primary trading calculations and charting.
-        /// </summary>
-        public class QuandlFuture : Quandl
-        {
-            /// <summary>
-            /// Initializes a new instance of the <see cref="QuandlFuture"/> class.
-            /// </summary>
-            public QuandlFuture()
-                : base(valueColumnName: "Settle")
-            {
-            }
-        }
     }
-}
+}

Algorithm.CSharp/IndiaDataRegressionAlgorithm.cs

@@ -0,0 +1,201 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using QuantConnect.Interfaces;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Regression algorithm demonstrating use of map files with India data
+    /// </summary>
+    /// <meta name="tag" content="using data" />
+    /// <meta name="tag" content="India data" />
+    /// <meta name="tag" content="regression test" />
+    /// <meta name="tag" content="rename event" />
+    /// <meta name="tag" content="map" />
+    /// <meta name="tag" content="mapping" />
+    /// <meta name="tag" content="map files" />
+    public class IndiaDataRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        private Symbol _mappingSymbol, _splitAndDividendSymbol;
+        private bool _initialMapping;
+        private bool _executionMapping;
+        private bool _receivedWarningEvent;
+        private bool _receivedOccurredEvent;
+
+        /// <summary>
+        /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
+        /// </summary>
+        public override void Initialize()
+        {
+            SetStartDate(2004, 5, 20);  //Set Start Date
+            SetEndDate(2016, 7, 26);    //Set End Date
+            _mappingSymbol = AddEquity("3MINDIA", Resolution.Daily, Market.India).Symbol;
+            _splitAndDividendSymbol = AddEquity("CCCL", Resolution.Daily, Market.India).Symbol;
+        }
+
+        /// <summary>
+        /// Raises the data event.
+        /// </summary>
+        /// <param name="data">Data.</param>
+        public void OnData(Dividends data)
+        {
+            if (data.ContainsKey(_splitAndDividendSymbol))
+            {
+                var dividend = data[_splitAndDividendSymbol];
+                if (Time.Date == new DateTime(2010, 06, 15) &&
+                    (dividend.Price != 0.5m || dividend.ReferencePrice != 88.8m || dividend.Distribution != 0.5m))
+                {
+                    throw new Exception("Did not receive expected dividend values");
+                }
+            }
+        }
+
+        /// <summary>
+        /// Raises the data event.
+        /// </summary>
+        /// <param name="data">Data.</param>
+        public void OnData(Splits data)
+        {
+            if (data.ContainsKey(_splitAndDividendSymbol))
+            {
+                var split = data[_splitAndDividendSymbol];
+                if (split.Type == SplitType.Warning)
+                {
+                    _receivedWarningEvent = true;
+                }
+                else if (split.Type == SplitType.SplitOccurred)
+                {
+                    _receivedOccurredEvent = true;
+                    if (split.Price != 421m || split.ReferencePrice != 421m || split.SplitFactor != 0.2m)
+                    {
+                        throw new Exception("Did not receive expected split values");
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Checks the symbol change event
+        /// </summary>
+        public override void OnData(Slice slice)
+        {
+            if (slice.SymbolChangedEvents.ContainsKey(_mappingSymbol))
+            {
+                var mappingEvent = slice.SymbolChangedEvents.Single(x => x.Key.SecurityType == SecurityType.Equity).Value;
+                Log($"{Time} - Ticker changed from: {mappingEvent.OldSymbol} to {mappingEvent.NewSymbol}");
+                if (Time.Date == new DateTime(1999, 01, 01))
+                {
+                    _initialMapping = true;
+                }
+                else if (Time.Date == new DateTime(2004, 06, 15))
+                {
+                    if (mappingEvent.NewSymbol == "3MINDIA"
+                        && mappingEvent.OldSymbol == "BIRLA3M")
+                    {
+                        _executionMapping = true;
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Final step of the algorithm
+        /// </summary>
+        public override void OnEndOfAlgorithm()
+        {
+            if (_initialMapping)
+            {
+                throw new Exception("The ticker generated the initial rename event");
+            }
+            if (!_executionMapping)
+            {
+                throw new Exception("The ticker did not rename throughout the course of its life even though it should have");
+            }
+            if (!_receivedOccurredEvent)
+            {
+                throw new Exception("Did not receive expected split event");
+            }
+            if (!_receivedWarningEvent)
+            {
+                throw new Exception("Did not receive expected split warning event");
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public Language[] Languages { get; } = { Language.CSharp, Language.Python };
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Trades", "0"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "0%"},
+            {"Drawdown", "0%"},
+            {"Expectancy", "0"},
+            {"Net Profit", "0%"},
+            {"Sharpe Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "0%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "-0.427"},
+            {"Tracking Error", "0.158"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "$0.00"},
+            {"Estimated Strategy Capacity", "$0"},
+            {"Lowest Capacity Asset", ""},
+            {"Fitness Score", "0"},
+            {"Kelly Criterion Estimate", "0"},
+            {"Kelly Criterion Probability Value", "0"},
+            {"Sortino Ratio", "79228162514264337593543950335"},
+            {"Return Over Maximum Drawdown", "79228162514264337593543950335"},
+            {"Portfolio Turnover", "0"},
+            {"Total Insights Generated", "0"},
+            {"Total Insights Closed", "0"},
+            {"Total Insights Analysis Completed", "0"},
+            {"Long Insight Count", "0"},
+            {"Short Insight Count", "0"},
+            {"Long/Short Ratio", "100%"},
+            {"Estimated Monthly Alpha Value", "$0"},
+            {"Total Accumulated Estimated Alpha Value", "$0"},
+            {"Mean Population Estimated Insight Value", "$0"},
+            {"Mean Population Direction", "0%"},
+            {"Mean Population Magnitude", "0%"},
+            {"Rolling Averaged Population Direction", "0%"},
+            {"Rolling Averaged Population Magnitude", "0%"},
+            {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"}
+        };
+    }
+}

Algorithm.CSharp/IndicatorSuiteAlgorithm.cs

@@ -16,7 +16,6 @@
 using System;
 using System.Collections.Generic;
 using QuantConnect.Data;
-using QuantConnect.Data.Custom;
 using QuantConnect.Data.Market;
 using QuantConnect.Indicators;
 using QuantConnect.Interfaces;
@@ -34,7 +33,7 @@ namespace QuantConnect.Algorithm.CSharp
     public class IndicatorSuiteAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
         private string _ticker = "SPY";
-        private string _customTicker = "WIKI/FB";
+        private string _customTicker = "IBM";
 
         private Symbol _symbol;
         private Symbol _customSymbol;
@@ -64,7 +63,7 @@ namespace QuantConnect.Algorithm.CSharp
             _symbol = AddSecurity(SecurityType.Equity, _ticker, Resolution.Daily).Symbol;
 
             //Add the Custom Data:
-            _customSymbol = AddData<Quandl>(_customTicker, Resolution.Daily).Symbol;
+            _customSymbol = AddData<CustomData>(_customTicker, Resolution.Daily).Symbol;
 
             //Set up default Indicators, these indicators are defined on the Value property of incoming data (except ATR and AROON which use the full TradeBar object)
             _indicators = new Indicators
@@ -118,9 +117,9 @@ namespace QuantConnect.Algorithm.CSharp
             // these are indicators that require multiple inputs. the most common of which is a ratio.
             // suppose we seek the ratio of BTC to SPY, we could write the following:
             var spyClose = Identity(_symbol);
-            var fbClose = Identity(_customSymbol);
+            var ibmClose = Identity(_customSymbol);
             // this will create a new indicator whose value is FB/SPY
-            _ratio = fbClose.Over(spyClose);
+            _ratio = ibmClose.Over(spyClose);
             // we can also easily plot our indicators each time they update using th PlotIndicator function
             PlotIndicator("Ratio", _ratio);
         }
@@ -128,8 +127,8 @@ namespace QuantConnect.Algorithm.CSharp
         /// <summary>
         /// Custom data event handler:
         /// </summary>
-        /// <param name="data">Quandl - dictionary Bars of Quandl Data</param>
-        public void OnData(Quandl data)
+        /// <param name="data">CustomData - dictionary Bars of custom data</param>
+        public void OnData(CustomData data)
         {
         }
 

Algorithm.CSharp/ParameterizedAlgorithm.cs

@@ -28,9 +28,9 @@ namespace QuantConnect.Algorithm.CSharp
     /// <meta name="tag" content="using quantconnect" />
     public class ParameterizedAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
-        // we place attributes on top of our fields or properties that should receive
+        // We place attributes on top of our fields or properties that should receive
         // their values from the job. The values 100 and 200 are just default values that
-        // or only used if the parameters do not exist
+        // are only used if the parameters do not exist.
         [Parameter("ema-fast")]
         public int FastPeriod = 100;
 

Algorithm.CSharp/QuandlFuturesDataAlgorithm.cs

@@ -1,73 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using QuantConnect.Data.Custom;
-
-namespace QuantConnect.Algorithm.CSharp
-{
-    /// <summary>
-    /// Futures demonstration algorithm.
-    /// QuantConnect allows importing generic data sources! This example demonstrates importing a futures
-    /// data from the popular open data source Quandl. QuantConnect has a special deal with Quandl giving you access
-    /// to Stevens Continuous Futurs (SCF) for free. If you'd like to download SCF for local backtesting, you can download it through Quandl.com.
-    /// </summary>
-    /// <meta name="tag" content="using data" />
-    /// <meta name="tag" content="quandl" />
-    /// <meta name="tag" content="custom data" />
-    /// <meta name="tag" content="futures" />
-    public class QCUQuandlFutures : QCAlgorithm
-    {
-        private string _crude = "SCF/CME_CL1_ON";
-
-        /// <summary>
-        /// Initialize the data and resolution you require for your strategy
-        /// </summary>
-        public override void Initialize()
-        {
-            SetStartDate(2000, 1, 1);
-            SetEndDate(DateTime.Now.Date.AddDays(-1));
-            SetCash(25000);
-            AddData<QuandlFuture>(_crude, Resolution.Daily);
-        }
-
-        /// <summary>
-        /// Data Event Handler: New data arrives here. "TradeBars" type is a dictionary of strings so you can access it by symbol.
-        /// </summary>
-        /// <param name="data">Data.</param>
-        public void OnData(Quandl data)
-        {
-            if (!Portfolio.HoldStock)
-            {
-                SetHoldings(_crude, 1);
-                Debug(Time.ToStringInvariant("u") + " Purchased Crude Oil: " + _crude);
-            }
-        }
-
-        /// <summary>
-        /// Custom quandl data type for setting customized value column name. Value column is used for the primary trading calculations and charting.
-        /// </summary>
-        public class QuandlFuture : Quandl
-        {
-            /// <summary>
-            /// Initializes a new instance of the <see cref="QuandlFuture"/> class.
-            /// </summary>
-            public QuandlFuture()
-                : base(valueColumnName: "Settle")
-            {
-            }
-        }
-    }
-}

Algorithm.CSharp/QuandlImporterAlgorithm.cs

@@ -1,70 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using QuantConnect.Data.Custom;
-using QuantConnect.Indicators;
-
-namespace QuantConnect.Algorithm.CSharp
-{
-    /// <summary>
-    /// Using the underlying dynamic data class "Quandl" QuantConnect take care of the data
-    /// importing and definition for you. Simply point QuantConnect to the Quandl Short Code.
-    /// The Quandl object has properties which match the spreadsheet headers.
-    /// If you have multiple quandl streams look at data.Symbol to distinguish them.
-    /// </summary>
-    /// <meta name="tag" content="custom data" />
-    /// <meta name="tag" content="using data" />
-    /// <meta name="tag" content="quandl" />
-    public class QuandlImporterAlgorithm : QCAlgorithm
-    {
-        private SimpleMovingAverage _sma;
-        string _quandlCode = "WIKI/IBM";
-
-        /// Initialize the data and resolution you require for your strategy:
-        public override void Initialize()
-        {
-            //Start and End Date range for the backtest:
-            SetStartDate(2013, 1, 1);
-            SetEndDate(DateTime.Now.Date.AddDays(-1));
-
-            //Cash allocation
-            SetCash(25000);
-
-            // Optional argument - personal token necessary for restricted dataset
-            // Quandl.SetAuthCode("your-quandl-token");
-
-            //Add Generic Quandl Data:
-            AddData<Quandl>(_quandlCode, Resolution.Daily);
-
-            _sma = SMA(_quandlCode, 14);
-        }
-
-        /// Data Event Handler: New data arrives here. "TradeBars" type is a dictionary of strings so you can access it by symbol
-        public void OnData(Quandl data)
-        {
-            if (!Portfolio.HoldStock)
-            {
-                //Order function places trades: enter the string symbol and the quantity you want:
-                SetHoldings(_quandlCode, 1);
-
-                //Debug sends messages to the user console: "Time" is the algorithm time keeper object
-                Debug("Purchased " + _quandlCode + " >> " + Time.ToShortDateString());
-            }
-
-            Plot("SPY", _sma);
-        }
-    }
-}

Algorithm.CSharp/QuantConnect.Algorithm.CSharp.csproj

@@ -35,7 +35,7 @@
     <DebugType>portable</DebugType>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Accord" Version="3.6.0" />
     <PackageReference Include="Accord.Fuzzy" Version="3.6.0" />
     <PackageReference Include="Accord.MachineLearning" Version="3.6.0" />

Algorithm.Framework/QuantConnect.Algorithm.Framework.csproj

@@ -30,7 +30,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Accord" Version="3.6.0" />
     <PackageReference Include="Accord.Math" Version="3.6.0" />
     <PackageReference Include="Accord.Statistics" Version="3.6.0" />

Algorithm.Python/BasicTemplateIndiaAlgorithm.py

@@ -0,0 +1,49 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+### <summary>
+### Basic template framework algorithm uses framework components to define the algorithm.
+### </summary>
+### <meta name="tag" content="using data" />
+### <meta name="tag" content="using quantconnect" />
+### <meta name="tag" content="trading and orders" />
+class BasicTemplateIndiaAlgorithm(QCAlgorithm):
+    '''Basic template framework algorithm uses framework components to define the algorithm.'''
+
+    def Initialize(self):
+        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
+
+        self.SetStartDate(2019, 1, 23)  #Set Start Date
+        self.SetEndDate(2019, 10, 31)    #Set End Date
+        self.SetCash(100000)           #Set Strategy Cash
+        # Find more symbols here: http://quantconnect.com/data
+        self.AddEquity("YESBANK", Resolution.Minute, Market.India)
+        self.Debug("numpy test >>> print numpy.pi: " + str(np.pi))
+
+        # Set Order Prperties as per the requirements for order placement
+        self.DefaultOrderProperties = IndiaOrderProperties(Exchange.NSE)
+
+    def OnData(self, data):
+        '''OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
+
+        Arguments:
+            data: Slice object keyed by symbol containing the stock data
+        '''
+        if not self.Portfolio.Invested:
+            self.SetHoldings("YESBANK", 1)
+
+    def OnOrderEvent(self, orderEvent):
+        if orderEvent.Status == OrderStatus.Filled:
+            self.Debug("Purchased Stock: {0}".format(orderEvent.Symbol))

Algorithm.Python/CustomDataIndicatorExtensionsAlgorithm.py

@@ -12,10 +12,11 @@
 # limitations under the License.
 
 from AlgorithmImports import *
+from HistoryAlgorithm import *
 
 ### <summary>
 ### The algorithm creates new indicator value with the existing indicator method by Indicator Extensions
-### Demonstration of using the external custom datasource Quandl to request the VIX and VXV daily data
+### Demonstration of using the external custom data to request the IBM and SPY daily data
 ### </summary>
 ### <meta name="tag" content="using data" />
 ### <meta name="tag" content="using quantconnect" />
@@ -33,38 +34,30 @@ class CustomDataIndicatorExtensionsAlgorithm(QCAlgorithm):
         self.SetEndDate(2018,1,1)  
         self.SetCash(25000)
         
-        self.vix = 'CBOE/VIX'
-        self.vxv = 'CBOE/VXV'
+        self.ibm = 'IBM'
+        self.spy = 'SPY'
         
         # Define the symbol and "type" of our generic data
-        self.AddData(QuandlVix, self.vix, Resolution.Daily)
-        self.AddData(Quandl, self.vxv, Resolution.Daily)
+        self.AddData(CustomDataEquity, self.ibm, Resolution.Daily)
+        self.AddData(CustomDataEquity, self.spy, Resolution.Daily)
         
         # Set up default Indicators, these are just 'identities' of the closing price
-        self.vix_sma = self.SMA(self.vix, 1, Resolution.Daily)
-        self.vxv_sma = self.SMA(self.vxv, 1, Resolution.Daily)
+        self.ibm_sma = self.SMA(self.ibm, 1, Resolution.Daily)
+        self.spy_sma = self.SMA(self.spy, 1, Resolution.Daily)
         
-        # This will create a new indicator whose value is smaVXV / smaVIX
-        self.ratio = IndicatorExtensions.Over(self.vxv_sma, self.vix_sma)
+        # This will create a new indicator whose value is smaSPY / smaIBM
+        self.ratio = IndicatorExtensions.Over(self.spy_sma, self.ibm_sma)
         
         # Plot indicators each time they update using the PlotIndicator function
         self.PlotIndicator("Ratio", self.ratio)
-        self.PlotIndicator("Data", self.vix_sma, self.vxv_sma)
+        self.PlotIndicator("Data", self.ibm_sma, self.spy_sma)
     
     # OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
     def OnData(self, data):
         
         # Wait for all indicators to fully initialize
-        if not (self.vix_sma.IsReady and self.vxv_sma.IsReady and self.ratio.IsReady): return
+        if not (self.ibm_sma.IsReady and self.spy_sma.IsReady and self.ratio.IsReady): return
         if not self.Portfolio.Invested and self.ratio.Current.Value > 1:
-            self.MarketOrder(self.vix, 100)
+            self.MarketOrder(self.ibm, 100)
         elif self.ratio.Current.Value < 1:
                 self.Liquidate()
-
-# In CBOE/VIX data, there is a "vix close" column instead of "close" which is the 
-# default column namein LEAN Quandl custom data implementation.
-# This class assigns new column name to match the the external datasource setting.
-class QuandlVix(PythonQuandl):
-    
-    def __init__(self):
-        self.ValueColumnName = "VIX Close"

Algorithm.Python/HistoryAlgorithm.py

@@ -30,12 +30,11 @@ class HistoryAlgorithm(QCAlgorithm):
         self.SetCash(100000)           #Set Strategy Cash
         # Find more symbols here: http://quantconnect.com/data
         self.AddEquity("SPY", Resolution.Daily)
-        self.AddData(QuandlFuture,"CHRIS/CME_SP1", Resolution.Daily)
+        self.AddData(CustomDataEquity, "IBM", Resolution.Daily)
         # specifying the exchange will allow the history methods that accept a number of bars to return to work properly
-        self.Securities["CHRIS/CME_SP1"].Exchange = EquityExchange()
 
         # we can get history in initialize to set up indicators and such
-        self.spyDailySma = SimpleMovingAverage(14)
+        self.dailySma = SimpleMovingAverage(14)
 
         # get the last calendar year's worth of SPY data at the configured resolution (daily)
         tradeBarHistory = self.History([self.Securities["SPY"].Symbol], timedelta(365))
@@ -56,56 +55,52 @@ class HistoryAlgorithm(QCAlgorithm):
         # we can loop over the return value from these functions and we get TradeBars
         # we can use these TradeBars to initialize indicators or perform other math
         for index, tradeBar in tradeBarHistory.loc["SPY"].iterrows():
-            self.spyDailySma.Update(index, tradeBar["close"])
+            self.dailySma.Update(index, tradeBar["close"])
 
-        # get the last calendar year's worth of quandl data at the configured resolution (daily)
-        quandlHistory = self.History(QuandlFuture, "CHRIS/CME_SP1", timedelta(365))
-        self.AssertHistoryCount("History(QuandlFuture, \"CHRIS/CME_SP1\", timedelta(365))", quandlHistory, 250)
+        # get the last calendar year's worth of customData data at the configured resolution (daily)
+        customDataHistory = self.History(CustomDataEquity, "IBM", timedelta(365))
+        self.AssertHistoryCount("History(CustomDataEquity, \"IBM\", timedelta(365))", customDataHistory, 10)
 
-        # get the last 14 bars of SPY at the configured resolution (daily)
-        quandlHistory = self.History(QuandlFuture, "CHRIS/CME_SP1", 14)
-        self.AssertHistoryCount("History(QuandlFuture, \"CHRIS/CME_SP1\", 14)", quandlHistory, 14)
+        # get the last 10 bars of IBM at the configured resolution (daily)
+        customDataHistory = self.History(CustomDataEquity, "IBM", 14)
+        self.AssertHistoryCount("History(CustomDataEquity, \"IBM\", 14)", customDataHistory, 10)
 
-        # we can loop over the return values from these functions and we'll get Quandl data
+        # we can loop over the return values from these functions and we'll get Custom data
         # this can be used in much the same way as the tradeBarHistory above
-        self.spyDailySma.Reset()
-        for index, quandl in quandlHistory.loc["CHRIS/CME_SP1"].iterrows():
-            self.spyDailySma.Update(index, quandl["settle"])
+        self.dailySma.Reset()
+        for index, customData in customDataHistory.loc["IBM"].iterrows():
+            self.dailySma.Update(index, customData["value"])
 
-        # get the last year's worth of all configured Quandl data at the configured resolution (daily)
-        #allQuandlData = self.History(QuandlFuture, timedelta(365))
-        #self.AssertHistoryCount("History(QuandlFuture, timedelta(365))", allQuandlData, 250)
+        # get the last 10 bars worth of Custom data for the specified symbols at the configured resolution (daily)
+        allCustomData = self.History(CustomDataEquity, self.Securities.Keys, 14)
+        self.AssertHistoryCount("History(CustomDataEquity, self.Securities.Keys, 14)", allCustomData, 10)
 
-        # get the last 14 bars worth of Quandl data for the specified symbols at the configured resolution (daily)
-        allQuandlData = self.History(QuandlFuture, self.Securities.Keys, 14)
-        self.AssertHistoryCount("History(QuandlFuture, self.Securities.Keys, 14)", allQuandlData, 14)
+        # NOTE: Using different resolutions require that they are properly implemented in your data type. If your
+        #  custom data source has different resolutions, it would need to be implemented in the GetSource and 
+        #  Reader methods properly.
+        #customDataHistory = self.History(CustomDataEquity, "IBM", timedelta(7), Resolution.Minute)
+        #customDataHistory = self.History(CustomDataEquity, "IBM", 14, Resolution.Minute)
+        #allCustomData = self.History(CustomDataEquity, timedelta(365), Resolution.Minute)
+        #allCustomData = self.History(CustomDataEquity, self.Securities.Keys, 14, Resolution.Minute)
+        #allCustomData = self.History(CustomDataEquity, self.Securities.Keys, timedelta(1), Resolution.Minute)
+        #allCustomData = self.History(CustomDataEquity, self.Securities.Keys, 14, Resolution.Minute)
 
-        # NOTE: using different resolutions require that they are properly implemented in your data type, since
-        #  Quandl doesn't support minute data, this won't actually work, but if your custom data source has
-        #  different resolutions, it would need to be implemented in the GetSource and Reader methods properly
-        #quandlHistory = self.History(QuandlFuture, "CHRIS/CME_SP1", timedelta(7), Resolution.Minute)
-        #quandlHistory = self.History(QuandlFuture, "CHRIS/CME_SP1", 14, Resolution.Minute)
-        #allQuandlData = self.History(QuandlFuture, timedelta(365), Resolution.Minute)
-        #allQuandlData = self.History(QuandlFuture, self.Securities.Keys, 14, Resolution.Minute)
-        #allQuandlData = self.History(QuandlFuture, self.Securities.Keys, timedelta(1), Resolution.Minute)
-        #allQuandlData = self.History(QuandlFuture, self.Securities.Keys, 14, Resolution.Minute)
-
-        # get the last calendar year's worth of all quandl data
-        allQuandlData = self.History(QuandlFuture, self.Securities.Keys, timedelta(365))
-        self.AssertHistoryCount("History(QuandlFuture, self.Securities.Keys, timedelta(365))", allQuandlData, 250)
+        # get the last calendar year's worth of all customData data
+        allCustomData = self.History(CustomDataEquity, self.Securities.Keys, timedelta(365))
+        self.AssertHistoryCount("History(CustomDataEquity, self.Securities.Keys, timedelta(365))", allCustomData, 10)
 
         # we can also access the return value from the multiple symbol functions to request a single
         # symbol and then loop over it
-        singleSymbolQuandl = allQuandlData.loc["CHRIS/CME_SP1"]
-        self.AssertHistoryCount("allQuandlData.loc[\"CHRIS/CME_SP1\"]", singleSymbolQuandl, 250)
-        for  quandl in singleSymbolQuandl:
-            # do something with 'CHRIS/CME_SP1.QuandlFuture' quandl data
+        singleSymbolCustom = allCustomData.loc["IBM"]
+        self.AssertHistoryCount("allCustomData.loc[\"IBM\"]", singleSymbolCustom, 10)
+        for  customData in singleSymbolCustom:
+            # do something with 'IBM.CustomDataEquity' customData data
             pass
 
-        quandlSpyLows = allQuandlData.loc["CHRIS/CME_SP1"]["low"]
-        self.AssertHistoryCount("allQuandlData.loc[\"CHRIS/CME_SP1\"][\"low\"]", quandlSpyLows, 250)
-        for  low in quandlSpyLows:
-            # do something with 'CHRIS/CME_SP1.QuandlFuture' quandl data
+        customDataSpyValues = allCustomData.loc["IBM"]["value"]
+        self.AssertHistoryCount("allCustomData.loc[\"IBM\"][\"value\"]", customDataSpyValues, 10)
+        for value in customDataSpyValues:
+            # do something with 'IBM.CustomDataEquity' value data
             pass
 
 
@@ -124,10 +119,20 @@ class HistoryAlgorithm(QCAlgorithm):
             raise Exception("{} expected {}, but received {}".format(methodCall, expected, count))
 
 
-class QuandlFuture(PythonQuandl):
-    '''Custom quandl data type for setting customized value column name. Value column is used for the primary trading calculations and charting.'''
-    def __init__(self):
-        # Define ValueColumnName: cannot be None, Empty or non-existant column name
-        # If ValueColumnName is "Close", do not use PythonQuandl, use Quandl:
-        # self.AddData[QuandlFuture](self.crude, Resolution.Daily)
-        self.ValueColumnName = "Settle"
+class CustomDataEquity(PythonData):
+    def GetSource(self, config, date, isLive):
+        source = "https://www.dl.dropboxusercontent.com/s/o6ili2svndzn556/custom_data.csv?dl=0"
+        return SubscriptionDataSource(source, SubscriptionTransportMedium.RemoteFile)
+
+    def Reader(self, config, line, date, isLive):
+        if line == None:
+            return None
+
+        customData = CustomDataEquity()
+        customData.Symbol = config.Symbol
+
+        csv = line.split(",")
+        customData.Time = datetime.strptime(csv[0], '%Y%m%d %H:%M')
+        customData.EndTime = customData.Time + timedelta(days=1)
+        customData.Value = float(csv[1])
+        return customData

Algorithm.Python/IndiaDataRegressionAlgorithm.py

@@ -0,0 +1,81 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+### <summary>
+### Basic template framework algorithm uses framework components to define the algorithm.
+### </summary>
+### <meta name="tag" content="using data" />
+### <meta name="tag" content="using quantconnect" />
+### <meta name="tag" content="trading and orders" />
+class IndiaDataRegressionAlgorithm(QCAlgorithm):
+    '''Basic template framework algorithm uses framework components to define the algorithm.'''
+
+    def Initialize(self):
+        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
+        self.SetStartDate(2004, 5, 20)
+        self.SetEndDate(2016, 7, 26) 
+        self._mappingSymbol = self.AddEquity("3MINDIA", Resolution.Daily, Market.India).Symbol
+        self._splitAndDividendSymbol = self.AddEquity("CCCL", Resolution.Daily, Market.India).Symbol
+        self._receivedWarningEvent = False
+        self._receivedOccurredEvent = False
+        self._initialMapping = False
+        self._executionMapping = False
+        self.Debug("numpy test >>> print numpy.pi: " + str(np.pi))
+
+    def OnData(self, data):
+        '''OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
+
+        Arguments:
+            data: Slice object keyed by symbol containing the stock data
+        '''
+
+        # dividend
+        if data.Dividends.ContainsKey(self._splitAndDividendSymbol):
+            dividend = data.Dividends[self._splitAndDividendSymbol]
+            if ((self.Time.year == 2010 and self.Time.month == 6 and self.Time.day == 15) and
+                    (dividend.Price != 0.5 or dividend.ReferencePrice != 88.8 or dividend.Distribution != 0.5)):
+                raise Exception("Did not receive expected dividend values")
+
+        # split
+        if data.Splits.ContainsKey(self._splitAndDividendSymbol):
+            split = data.Splits[self._splitAndDividendSymbol]
+            if split.Type == SplitType.Warning:
+                self._receivedWarningEvent = True
+            elif split.Type == SplitType.SplitOccurred:
+                self._receivedOccurredEvent = True
+                if split.Price != 421.0 or split.ReferencePrice != 421.0 or split.SplitFactor != 0.2:
+                    raise Exception("Did not receive expected price values")
+
+        # mapping
+        if data.SymbolChangedEvents.ContainsKey(self._mappingSymbol):
+                mappingEvent = [x.Value for x in data.SymbolChangedEvents if x.Key.SecurityType == 1][0]
+                if self.Time.year == 1999 and self.Time.month == 1 and self.Time.day == 1:
+                    self._initialMapping = True
+                elif self.Time.year == 2004 and self.Time.month == 6 and self.Time.day == 15:
+                    if mappingEvent.NewSymbol == "3MINDIA" and mappingEvent.OldSymbol == "BIRLA3M":
+                        self._executionMapping = True
+    
+    def OnEndOfAlgorithm(self):
+        if self._initialMapping:
+            raise Exception("The ticker generated the initial rename event")
+
+        if not self._executionMapping:
+            raise Exception("The ticker did not rename throughout the course of its life even though it should have")
+        
+        if not self._receivedOccurredEvent:
+            raise Exception("Did not receive expected split event")
+
+        if not self._receivedWarningEvent:
+            raise Exception("Did not receive expected split warning event")

Algorithm.Python/IndicatorSuiteAlgorithm.py

@@ -27,7 +27,7 @@ class IndicatorSuiteAlgorithm(QCAlgorithm):
         '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
 
         self.symbol = "SPY"
-        self.customSymbol = "WIKI/FB"
+        self.customSymbol = "IBM"
         self.price = None
 
         self.SetStartDate(2013, 1, 1)  #Set Start Date
@@ -36,7 +36,7 @@ class IndicatorSuiteAlgorithm(QCAlgorithm):
         # Find more symbols here: http://quantconnect.com/data
 
         self.AddEquity(self.symbol, Resolution.Daily)
-        self.AddData(Quandl, self.customSymbol, Resolution.Daily)
+        self.AddData(CustomData, self.customSymbol, Resolution.Daily)
 
         # Set up default Indicators, these indicators are defined on the Value property of incoming data (except ATR and AROON which use the full TradeBar object)
         self.indicators = {
@@ -88,10 +88,10 @@ class IndicatorSuiteAlgorithm(QCAlgorithm):
         # these are indicators that require multiple inputs. the most common of which is a ratio.
         # suppose we seek the ratio of BTC to SPY, we could write the following:
         spyClose = Identity(self.symbol)
-        fbClose = Identity(self.customSymbol)
+        ibmClose = Identity(self.customSymbol)
 
-        # this will create a new indicator whose value is FB/SPY
-        self.ratio = IndicatorExtensions.Over(fbClose, spyClose)
+        # this will create a new indicator whose value is IBM/SPY
+        self.ratio = IndicatorExtensions.Over(ibmClose, spyClose)
 
         # we can also easily plot our indicators each time they update using th PlotIndicator function
         self.PlotIndicator("Ratio", self.ratio)

Algorithm.Python/QuandlFuturesDataAlgorithm.py

@@ -1,53 +0,0 @@
-# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
-# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from AlgorithmImports import *
-
-### <summary>
-### Futures demonstration algorithm.
-### QuantConnect allows importing generic data sources! This example demonstrates importing a futures
-### data from the popular open data source Quandl. QuantConnect has a special deal with Quandl giving you access
-### to Stevens Continuous Futurs (SCF) for free. If you'd like to download SCF for local backtesting, you can download it through Quandl.com.
-### </summary>
-### <meta name="tag" content="using data" />
-### <meta name="tag" content="quandl" />
-### <meta name="tag" content="custom data" />
-### <meta name="tag" content="futures" />
-class QuandlFuturesDataAlgorithm(QCAlgorithm):
-
-    def Initialize(self):
-        ''' Initialize the data and resolution you require for your strategy '''
-        self.SetStartDate(2000, 1, 1)
-        self.SetEndDate(datetime.now().date() - timedelta(1))
-        self.SetCash(25000)
-
-        # Symbol corresponding to the quandl code
-        self.crude = "SCF/CME_CL1_ON"
-        self.AddData(QuandlFuture, self.crude, Resolution.Daily)
-
-
-    def OnData(self, data):
-        '''Data Event Handler: New data arrives here. "TradeBars" type is a dictionary of strings so you can access it by symbol.'''
-        if self.Portfolio.HoldStock: return
-
-        self.SetHoldings(self.crude, 1)
-        self.Debug(str(self.Time) + str(" Purchased Crude Oil: ") + self.crude)
-
-
-class QuandlFuture(PythonQuandl):
-    '''Custom quandl data type for setting customized value column name. Value column is used for the primary trading calculations and charting.'''
-    def __init__(self):
-        # Define ValueColumnName: cannot be None, Empty or non-existant column name
-        # If ValueColumnName is "Close", do not use PythonQuandl, use Quandl:
-        # self.AddData[QuandlFuture](self.crude, Resolution.Daily)
-        self.ValueColumnName = "Settle"

Algorithm.Python/QuandlImporterAlgorithm.py

@@ -1,51 +0,0 @@
-# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
-# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from AlgorithmImports import *
-
-### <summary>
-### Using the underlying dynamic data class "Quandl" QuantConnect take care of the data
-### importing and definition for you. Simply point QuantConnect to the Quandl Short Code.
-### The Quandl object has properties which match the spreadsheet headers.
-### If you have multiple quandl streams look at data.Symbol to distinguish them.
-### </summary>
-### <meta name="tag" content="custom data" />
-### <meta name="tag" content="using data" />
-### <meta name="tag" content="quandl" />
-class QuandlImporterAlgorithm(QCAlgorithm):
-
-    def Initialize(self):
-        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
-        self.quandlCode = "WIKI/IBM"
-        ## Optional argument - personal token necessary for restricted dataset
-        # Quandl.SetAuthCode("your-quandl-token")
-        self.SetStartDate(2014,4,1)                                 #Set Start Date
-        self.SetEndDate(datetime.today() - timedelta(1))            #Set End Date
-        self.SetCash(25000)                                         #Set Strategy Cash
-        self.AddData(QuandlCustomColumns, self.quandlCode, Resolution.Daily, TimeZones.NewYork)
-        self.sma = self.SMA(self.quandlCode, 14)
-
-    def OnData(self, data):
-        '''OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.'''
-        if not self.Portfolio.HoldStock:
-            self.SetHoldings(self.quandlCode, 1)
-            self.Debug("Purchased {0} >> {1}".format(self.quandlCode, self.Time))
-
-        self.Plot(self.quandlCode, "PriceSMA", self.sma.Current.Value)
-        
-# Quandl often doesn't use close columns so need to tell LEAN which is the "value" column.
-class QuandlCustomColumns(PythonQuandl):
-    '''Custom quandl data type for setting customized value column name. Value column is used for the primary trading calculations and charting.'''
-    def __init__(self):
-        # Define ValueColumnName: cannot be None, Empty or non-existant column name
-        self.ValueColumnName = "adj. close"

Algorithm.Python/QuantConnect.Algorithm.Python.csproj

@@ -37,7 +37,7 @@
     <Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
@@ -222,8 +222,6 @@
     <None Include="OptionSplitRegressionAlgorithm.py" />
     <None Include="OrderTicketDemoAlgorithm.py" />
     <None Include="ParameterizedAlgorithm.py" />
-    <None Include="QuandlFuturesDataAlgorithm.py" />
-    <None Include="QuandlImporterAlgorithm.py" />
     <None Include="readme.md" />
     <None Include="RawPricesCoarseUniverseAlgorithm.py" />
     <None Include="RegressionAlgorithm.py" />

Algorithm/QCAlgorithm.History.cs

@@ -804,9 +804,30 @@ namespace QuantConnect.Algorithm
             Security security;
             if (Securities.TryGetValue(symbol, out security))
             {
-                return resolution ?? SubscriptionManager.SubscriptionDataConfigService
-                    .GetSubscriptionDataConfigs(symbol)
-                    .GetHighestResolution();
+                if (resolution != null)
+                {
+                    return resolution.Value;
+                }
+
+                Resolution? result = null;
+                var hasNonInternal = false;
+                foreach (var config in SubscriptionManager.SubscriptionDataConfigService
+                    .GetSubscriptionDataConfigs(symbol, includeInternalConfigs: true)
+                    // we process non internal configs first
+                    .OrderBy(config => config.IsInternalFeed ? 1 : 0))
+                {
+                    if (!config.IsInternalFeed || !hasNonInternal)
+                    {
+                        // once we find a non internal config we ignore internals
+                        hasNonInternal |= !config.IsInternalFeed;
+                        if (!result.HasValue || config.Resolution < result)
+                        {
+                            result = config.Resolution;
+                        }
+                    }
+                }
+
+                return result ?? UniverseSettings.Resolution;
             }
             else
             {

Algorithm/QCAlgorithm.Universe.cs

@@ -657,27 +657,9 @@ namespace QuantConnect.Algorithm
                 bar.Symbol = security.Symbol;
                 
                 var maxSupportedResolution = bar.SupportedResolutions().Max();
-
                 var updateFrequency = maxSupportedResolution.ToTimeSpan();
-                int periods;
-                switch (maxSupportedResolution)
-                {
-                    case Resolution.Tick:
-                    case Resolution.Second:
-                        periods = 600;
-                        break;
-                    case Resolution.Minute:
-                        periods = 60 * 24;
-                        break;
-                    case Resolution.Hour:
-                        periods = 24 * 30;
-                        break;
-                    default:
-                        periods = 30;
-                        break;
-                }
                 
-                security.VolatilityModel = new StandardDeviationOfReturnsVolatilityModel(periods, maxSupportedResolution, updateFrequency);
+                security.VolatilityModel = new StandardDeviationOfReturnsVolatilityModel(maxSupportedResolution, updateFrequency);
             }
         }
 

Algorithm/QCAlgorithm.cs

@@ -1680,6 +1680,12 @@ namespace QuantConnect.Algorithm
                             contractDepthOffset: contractOffset
                         );
 
+                        // let's add a MHDB entry for the continuous symbol using the associated security
+                        var continuousContractSymbol = ContinuousContractUniverse.CreateSymbol(security.Symbol);
+                        MarketHoursDatabase.SetEntry(continuousContractSymbol.ID.Market,
+                            continuousContractSymbol.ID.Symbol,
+                            continuousContractSymbol.ID.SecurityType,
+                            security.Exchange.Hours);
                         AddUniverse(new ContinuousContractUniverse(security, new UniverseSettings(settings)
                             {
                                 DataMappingMode = continuousConfigs.First().DataMappingMode,
@@ -1687,7 +1693,7 @@ namespace QuantConnect.Algorithm
                                 ContractDepthOffset = (int)continuousConfigs.First().ContractDepthOffset,
                                 SubscriptionDataTypes = dataTypes
                             }, LiveMode,
-                            new SubscriptionDataConfig(canonicalConfig, symbol: ContinuousContractUniverse.CreateSymbol(security.Symbol))));
+                            new SubscriptionDataConfig(canonicalConfig, symbol: continuousContractSymbol)));
 
                         universe = new FuturesChainUniverse((Future)security, settings);
                     }

Algorithm/QuantConnect.Algorithm.csproj

@@ -30,7 +30,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>

AlgorithmFactory/DebuggerHelper.cs

@@ -48,11 +48,18 @@ namespace QuantConnect.AlgorithmFactory
 
             /// <summary>
             ///  Python Tool for Visual Studio Debugger for remote python debugging.
-            /// <see cref="Language.Python"/> will use 'Python Extension in VS Code' 
-            ///or 'Python Tools in Visual Studio'
+            /// <see cref="Language.Python"/>. Deprecated, routes to DebugPy which
+            /// is it's replacement. Used in the same way.
             /// </summary>
             PTVSD,
 
+            /// <summary>
+            ///  DebugPy - a debugger for Python.
+            /// <see cref="Language.Python"/> can use  `Python Extension` in VS Code
+            /// or attach to Python in Visual Studio
+            /// </summary>
+            DebugPy,
+
             /// <summary>
             ///  PyCharm PyDev Debugger for remote python debugging.
             /// <see cref="Language.Python"/> will use 'Python Debug Server' in PyCharm
@@ -68,7 +75,7 @@ namespace QuantConnect.AlgorithmFactory
             if (language == Language.Python)
             {
                 DebuggingMethod debuggingType;
-                Enum.TryParse(Config.Get("debugging-method", DebuggingMethod.LocalCmdline.ToString()), out debuggingType);
+                Enum.TryParse(Config.Get("debugging-method", DebuggingMethod.LocalCmdline.ToString()), true, out debuggingType);
 
                 Log.Trace("DebuggerHelper.Initialize(): initializing python...");
                 PythonInitializer.Initialize();
@@ -91,8 +98,9 @@ while not sys.gettrace():
                             break;
 
                         case DebuggingMethod.PTVSD:
-                            Log.Trace("DebuggerHelper.Initialize(): waiting for PTVSD debugger to attach at localhost:5678...");
-                            PythonEngine.RunSimpleString("import ptvsd; ptvsd.enable_attach(); ptvsd.wait_for_attach()");
+                        case DebuggingMethod.DebugPy:
+                            Log.Trace("DebuggerHelper.Initialize(): debugpy waiting for attach at port 5678...");
+                            PythonEngine.RunSimpleString("import debugpy; debugpy.listen(('0.0.0.0', 5678)); debugpy.wait_for_client()");
                             break;
 
                         case DebuggingMethod.PyCharm:

AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs

@@ -83,7 +83,7 @@ namespace QuantConnect.AlgorithmFactory.Python.Wrappers
                         var repr = attr.Repr().GetStringBetweenChars('\'', '\'');
 
                         if (repr.StartsWith(moduleName) &&                // Must be defined in the module
-                            attr.TryConvert(out type) &&                  // Must be a Type
+                            attr.TryConvert(out type, true) &&                  // Must be a Type
                             typeof(QCAlgorithm).IsAssignableFrom(type))   // Must inherit from QCAlgorithm
                         {
                             Logging.Log.Trace("AlgorithmPythonWrapper(): Creating IAlgorithm instance.");

AlgorithmFactory/QuantConnect.AlgorithmFactory.csproj

@@ -29,7 +29,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

Brokerages/InteractiveBrokers/InteractiveBrokersBrokerage.cs

@@ -55,7 +55,7 @@ namespace QuantConnect.Brokerages.InteractiveBrokers
         /// <summary>
         /// The default gateway version to use
         /// </summary>
-        public static string DefaultVersion { get; } = "985";
+        public static string DefaultVersion { get; } = "1012";
 
         private IBAutomater.IBAutomater _ibAutomater;
 

Common/AlgorithmImports.py

@@ -60,6 +60,7 @@ from QuantConnect.Data.Shortable import *
 from QuantConnect.Orders.Slippage import *
 from QuantConnect.Securities.Forex import *
 from QuantConnect.Data.Fundamental import *
+from QuantConnect.Algorithm.CSharp import *
 from QuantConnect.Securities.Option import *
 from QuantConnect.Securities.Equity import *
 from QuantConnect.Securities.Future import *
@@ -69,6 +70,7 @@ from QuantConnect.Algorithm.Framework import *
 from QuantConnect.Securities.Volatility import *
 from QuantConnect.Securities.Interfaces import *
 from QuantConnect.Data.UniverseSelection import *
+from QuantConnect.Data.Custom.IconicTypes import *
 from QuantConnect.Data.Custom.AlphaStreams import *
 from QuantConnect.Algorithm.Framework.Risk import *
 from QuantConnect.Algorithm.Framework.Alphas import *

Common/Brokerages/BrokerageName.cs

@@ -104,6 +104,11 @@ namespace QuantConnect.Brokerages
         /// <summary>
         /// Transaction and submit/execution rules will use ftx models
         /// </summary>
-        FTX
+        FTX,
+
+        /// <summary>
+        /// Transaction and submit/execution rules will use ftx us models
+        /// </summary>
+        FTXUS
     }
 }

Common/Brokerages/FTXBrokerageModel.cs

@@ -29,6 +29,16 @@ namespace QuantConnect.Brokerages
     {
         private const decimal _defaultLeverage = 3m;
 
+        /// <summary>
+        /// market name
+        /// </summary>
+        protected virtual string MarketName => Market.FTX;
+
+        /// <summary>
+        /// Gets a map of the default markets to be used for each security type
+        /// </summary>
+        public override IReadOnlyDictionary<SecurityType, string> DefaultMarkets { get; } = GetDefaultMarkets(Market.FTX);
+
         /// <summary>
         /// Creates an instance of <see cref="FTXBrokerageModel"/> class
         /// </summary>
@@ -36,13 +46,7 @@ namespace QuantConnect.Brokerages
         public FTXBrokerageModel(AccountType accountType = AccountType.Margin) : base(accountType)
         {
         }
-
-        /// <summary>
-        /// Gets a map of the default markets to be used for each security type
-        /// </summary>
-        public override IReadOnlyDictionary<SecurityType, string> DefaultMarkets { get; } = GetDefaultMarkets();
-
-
+        
         /// <summary>
         /// Gets a new buying power model for the security, returning the default model with the security's configured leverage.
         /// For cash accounts, leverage = 1 is used.
@@ -87,7 +91,7 @@ namespace QuantConnect.Brokerages
         /// <returns>The benchmark for this brokerage</returns>
         public override IBenchmark GetBenchmark(SecurityManager securities)
         {
-            var symbol = Symbol.Create("BTCUSD", SecurityType.Crypto, Market.FTX);
+            var symbol = Symbol.Create("BTCUSD", SecurityType.Crypto, MarketName);
             return SecurityBenchmark.CreateInstance(securities, symbol);
         }
 
@@ -158,7 +162,7 @@ namespace QuantConnect.Brokerages
             if (security.Type != SecurityType.Crypto)
             {
                 message = new BrokerageMessageEvent(BrokerageMessageType.Warning, "NotSupported",
-                    StringExtensions.Invariant($"The {nameof(FTXBrokerageModel)} does not support {security.Type} security type.")
+                    StringExtensions.Invariant($"The {this.GetType().Name} does not support {security.Type} security type.")
                 );
 
                 return false;
@@ -186,10 +190,10 @@ namespace QuantConnect.Brokerages
             return false;
         }
 
-        private static IReadOnlyDictionary<SecurityType, string> GetDefaultMarkets()
+        protected static IReadOnlyDictionary<SecurityType, string> GetDefaultMarkets(string market)
         {
             var map = DefaultMarketMap.ToDictionary();
-            map[SecurityType.Crypto] = Market.FTX;
+            map[SecurityType.Crypto] = market;
             return map.ToReadOnlyDictionary();
         }
     }

Common/Brokerages/FTXUSBrokerageModel.cs

@@ -0,0 +1,53 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.Collections.Generic;
+using QuantConnect.Orders.Fees;
+using QuantConnect.Securities;
+
+namespace QuantConnect.Brokerages
+{
+    /// <summary>
+    /// FTX.US Brokerage model
+    /// </summary>
+    public class FTXUSBrokerageModel : FTXBrokerageModel
+    {
+        /// <summary>
+        /// Market name
+        /// </summary>
+        protected override string MarketName => Market.FTXUS;
+
+        /// <summary>
+        /// Gets a map of the default markets to be used for each security type
+        /// </summary>
+        public override IReadOnlyDictionary<SecurityType, string> DefaultMarkets { get; } = GetDefaultMarkets(Market.FTXUS);
+
+        /// <summary>
+        /// Creates an instance of <see cref="FTXUSBrokerageModel"/> class
+        /// </summary>
+        /// <param name="accountType">Cash or Margin</param>
+        public FTXUSBrokerageModel(AccountType accountType = AccountType.Margin) : base(accountType)
+        {
+        }
+
+        /// <summary>
+        /// Provides FTX.US fee model
+        /// </summary>
+        /// <param name="security">The security to get a fee model for</param>
+        /// <returns>The new fee model for this brokerage</returns>
+        public override IFeeModel GetFeeModel(Security security)
+            => new FTXUSFeeModel();
+    }
+}

Common/Brokerages/IBrokerageModel.cs

@@ -231,6 +231,9 @@ namespace QuantConnect.Brokerages
                 case BrokerageName.FTX:
                     return new FTXBrokerageModel(accountType);
 
+                case BrokerageName.FTXUS:
+                    return new FTXUSBrokerageModel(accountType);
+
                 default:
                     throw new ArgumentOutOfRangeException(nameof(brokerage), brokerage, null);
             }

Common/Currencies.cs

@@ -23,24 +23,34 @@ namespace QuantConnect
     public static class Currencies
     {
         /// <summary>
-        /// USD currency string
+        /// USD (United States Dollar) currency string
         /// </summary>
-        public static string USD = "USD";
+        public const string USD = "USD";
 
         /// <summary>
-        /// EUR currency string
+        /// EUR (Euro) currency string
         /// </summary>
-        public static string EUR = "EUR";
+        public const string EUR = "EUR";
 
         /// <summary>
-        /// GBP currency string
+        /// GBP (British pound sterling) currency string
         /// </summary>
-        public static string GBP = "GBP";
+        public const string GBP = "GBP";
 
         /// <summary>
-        /// INR currency string
+        /// INR (Indian rupee) currency string
         /// </summary>
-        public static string INR = "INR";
+        public const string INR = "INR";
+
+        /// <summary>
+        /// CNH (Chinese Yuan Renminbi) currency string
+        /// </summary>
+        public const string CNH = "CNH";
+
+        /// <summary>
+        /// HKD (Hong Kong dollar) currency string
+        /// </summary>
+        public const string HKD = "HKD";
 
         /// <summary>
         /// Null currency used when a real one is not required
@@ -55,19 +65,19 @@ namespace QuantConnect
         /// </remarks>
         public static readonly IReadOnlyDictionary<string, string> CurrencySymbols = new Dictionary<string, string>
         {
-            {"USD", "$"},
-            {"GBP", "₤"},
+            {USD, "$"},
+            {GBP, "₤"},
             {"JPY", "¥"},
-            {"EUR", "€"},
+            {EUR, "€"},
             {"NZD", "$"},
             {"AUD", "$"},
             {"CAD", "$"},
             {"CHF", "Fr"},
-            {"HKD", "$"},
+            {HKD, "$"},
             {"SGD", "$"},
             {"XAG", "Ag"},
             {"XAU", "Au"},
-            {"CNH", "¥"},
+            {CNH, "¥"},
             {"CNY", "¥"},
             {"CZK", "Kč"},
             {"DKK", "kr"},

Common/Data/Custom/Quandl.cs

@@ -1,156 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using System.Collections.Generic;
-using System.Globalization;
-using System.Net;
-
-namespace QuantConnect.Data.Custom
-{
-    /// <summary>
-    /// Quandl Data Type - Import generic data from quandl, without needing to define Reader methods.
-    /// This reads the headers of the data imported, and dynamically creates properties for the imported data.
-    /// </summary>
-    public class Quandl : DynamicData
-    {
-        private bool _isInitialized;
-        private readonly List<string> _propertyNames = new List<string>();
-        private readonly string _valueColumn;
-        private static string _authCode = "";
-
-        /// <summary>
-        /// Static constructor for the <see cref="Quandl"/> class
-        /// </summary>
-        static Quandl()
-        {
-            // The Quandl API now requires TLS 1.2 for API requests (since 9/18/2018).
-            // NET 4.5.2 and below does not enable this more secure protocol by default, so we add it in here
-            ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
-        }
-
-        /// <summary>
-        /// Flag indicating whether or not the Quanl auth code has been set yet
-        /// </summary>
-        public static bool IsAuthCodeSet
-        {
-            get;
-            private set;
-        }
-
-        /// <summary>
-        /// The end time of this data. Some data covers spans (trade bars) and as such we want
-        /// to know the entire time span covered
-        /// </summary>
-        public override DateTime EndTime
-        {
-            get { return Time + Period; }
-            set { Time = value - Period; }
-        }
-
-        /// <summary>
-        /// Gets a time span of one day
-        /// </summary>
-        public TimeSpan Period
-        {
-            get { return QuantConnect.Time.OneDay; }
-        }
-
-        /// <summary>
-        /// Default quandl constructor uses Close as its value column
-        /// </summary>
-        public Quandl() : this("Close")
-        {
-        }
-
-        /// <summary>
-        /// Constructor for creating customized quandl instance which doesn't use "Close" as its value item.
-        /// </summary>
-        /// <param name="valueColumnName"></param>
-        protected Quandl(string valueColumnName)
-        {
-            _valueColumn = valueColumnName;
-        }
-
-        /// <summary>
-        /// Generic Reader Implementation for Quandl Data.
-        /// </summary>
-        /// <param name="config">Subscription configuration</param>
-        /// <param name="line">CSV line of data from the souce</param>
-        /// <param name="date">Date of the requested line</param>
-        /// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
-        /// <returns></returns>
-        public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
-        {
-            // be sure to instantiate the correct type
-            var data = (Quandl) Activator.CreateInstance(GetType());
-            data.Symbol = config.Symbol;
-            var csv = line.Split(',');
-
-            if (!_isInitialized)
-            {
-                _isInitialized = true;
-                foreach (var propertyName in csv)
-                {
-                    var property = propertyName.Trim();
-                    // should we remove property names like Time?
-                    // do we need to alias the Time??
-                    data.SetProperty(property, 0m);
-                    _propertyNames.Add(property);
-                }
-                // Returns null at this point where we are only reading the properties names
-                return null;
-            }
-
-            data.Time = DateTime.ParseExact(csv[0], "yyyy-MM-dd", CultureInfo.InvariantCulture);
-
-            for (var i = 1; i < csv.Length; i++)
-            {
-                var value = csv[i].ToDecimal();
-                data.SetProperty(_propertyNames[i], value);
-            }
-
-            // we know that there is a close property, we want to set that to 'Value'
-            data.Value = (decimal)data.GetProperty(_valueColumn);
-
-            return data;
-        }
-
-        /// <summary>
-        /// Quandl Source Locator: Using the Quandl V1 API automatically set the URL for the dataset.
-        /// </summary>
-        /// <param name="config">Subscription configuration object</param>
-        /// <param name="date">Date of the data file we're looking for</param>
-        /// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
-        /// <returns>STRING API Url for Quandl.</returns>
-        public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
-        {
-            var source = $"https://www.quandl.com/api/v3/datasets/{config.Symbol.Value}.csv?order=asc&api_key={_authCode}";
-            return new SubscriptionDataSource(source, SubscriptionTransportMedium.RemoteFile);
-        }
-
-        /// <summary>
-        /// Set the auth code for the quandl set to the QuantConnect auth code.
-        /// </summary>
-        /// <param name="authCode"></param>
-        public static void SetAuthCode(string authCode)
-        {
-            if (string.IsNullOrWhiteSpace(authCode)) return;
-
-            _authCode = authCode;
-            IsAuthCodeSet = true;
-        }
-    }
-}

Common/Data/DiskDataCacheProvider.cs

@@ -16,9 +16,11 @@
 
 using System.IO;
 using Ionic.Zip;
-using QuantConnect.Interfaces;
-using QuantConnect.Logging;
+using System.Linq;
 using QuantConnect.Util;
+using QuantConnect.Logging;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
 
 namespace QuantConnect.Data
 {
@@ -93,6 +95,15 @@ namespace QuantConnect.Data
             Compression.ZipCreateAppendData(filePath, entryName, data, true);
         }
 
+        /// <summary>
+        /// Returns a list of zip entries in a provided zip file
+        /// </summary>
+        public List<string> GetZipEntries(string zipFile)
+        {
+            using var stream = new FileStream(zipFile, FileMode.Open, FileAccess.Read);
+            return Compression.GetZipEntryFileNames(stream).ToList();
+        }
+
         /// <summary>
         /// Dispose for this class
         /// </summary>

Common/Data/Market/OptionContract.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -13,8 +13,10 @@
  * limitations under the License.
 */
 
-using System;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
 using QuantConnect.Securities.Option;
+using System;
 
 namespace QuantConnect.Data.Market
 {
@@ -23,8 +25,7 @@ namespace QuantConnect.Data.Market
     /// </summary>
     public class OptionContract
     {
-        private Lazy<OptionPriceModelResult> _optionPriceModelResult = new Lazy<OptionPriceModelResult>(() => 
-                                                                            new OptionPriceModelResult(0m, new Greeks())); 
+        private Lazy<OptionPriceModelResult> _optionPriceModelResult = new(() => OptionPriceModelResult.None);
 
         /// <summary>
         /// Gets the option contract's symbol
@@ -176,5 +177,40 @@ namespace QuantConnect.Data.Market
         /// A string that represents the current object.
         /// </returns>
         public override string ToString() => Symbol.Value;
+
+        /// <summary>
+        /// Creates a <see cref="OptionContract"/> 
+        /// </summary>
+        /// <param name="baseData"></param>
+        /// <param name="security">provides price properties for a <see cref="Security"/></param>
+        /// <param name="underlyingLastPrice">last price the underlying security traded at</param>
+        /// <returns>Option contract</returns>
+        public static OptionContract Create(BaseData baseData, ISecurityPrice security, decimal underlyingLastPrice)
+            => Create(baseData.Symbol, baseData.Symbol.Underlying, baseData.EndTime, security, underlyingLastPrice);
+
+        /// <summary>
+        /// Creates a <see cref="OptionContract"/>
+        /// </summary>
+        /// <param name="symbol">The option contract symbol</param>
+        /// <param name="underlyingSymbol">The symbol of the underlying security</param>
+        /// <param name="endTime">local date time this contract's data was last updated</param>
+        /// <param name="security">provides price properties for a <see cref="Security"/></param>
+        /// <param name="underlyingLastPrice">last price the underlying security traded at</param>
+        /// <returns>Option contract</returns>
+        public static OptionContract Create(Symbol symbol, Symbol underlyingSymbol, DateTime endTime, ISecurityPrice security, decimal underlyingLastPrice)
+        {
+            return new OptionContract(symbol, underlyingSymbol)
+            {
+                Time = endTime,
+                LastPrice = security.Close,
+                Volume = (long)security.Volume,
+                BidPrice = security.BidPrice,
+                BidSize = (long)security.BidSize,
+                AskPrice = security.AskPrice,
+                AskSize = (long)security.AskSize,
+                OpenInterest = security.OpenInterest,
+                UnderlyingLastPrice = underlyingLastPrice
+            };
+        }
     }
 }

Common/Data/Slice.cs

@@ -18,7 +18,7 @@ using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using System.Reflection;
-using QuantConnect.Data.Custom;
+using QuantConnect.Data.Custom.IconicTypes;
 using QuantConnect.Data.Market;
 using QuantConnect.Python;
 
@@ -44,7 +44,7 @@ namespace QuantConnect.Data
         // string -> data   for non-tick data
         // string -> list{data} for tick data
         private readonly Lazy<DataDictionary<SymbolData>> _data;
-        // Quandl -> DataDictonary<Quandl>
+        // UnlinkedData -> DataDictonary<UnlinkedData>
         private Dictionary<Type, object> _dataByType;
 
         /// <summary>
@@ -284,7 +284,7 @@ namespace QuantConnect.Data
         /// Gets the data corresponding to the specified symbol. If the requested data
         /// is of <see cref="MarketDataType.Tick"/>, then a <see cref="List{Tick}"/> will
         /// be returned, otherwise, it will be the subscribed type, for example, <see cref="TradeBar"/>
-        /// or event <see cref="Quandl"/> for custom data.
+        /// or event <see cref="UnlinkedData"/> for custom data.
         /// </summary>
         /// <param name="symbol">The data's symbols</param>
         /// <returns>The data for the specified symbol</returns>
@@ -304,7 +304,7 @@ namespace QuantConnect.Data
         /// <summary>
         /// Gets the <see cref="DataDictionary{T}"/> for all data of the specified type
         /// </summary>
-        /// <typeparam name="T">The type of data we want, for example, <see cref="TradeBar"/> or <see cref="Quandl"/>, ect...</typeparam>
+        /// <typeparam name="T">The type of data we want, for example, <see cref="TradeBar"/> or <see cref="UnlinkedData"/>, ect...</typeparam>
         /// <returns>The <see cref="DataDictionary{T}"/> containing the data of the specified type</returns>
         public DataDictionary<T> Get<T>()
             where T : IBaseData

Common/Extensions.cs

@@ -2505,11 +2505,25 @@ namespace QuantConnect
             {
                 try
                 {
+                    // We must first check if allowPythonDerivative is true to then only return true
+                    // when the PyObject is assignable from Type or IEnumerable and is a C# type
+                    // wrapped in PyObject
+                    if (allowPythonDerivative)
+                    {
+                        result = (T)pyObject.AsManagedObject(type);
+                        return true;
+                    }
+
                     // Special case: Type
                     if (typeof(Type).IsAssignableFrom(type))
                     {
                         result = (T)pyObject.AsManagedObject(type);
-                        return true;
+                        // pyObject is a C# object wrapped in PyObject, in this case return true
+                        // Otherwise, pyObject is a python object that subclass a C# class, only return true if 'allowPythonDerivative'
+                        var castedResult = (Type)pyObject.AsManagedObject(type);
+                        var pythonName = pyObject.GetAttr("__name__").GetAndDispose<string>();
+
+                        return pythonName == castedResult.Name;
                     }
 
                     // Special case: IEnumerable
@@ -2535,7 +2549,7 @@ namespace QuantConnect
                     // Otherwise, pyObject is a python object that subclass a C# class, only return true if 'allowPythonDerivative'
                     var name = (((dynamic) pythonType).__name__ as PyObject).GetAndDispose<string>();
                     pythonType.Dispose();
-                    return allowPythonDerivative || name == result.GetType().Name;
+                    return name == result.GetType().Name;
                 }
                 catch
                 {
@@ -2835,9 +2849,7 @@ namespace QuantConnect
         public static Type CreateType(this PyObject pyObject)
         {
             Type type;
-            if (pyObject.TryConvert(out type) &&
-                type != typeof(PythonQuandl) &&
-                type != typeof(PythonData))
+            if (pyObject.TryConvert(out type))
             {
                 return type;
             }

Common/Interfaces/IDataCacheProvider.cs

@@ -16,6 +16,7 @@
 
 using System;
 using System.IO;
+using System.Collections.Generic;
 
 namespace QuantConnect.Interfaces
 {
@@ -42,5 +43,10 @@ namespace QuantConnect.Interfaces
         /// <param name="key">The source of the data, used as a key to retrieve data in the cache</param>
         /// <param name="data">The data to cache as a byte array</param>
         void Store(string key, byte[] data);
+
+        /// <summary>
+        /// Returns a list of zip entries in a provided zip file
+        /// </summary>
+        List<string> GetZipEntries(string zipFile);
     }
 }

Common/Market.cs

@@ -62,7 +62,8 @@ namespace QuantConnect
             Tuple.Create(HKFE, 25),
 
             Tuple.Create(CFE, 33),
-            Tuple.Create(FTX, 34)
+            Tuple.Create(FTX, 34),
+            Tuple.Create(FTXUS, 35)
         };
 
         static Market()
@@ -212,6 +213,11 @@ namespace QuantConnect
         /// </summary>
         public const string FTX = "ftx";
 
+        /// <summary>
+        /// FTX.US
+        /// </summary>
+        public const string FTXUS = "ftxus";
+
 
         /// <summary>
         /// Adds the specified market to the map of available markets with the specified identifier.
@@ -279,5 +285,13 @@ namespace QuantConnect
         {
             return !ReverseMarkets.TryGetValue(code, out var market) ? null : market;
         }
+
+        /// <summary>
+        /// Returns a list of the supported markets
+        /// </summary>
+        public static List<string> SupportedMarkets()
+        {
+            return Markets.Keys.ToList();
+        }
     }
 }

Common/Orders/Fees/FTXFeeModel.cs

@@ -14,7 +14,6 @@
 */
 
 using QuantConnect.Securities;
-using QuantConnect.Securities.Crypto;
 
 namespace QuantConnect.Orders.Fees
 {
@@ -27,11 +26,12 @@ namespace QuantConnect.Orders.Fees
         /// <summary>
         /// Tier 1 maker fees
         /// </summary>
-        public const decimal MakerFee = 0.0002m;
+        public virtual decimal MakerFee => 0.0002m;
+        
         /// <summary>
         /// Tier 1 taker fees
         /// </summary>
-        public const decimal TakerFee = 0.0007m;
+        public virtual decimal TakerFee => 0.0007m;
 
         /// <summary>
         /// Get the fee for this order in quote currency

Common/Orders/Fees/FTXUSFeeModel.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -13,33 +13,22 @@
  * limitations under the License.
 */
 
-using QuantConnect.Data;
-using QuantConnect.Data.Custom;
-using System;
-using System.Collections.Generic;
-
-namespace QuantConnect.Python
+namespace QuantConnect.Orders.Fees
 {
     /// <summary>
-    /// Dynamic data class for Python algorithms.
+    /// Provides an implementation of <see cref="FeeModel"/> that models FTX order fees
+    /// https://help.ftx.us/hc/en-us/articles/360043579273-Fees
     /// </summary>
-    public class PythonQuandl : Quandl
+    public class FTXUSFeeModel : FTXFeeModel
     {
         /// <summary>
-        /// Constructor for initialising the PythonQuandl class
+        /// Tier 1 maker fees
         /// </summary>
-        public PythonQuandl() : base("Close")
-        {
-            //Empty constructor required for fast-reflection initialization
-        }
-
+        public override decimal MakerFee => 0.001m;
+        
         /// <summary>
-        /// Constructor for creating customized quandl instance which doesn't use "Close" as its value item.
+        /// Tier 1 taker fees
         /// </summary>
-        /// <param name="valueColumnName"></param>
-        public PythonQuandl(string valueColumnName) : base(valueColumnName)
-        {
-            //
-        }
+        public override decimal TakerFee => 0.004m;
     }
-}
+}

Common/Orders/Fees/InteractiveBrokersFeeModel.cs

@@ -33,9 +33,16 @@ namespace QuantConnect.Orders.Fees
         private readonly Dictionary<string, Func<decimal, decimal, CashAmount>> _optionFee =
             new Dictionary<string, Func<decimal, decimal, CashAmount>>();
 
-        private readonly Dictionary<string, CashAmount> _futureFee =
+        /// <summary>
+        /// Reference at https://www.interactivebrokers.com/en/index.php?f=commission&p=futures1
+        /// </summary>
+        private readonly Dictionary<string, Func<Security, CashAmount>> _futureFee =
             //                                                               IB fee + exchange fee
-            new Dictionary<string, CashAmount> { { Market.USA, new CashAmount(0.85m + 1, "USD") } };
+            new()
+            {
+                { Market.USA, UnitedStatesFutureFees },
+                { Market.HKFE, HongKongFutureFees }
+            };
 
         /// <summary>
         /// Initializes a new instance of the <see cref="ImmediateFillModel"/>
@@ -109,17 +116,18 @@ namespace QuantConnect.Orders.Fees
                     if (market == Market.Globex || market == Market.NYMEX
                         || market == Market.CBOT || market == Market.ICE
                         || market == Market.CFE || market == Market.COMEX
-                        || market == Market.CME || market == Market.HKFE)
+                        || market == Market.CME)
                     {
                         // just in case...
                         market = Market.USA;
                     }
 
-                    CashAmount feeRatePerContract;
-                    if (!_futureFee.TryGetValue(market, out feeRatePerContract))
+                    if (!_futureFee.TryGetValue(market, out var feeRatePerContractFunc))
                     {
                         throw new KeyNotFoundException($"InteractiveBrokersFeeModel(): unexpected future Market {market}");
                     }
+
+                    var feeRatePerContract = feeRatePerContractFunc(security);
                     feeResult = order.AbsoluteQuantity * feeRatePerContract.Amount;
                     feeCurrency = feeRatePerContract.Currency;
                     break;
@@ -129,7 +137,7 @@ namespace QuantConnect.Orders.Fees
                     switch (market)
                     {
                         case Market.USA:
-                            equityFee = new EquityFee("USD", feePerShare: 0.005m, minimumFee: 1, maximumFeeRate: 0.005m);
+                            equityFee = new EquityFee(Currencies.USD, feePerShare: 0.005m, minimumFee: 1, maximumFeeRate: 0.005m);
                             break;
                         default:
                             throw new KeyNotFoundException($"InteractiveBrokersFeeModel(): unexpected equity Market {market}");
@@ -235,6 +243,42 @@ namespace QuantConnect.Orders.Fees
             }
         }
 
+        private static CashAmount UnitedStatesFutureFees(Security security)
+        {
+            return new CashAmount(0.85m + 1, Currencies.USD);
+        }
+
+        /// <summary>
+        /// See https://www.hkex.com.hk/Services/Rules-and-Forms-and-Fees/Fees/Listed-Derivatives/Trading/Transaction?sc_lang=en
+        /// </summary>
+        private static CashAmount HongKongFutureFees(Security security)
+        {
+            if (security.Symbol.ID.Symbol.Equals("HSI", StringComparison.InvariantCultureIgnoreCase))
+            {
+                // IB fee + exchange fee
+                return new CashAmount(30 + 10, Currencies.HKD);
+            }
+
+            decimal ibFeePerContract;
+            switch (security.QuoteCurrency.Symbol)
+            {
+                case Currencies.CNH:
+                    ibFeePerContract = 13;
+                    break;
+                case Currencies.HKD:
+                    ibFeePerContract = 20;
+                    break;
+                case Currencies.USD:
+                    ibFeePerContract = 2.40m;
+                    break;
+                default:
+                    throw new ArgumentException($"Unexpected quote currency {security.QuoteCurrency.Symbol} for Hong Kong futures exchange");
+            }
+
+            // let's add a 50% extra charge for exchange fees
+            return new CashAmount(ibFeePerContract * 1.5m, security.QuoteCurrency.Symbol);
+        }
+
         /// <summary>
         /// Helper class to handle IB Equity fees
         /// </summary>

Common/Python/PythonActivator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -42,40 +42,13 @@ namespace QuantConnect.Python
         {
             Type = type;
 
-            var isPythonQuandl = false;
-
-            using (Py.GIL())
+            Factory = x =>
             {
-                var pythonType = value.Invoke().GetPythonType();
-                isPythonQuandl = pythonType.As<Type>() == typeof(PythonQuandl);
-                pythonType.Dispose();
-            }
-
-            if (isPythonQuandl)
-            {
-                Factory = x =>
+                using (Py.GIL())
                 {
-                    using (Py.GIL())
-                    {
-                        var instance = value.Invoke();
-                        var pyValueColumnName = instance.GetAttr("ValueColumnName");
-                        var valueColumnName = pyValueColumnName.ToString();
-                        instance.Dispose();
-                        pyValueColumnName.Dispose();
-                        return new PythonQuandl(valueColumnName);
-                    }
-                };
-            }
-            else
-            {
-                Factory = x =>
-                {
-                    using (Py.GIL())
-                    {
-                        var instance = value.Invoke();
-                        return new PythonData(instance);
-                    }
-                };
+                    var instance = value.Invoke();
+                    return new PythonData(instance);
+                }
             };
         }
     }

Common/Python/PythonSlice.cs

@@ -15,7 +15,7 @@
 
 using Python.Runtime;
 using QuantConnect.Data;
-using QuantConnect.Data.Custom;
+using QuantConnect.Data.Custom.IconicTypes;
 using QuantConnect.Data.Market;
 using System.Collections.Generic;
 
@@ -111,7 +111,7 @@ namespace QuantConnect.Python
         /// Gets the data corresponding to the specified symbol. If the requested data
         /// is of <see cref="MarketDataType.Tick"/>, then a <see cref="List{Tick}"/> will
         /// be returned, otherwise, it will be the subscribed type, for example, <see cref="TradeBar"/>
-        /// or event <see cref="Quandl"/> for custom data.
+        /// or event <see cref="UnlinkedData"/> for custom data.
         /// </summary>
         /// <param name="symbol">The data's symbols</param>
         /// <returns>The data for the specified symbol</returns>

Common/QuantConnect.csproj

@@ -35,7 +35,7 @@
     <Message Text="SelectedOptimization $(SelectedOptimization)" Importance="high" />
   </Target>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="CloneExtensions" Version="1.3.0" />
     <PackageReference Include="fasterflect" Version="3.0.0" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />

Common/Securities/Option/ConstantQLDividendYieldEstimator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -13,22 +13,15 @@
  * limitations under the License.
 */
 
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
-using QLNet;
-using QuantConnect.Util;
 
 namespace QuantConnect.Securities.Option
 {
     /// <summary>
     /// Class implements default flat dividend yield curve estimator, implementing <see cref="IQLDividendYieldEstimator"/>.  
     /// </summary>
-    class ConstantQLDividendYieldEstimator : IQLDividendYieldEstimator
+    public class ConstantQLDividendYieldEstimator : IQLDividendYieldEstimator
     {
         private readonly double _dividendYield;
         /// <summary>

Common/Securities/Option/ConstantQLRiskFreeRateEstimator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -14,14 +14,8 @@
 */
 
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
-using QuantConnect.Util;
-using QLNet;
 
 namespace QuantConnect.Securities.Option
 {
@@ -30,12 +24,12 @@ namespace QuantConnect.Securities.Option
     /// </summary>
     public class ConstantQLRiskFreeRateEstimator : IQLRiskFreeRateEstimator
     {
-        private readonly double _riskFreeRate;
+        private readonly decimal _riskFreeRate;
         /// <summary>
         /// Constructor initializes class with risk free rate constant
         /// </summary>
         /// <param name="riskFreeRate"></param>
-        public ConstantQLRiskFreeRateEstimator(double riskFreeRate = 0.01)
+        public ConstantQLRiskFreeRateEstimator(decimal riskFreeRate = 0.01m)
         {
             _riskFreeRate = riskFreeRate;
         }
@@ -49,8 +43,6 @@ namespace QuantConnect.Securities.Option
         /// <param name="contract">The option contract to evaluate</param>
         /// <returns>The estimate</returns>
         public double Estimate(Security security, Slice slice, OptionContract contract)
-        {
-            return _riskFreeRate;
-        }
+            => Convert.ToDouble(_riskFreeRate);
     }
 }

Common/Securities/Option/ConstantQLUnderlyingVolatilityEstimator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -13,15 +13,8 @@
  * limitations under the License.
 */
 
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
-using QuantConnect.Util;
-using QLNet;
 
 namespace QuantConnect.Securities.Option
 {
@@ -29,8 +22,13 @@ namespace QuantConnect.Securities.Option
     /// Class implements default underlying constant volatility estimator (<see cref="IQLUnderlyingVolatilityEstimator"/>.), that projects the underlying own volatility 
     /// model into corresponding option pricing model.
     /// </summary>
-    class ConstantQLUnderlyingVolatilityEstimator : IQLUnderlyingVolatilityEstimator
+    public class ConstantQLUnderlyingVolatilityEstimator : IQLUnderlyingVolatilityEstimator
     {
+        /// <summary>
+        /// Indicates whether volatility model has been warmed ot not
+        /// </summary>
+        public bool IsReady { get; private set; }
+
         /// <summary>
         /// Returns current estimate of the underlying volatility
         /// </summary>
@@ -48,6 +46,7 @@ namespace QuantConnect.Securities.Option
                 option.Underlying.VolatilityModel != null &&
                 option.Underlying.VolatilityModel.Volatility > 0m)
             {
+                IsReady = true;
                 return (double)option.Underlying.VolatilityModel.Volatility;
             }
 

Common/Securities/Option/IQLDividendYieldEstimator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -15,12 +15,6 @@
 
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
-using QLNet;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace QuantConnect.Securities.Option
 {
@@ -28,7 +22,7 @@ namespace QuantConnect.Securities.Option
     /// Defines QuantLib dividend yield estimator for option pricing model. User may define his own estimators, 
     /// including those forward and backward looking ones.
     /// </summary>
-    interface IQLDividendYieldEstimator
+    public interface IQLDividendYieldEstimator
     {
         /// <summary>
         /// Returns current estimate of the stock dividend yield

Common/Securities/Option/IQLRiskFreeRateEstimator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -15,19 +15,13 @@
 
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
-using QLNet;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace QuantConnect.Securities.Option
 {
     /// <summary>
     /// Defines QuantLib risk free rate estimator for option pricing model. 
     /// </summary>
-    interface IQLRiskFreeRateEstimator
+    public interface IQLRiskFreeRateEstimator
     {
         /// <summary>
         /// Returns current estimate of the risk free rate

Common/Securities/Option/IQLUnderlyingVolatilityEstimator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -15,12 +15,6 @@
 
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
-using QLNet;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace QuantConnect.Securities.Option
 {
@@ -28,7 +22,7 @@ namespace QuantConnect.Securities.Option
     /// Defines QuantLib underlying volatility estimator for option pricing model. User may define his own estimators, 
     /// including those forward and backward looking ones.
     /// </summary>
-    interface IQLUnderlyingVolatilityEstimator
+    public interface IQLUnderlyingVolatilityEstimator
     {
         /// <summary>
         /// Returns current estimate of the underlying volatility
@@ -39,5 +33,10 @@ namespace QuantConnect.Securities.Option
         /// <param name="contract">The option contract to evaluate</param>
         /// <returns>Volatility</returns>
         double Estimate(Security security, Slice slice, OptionContract contract);
+
+        /// <summary>
+        /// Indicates whether volatility model is warmed up or no
+        /// </summary>
+        bool IsReady { get; }
     }
 }

Common/Securities/Option/OptionPriceModelResult.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -23,6 +23,11 @@ namespace QuantConnect.Securities.Option
     /// </summary>
     public class OptionPriceModelResult
     {
+        /// <summary>
+        /// Represents the zero option price and greeks.
+        /// </summary>
+        public static OptionPriceModelResult None { get; } = new(0, new Greeks());
+
         private readonly Lazy<Greeks> _greeks;
         private readonly Lazy<decimal> _impliedVolatility;
 
@@ -81,4 +86,4 @@ namespace QuantConnect.Securities.Option
             _greeks = new Lazy<Greeks>(greeks);
         }
     }
-}
+}

Common/Securities/Option/OptionPriceModels.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -13,12 +13,11 @@
  * limitations under the License.
 */
 
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using QLNet;
+using System;
+using System.Globalization;
+using System.Linq;
+using Fasterflect;
 
 namespace QuantConnect.Securities.Option
 {
@@ -43,7 +42,27 @@ namespace QuantConnect.Securities.Option
         private const int _timeStepsFD = 100;
 
         /// <summary>
-        /// Pricing engine for European vanilla options using analytical formulae. 
+        /// Creates pricing engine by engine type name. 
+        /// </summary>
+        /// <param name="priceEngineName">QL price engine name</param>
+        /// <param name="riskFree">The risk free rate</param>
+        /// <returns>New option price model instance of specific engine</returns>
+        public static IOptionPriceModel Create(string priceEngineName, decimal riskFree)
+        {
+            var type = AppDomain.CurrentDomain.GetAssemblies()
+                .Where(a => !a.IsDynamic)
+                .SelectMany(a => a.GetTypes())
+                .Where(s => s.Implements(typeof(IPricingEngine)))
+                .FirstOrDefault(t => t.FullName?.EndsWith(priceEngineName, StringComparison.InvariantCulture) == true);
+
+            return new QLOptionPriceModel(process => (IPricingEngine)Activator.CreateInstance(type, process),
+                _underlyingVolEstimator,
+                new ConstantQLRiskFreeRateEstimator(riskFree),
+                _dividendYieldEstimator);
+        }
+
+        /// <summary>
+        /// Pricing engine for European vanilla options using analytical formula. 
         /// QuantLib reference: http://quantlib.org/reference/class_quant_lib_1_1_analytic_european_engine.html
         /// </summary>
         /// <returns>New option price model instance</returns>
@@ -103,7 +122,7 @@ namespace QuantConnect.Securities.Option
         {
             PricingEngineFuncEx pricingEngineFunc = (symbol, process) =>
                             symbol.ID.OptionStyle == OptionStyle.American ?
-                            new FDAmericanEngine(process, _timeStepsFD, _timeStepsFD - 1) as IPricingEngine:
+                            new FDAmericanEngine(process, _timeStepsFD, _timeStepsFD - 1) as IPricingEngine :
                             new FDEuropeanEngine(process, _timeStepsFD, _timeStepsFD - 1) as IPricingEngine;
 
             return new QLOptionPriceModel(pricingEngineFunc,

Common/Securities/Option/QLOptionPriceModel.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -28,7 +28,7 @@ namespace QuantConnect.Securities.Option
     /// <summary>
     /// Provides QuantLib(QL) implementation of <see cref="IOptionPriceModel"/> to support major option pricing models, available in QL.
     /// </summary>
-    class QLOptionPriceModel : IOptionPriceModel
+    public class QLOptionPriceModel : IOptionPriceModel
     {
         private readonly IQLUnderlyingVolatilityEstimator _underlyingVolEstimator;
         private readonly IQLRiskFreeRateEstimator _riskFreeRateEstimator;
@@ -41,6 +41,11 @@ namespace QuantConnect.Securities.Option
         /// </summary>
         public bool EnableGreekApproximation { get; set; } = true;
 
+        /// <summary>
+        /// True if volatility model is warmed up, i.e. has generated volatility value different from zero, otherwise false.
+        /// </summary>
+        public bool VolatilityEstimatorWarmedUp => _underlyingVolEstimator.IsReady;
+
         /// <summary>
         /// Method constructs QuantLib option price model with necessary estimators of underlying volatility, risk free rate, and underlying dividend yield
         /// </summary>
@@ -83,6 +88,12 @@ namespace QuantConnect.Securities.Option
         {
             try
             {
+                // expired options has no price
+                if (contract.Time > contract.Expiry)
+                {
+                    return OptionPriceModelResult.None;
+                }
+
                 // setting up option pricing parameters
                 var calendar = new UnitedStates();
                 var dayCounter = new Actual365Fixed();
@@ -101,6 +112,11 @@ namespace QuantConnect.Securities.Option
                 var underlyingVolValue = new SimpleQuote(_underlyingVolEstimator.Estimate(security, slice, contract));
                 var underlyingVol = new Handle<BlackVolTermStructure>(new BlackConstantVol(0, calendar, new Handle<Quote>(underlyingVolValue), dayCounter));
 
+                if (!_underlyingVolEstimator.IsReady)
+                {
+                    return OptionPriceModelResult.None;
+                }
+
                 // preparing stochastic process and payoff functions
                 var stochasticProcess = new BlackScholesMertonProcess(new Handle<Quote>(underlyingQuoteValue), dividendYield, riskFreeRate, underlyingVol);
                 var payoff = new PlainVanillaPayoff(contract.Right == OptionRight.Call ? QLNet.Option.Type.Call : QLNet.Option.Type.Put, (double)contract.Strike);
@@ -116,7 +132,8 @@ namespace QuantConnect.Securities.Option
                 option.setPricingEngine(_pricingEngineFunc(contract.Symbol, stochasticProcess));
 
                 // running calculations
-                var npv = EvaluateOption(option);
+                // can return negative value in neighbourhood of 0
+                var npv = Math.Max(0, EvaluateOption(option));
 
                 // function extracts QL greeks catching exception if greek is not generated by the pricing engine and reevaluates option to get numerical estimate of the seisitivity
                 Func<Func<double>, Func<double>, decimal> tryGetGreekOrReevaluate = (greek, reevalFunc) =>
@@ -217,10 +234,10 @@ namespace QuantConnect.Securities.Option
                                             () => tryGetGreekOrReevaluate(() => option.rho(), reevalRho),
                                             () => tryGetGreek(() => option.elasticity())));
             }
-            catch(Exception err)
+            catch (Exception err)
             {
                 Log.Debug($"QLOptionPriceModel.Evaluate() error: {err.Message}");
-                return new OptionPriceModelResult(0m, new Greeks());
+                return OptionPriceModelResult.None;
             }
         }
 

Common/Securities/Volatility/StandardDeviationOfReturnsVolatilityModel.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -35,10 +35,10 @@ namespace QuantConnect.Securities
         private decimal _volatility;
         private DateTime _lastUpdate = DateTime.MinValue;
         private decimal _lastPrice;
-        private readonly Resolution? _resolution;
-        private readonly TimeSpan _periodSpan;
+        private Resolution? _resolution;
+        private TimeSpan _periodSpan;
         private readonly object _sync = new object();
-        private readonly RollingWindow<double> _window;
+        private RollingWindow<double> _window;
 
         /// <summary>
         /// Gets the volatility of the security as a percentage
@@ -92,7 +92,7 @@ namespace QuantConnect.Securities
         {
             if (periods < 2)
             {
-                throw new ArgumentOutOfRangeException("periods", "'periods' must be greater than or equal to 2.");
+                throw new ArgumentOutOfRangeException(nameof(periods), "'periods' must be greater than or equal to 2.");
             }
 
             _window = new RollingWindow<double>(periods);
@@ -100,6 +100,30 @@ namespace QuantConnect.Securities
             _periodSpan = updateFrequency ?? resolution?.ToTimeSpan() ?? TimeSpan.FromDays(1);
         }
 
+        /// <summary>
+        /// Initializes a new instance of the <see cref="StandardDeviationOfReturnsVolatilityModel"/> class
+        /// </summary>
+        /// <param name="resolution">
+        /// Resolution of the price data inserted into the rolling window series to calculate standard deviation.
+        /// Will be used as the default value for update frequency if a value is not provided for <paramref name="updateFrequency"/>.
+        /// This only has a material effect in live mode. For backtesting, this value does not cause any behavioral changes.
+        /// </param>
+        /// <param name="updateFrequency">Frequency at which we insert new values into the rolling window for the standard deviation calculation</param>
+        /// <remarks>
+        /// The volatility model will be updated with the most granular/highest resolution data that was added to your algorithm.
+        /// That means that if I added <see cref="Resolution.Tick"/> data for my Futures strategy, that this model will be
+        /// updated using <see cref="Resolution.Tick"/> data as the algorithm progresses in time.
+        ///
+        /// Keep this in mind when setting the period and update frequency. The Resolution parameter is only used for live mode, or for
+        /// the default value of the <paramref name="updateFrequency"/> if no value is provided.
+        /// </remarks>
+        public StandardDeviationOfReturnsVolatilityModel(
+            Resolution resolution,
+            TimeSpan? updateFrequency = null
+            ) : this(PeriodsInResolution(resolution), resolution, updateFrequency)
+        {
+        }
+
         /// <summary>
         /// Updates this model using the new price information in
         /// the specified security instance
@@ -139,5 +163,28 @@ namespace QuantConnect.Securities
                 _resolution,
                 _window.Size + 1);
         }
+
+        private static int PeriodsInResolution(Resolution resolution)
+        {
+            int periods;
+            switch (resolution)
+            {
+                case Resolution.Tick:
+                case Resolution.Second:
+                    periods = 600;
+                    break;
+                case Resolution.Minute:
+                    periods = 60 * 24;
+                    break;
+                case Resolution.Hour:
+                    periods = 24 * 30;
+                    break;
+                default:
+                    periods = 30;
+                    break;
+            }
+
+            return periods;
+        }
     }
-}
+}

Configuration/Config.cs

@@ -298,8 +298,8 @@ namespace QuantConnect.Configuration
         /// </summary>
         /// <typeparam name="T">The desired output type</typeparam>
         /// <param name="key">The configuration key</param>
-        /// <param name="value">The output value</param>
-        /// <returns>True on successful parse, false when output value is default(T)</returns>
+        /// <param name="value">The output value. If the key is found and parsed successfully, it will be the parsed value, else default(T).</param>
+        /// <returns>True on successful parse or if they key is not found. False only when key is found but fails to parse.</returns>
         public static bool TryGetValue<T>(string key, out T value)
         {
             return TryGetValue(key, default(T), out value);
@@ -312,8 +312,8 @@ namespace QuantConnect.Configuration
         /// <typeparam name="T">The desired output type</typeparam>
         /// <param name="key">The configuration key</param>
         /// <param name="defaultValue">The default value to use on key not found or unsuccessful parse</param>
-        /// <param name="value">The output value</param>
-        /// <returns>True on successful parse, false when output value is defaultValue</returns>
+        /// <param name="value">The output value. If the key is found and parsed successfully, it will be the parsed value, else defaultValue.</param>
+        /// <returns>True on successful parse or if they key is not found and using defaultValue. False only when key is found but fails to parse.</returns>
         public static bool TryGetValue<T>(string key, T defaultValue, out T value)
         {
             try

Configuration/ToolboxArgumentParser.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -77,6 +77,9 @@ namespace QuantConnect.Configuration
                 new CommandLineOption("splits-percentage", CommandOptionType.SingleValue, "[OPTIONAL for RandomDataGenerator. Sets the probability each equity generated will have a stock split event. Note that this is not the total probability for all symbols generated. Only used for Equity. Defaults to 15.0: Example: --splits-percentage=10.0 ]"),
                 new CommandLineOption("dividends-percentage", CommandOptionType.SingleValue, "[OPTIONAL for RandomDataGenerator. Sets the probability each equity generated will have dividends. Note that this is not the probability for all symbols genearted. Only used for Equity. Defaults to 60.0: Example: --dividends-percentage=25.5 ]"),
                 new CommandLineOption("dividend-every-quarter-percentage", CommandOptionType.SingleValue, "[OPTIONAL for RandomDataGenerator. Sets the probability each equity generated will have a dividend event every quarter. Note that this is not the total probability for all symbols generated. Only used for Equity. Defaults to 30.0: Example: --dividend-every-quarter-percentage=15.0 ]"),
+                new CommandLineOption("option-price-engine", CommandOptionType.SingleValue, "[OPTIONAL for RandomDataGenerator. Sets the stochastic process, and returns new pricing engine to run calculations for that option. Defaults to BaroneAdesiWhaleyApproximationEngine: Example: --option-price-engine=BaroneAdesiWhaleyApproximationEngine ]"),
+                new CommandLineOption("volatility-model-resolution", CommandOptionType.SingleValue, "[OPTIONAL for RandomDataGenerator. Sets the volatility model period span. Defaults to Daily: Example: --volatility-model-resolution=Daily ]"),
+                new CommandLineOption("chain-symbol-count", CommandOptionType.SingleValue, "[OPTIONAL for RandomDataGenerator. Sets the size of the option chain. Defaults to 1 put and 1 call: Example: --chain-symbol-count=2 ]")
             };
 
         /// <summary>

Data/equity/india/factor_files/cccl.csv

@@ -0,0 +1,5 @@
+20100104,0.9800619,0.2,413
+20100209,0.9800619,0.2,421
+20100615,0.9800619,1,88.8
+20110615,0.9856115,1,34.75
+20501231,1,1,0

Data/equity/india/map_files/3mindia.csv

@@ -0,0 +1,3 @@
+19990101,birla3m
+20040615,birla3m
+20501231,3mindia

Data/market-hours/market-hours-database.json

@@ -37514,6 +37514,60 @@
         }
       ],
       "holidays": []
+    },
+    "Crypto-ftxus-[*]": {
+      "dataTimeZone": "UTC",
+      "exchangeTimeZone": "UTC",
+      "sunday": [
+        {
+          "start": "00:00:00",
+          "end": "1.00:00:00",
+          "state": "market"
+        }
+      ],
+      "monday": [
+        {
+          "start": "00:00:00",
+          "end": "1.00:00:00",
+          "state": "market"
+        }
+      ],
+      "tuesday": [
+        {
+          "start": "00:00:00",
+          "end": "1.00:00:00",
+          "state": "market"
+        }
+      ],
+      "wednesday": [
+        {
+          "start": "00:00:00",
+          "end": "1.00:00:00",
+          "state": "market"
+        }
+      ],
+      "thursday": [
+        {
+          "start": "00:00:00",
+          "end": "1.00:00:00",
+          "state": "market"
+        }
+      ],
+      "friday": [
+        {
+          "start": "00:00:00",
+          "end": "1.00:00:00",
+          "state": "market"
+        }
+      ],
+      "saturday": [
+        {
+          "start": "00:00:00",
+          "end": "1.00:00:00",
+          "state": "market"
+        }
+      ],
+      "holidays": []
     }
   }
 }

Data/symbol-properties/symbol-properties-database.csv

@@ -2822,21 +2822,22 @@ ftx,AAVEUSDT,crypto,AAVE/USDT,USDT,1,0.01,0.01,AAVE/USDT,0.01
 ftx,ABNBUSD,crypto,ABNB/USD,USD,1,0.005,0.025,ABNB/USD,0.025
 ftx,ACBUSD,crypto,ACB/USD,USD,1,0.001,0.1,ACB/USD,0.1
 ftx,ADABEARUSD,crypto,ADABEAR/USD,USD,1,0.00000001,10000000.0,ADABEAR/USD,10000000.0
-ftx,ADABULLUSD,crypto,ADABULL/USD,USD,1,0.5,0.0001,ADABULL/USD,0.0001
+ftx,ADABULLUSD,crypto,ADABULL/USD,USD,1,0.025,0.001,ADABULL/USD,0.001
 ftx,ADAHALFUSD,crypto,ADAHALF/USD,USD,1,2.5,0.000001,ADAHALF/USD,0.000001
-ftx,ADAHEDGEUSD,crypto,ADAHEDGE/USD,USD,1,0.0005,0.01,ADAHEDGE/USD,0.01
+ftx,ADAHEDGEUSD,crypto,ADAHEDGE/USD,USD,1,0.001,0.01,ADAHEDGE/USD,0.01
 ftx,AGLDUSD,crypto,AGLD/USD,USD,1,0.0005,0.1,AGLD/USD,0.1
 ftx,AKROUSD,crypto,AKRO/USD,USD,1,0.000005,1.0,AKRO/USD,1.0
 ftx,AKROUSDT,crypto,AKRO/USDT,USDT,1,0.000005,1.0,AKRO/USDT,1.0
 ftx,ALCXUSD,crypto,ALCX/USD,USD,1,0.1,0.001,ALCX/USD,0.001
-ftx,ALGOBEARUSD,crypto,ALGOBEAR/USD,USD,1,0.00000001,100000000.0,ALGOBEAR/USD,100000000.0
-ftx,ALGOBULLUSD,crypto,ALGOBULL/USD,USD,1,0.00000001,10000.0,ALGOBULL/USD,10000.0
+ftx,ALEPHUSD,crypto,ALEPH/USD,USD,1,0.00005,1.0,ALEPH/USD,1.0
+ftx,ALGOBEARUSD,crypto,ALGOBEAR/USD,USD,1,0.00000001,1000000000000.0,ALGOBEAR/USD,1000000000000.0
+ftx,ALGOBULLUSD,crypto,ALGOBULL/USD,USD,1,0.00000001,100000.0,ALGOBULL/USD,100000.0
 ftx,ALGOHALFUSD,crypto,ALGOHALF/USD,USD,1,1.0,0.00001,ALGOHALF/USD,0.00001
-ftx,ALGOHEDGEUSD,crypto,ALGOHEDGE/USD,USD,1,0.005,0.001,ALGOHEDGE/USD,0.001
+ftx,ALGOHEDGEUSD,crypto,ALGOHEDGE/USD,USD,1,0.0025,0.01,ALGOHEDGE/USD,0.01
 ftx,ALICEUSD,crypto,ALICE/USD,USD,1,0.0025,0.1,ALICE/USD,0.1
 ftx,ALPHAUSD,crypto,ALPHA/USD,USD,1,0.00005,1.0,ALPHA/USD,1.0
 ftx,ALTBEARUSD,crypto,ALTBEAR/USD,USD,1,0.00000001,1000.0,ALTBEAR/USD,1000.0
-ftx,ALTBULLUSD,crypto,ALTBULL/USD,USD,1,0.01,0.001,ALTBULL/USD,0.001
+ftx,ALTBULLUSD,crypto,ALTBULL/USD,USD,1,0.0025,0.01,ALTBULL/USD,0.01
 ftx,ALTHALFUSD,crypto,ALTHALF/USD,USD,1,1.0,0.00001,ALTHALF/USD,0.00001
 ftx,ALTHEDGEUSD,crypto,ALTHEDGE/USD,USD,1,0.01,0.001,ALTHEDGE/USD,0.001
 ftx,AMCUSD,crypto,AMC/USD,USD,1,0.0025,0.1,AMC/USD,0.1
@@ -2849,17 +2850,21 @@ ftx,ARKKUSD,crypto,ARKK/USD,USD,1,0.01,0.01,ARKK/USD,0.01
 ftx,ASDUSD,crypto,ASD/USD,USD,1,0.00001,0.1,ASD/USD,0.1
 ftx,ASDBEARUSD,crypto,ASDBEAR/USD,USD,1,0.00000001,100000.0,ASDBEAR/USD,100000.0
 ftx,ASDBEARUSDT,crypto,ASDBEAR/USDT,USDT,1,0.00000001,100000.0,ASDBEAR/USDT,100000.0
-ftx,ASDBULLUSD,crypto,ASDBULL/USD,USD,1,0.00025,0.1,ASDBULL/USD,0.1
-ftx,ASDBULLUSDT,crypto,ASDBULL/USDT,USDT,1,0.00025,0.1,ASDBULL/USDT,0.1
+ftx,ASDBULLUSD,crypto,ASDBULL/USD,USD,1,0.00005,1.0,ASDBULL/USD,1.0
+ftx,ASDBULLUSDT,crypto,ASDBULL/USDT,USDT,1,0.00005,1.0,ASDBULL/USDT,1.0
 ftx,ASDHALFUSD,crypto,ASDHALF/USD,USD,1,1.0,0.00001,ASDHALF/USD,0.00001
 ftx,ASDHEDGEUSD,crypto,ASDHEDGE/USD,USD,1,0.005,0.001,ASDHEDGE/USD,0.001
 ftx,ATLASUSD,crypto,ATLAS/USD,USD,1,0.00000025,10.0,ATLAS/USD,10.0
-ftx,ATOMBEARUSD,crypto,ATOMBEAR/USD,USD,1,0.00000001,10000000.0,ATOMBEAR/USD,10000000.0
-ftx,ATOMBULLUSD,crypto,ATOMBULL/USD,USD,1,0.00005,1.0,ATOMBULL/USD,1.0
+ftx,ATOMBEARUSD,crypto,ATOMBEAR/USD,USD,1,0.00000001,100000000.0,ATOMBEAR/USD,100000000.0
+ftx,ATOMBULLUSD,crypto,ATOMBULL/USD,USD,1,0.000005,10.0,ATOMBULL/USD,10.0
 ftx,ATOMHALFUSD,crypto,ATOMHALF/USD,USD,1,1.0,0.00001,ATOMHALF/USD,0.00001
-ftx,ATOMHEDGEUSD,crypto,ATOMHEDGE/USD,USD,1,0.0025,0.01,ATOMHEDGE/USD,0.01
+ftx,ATOMHEDGEUSD,crypto,ATOMHEDGE/USD,USD,1,0.001,0.01,ATOMHEDGE/USD,0.01
 ftx,AUDIOUSD,crypto,AUDIO/USD,USD,1,0.00005,1.0,AUDIO/USD,1.0
 ftx,AUDIOUSDT,crypto,AUDIO/USDT,USDT,1,0.00005,1.0,AUDIO/USDT,1.0
+ftx,AURYUSD,crypto,AURY/USD,USD,1,0.00025,1.0,AURY/USD,1.0
+ftx,AVAXBTC,crypto,AVAX/BTC,BTC,1,0.0000001,0.1,AVAX/BTC,0.1
+ftx,AVAXUSD,crypto,AVAX/USD,USD,1,0.0005,0.1,AVAX/USD,0.1
+ftx,AVAXUSDT,crypto,AVAX/USDT,USDT,1,0.0005,0.1,AVAX/USDT,0.1
 ftx,AXSUSD,crypto,AXS/USD,USD,1,0.001,0.1,AXS/USD,0.1
 ftx,BABAUSD,crypto,BABA/USD,USD,1,0.05,0.005,BABA/USD,0.005
 ftx,BADGERUSD,crypto,BADGER/USD,USD,1,0.005,0.01,BADGER/USD,0.01
@@ -2867,8 +2872,8 @@ ftx,BALUSD,crypto,BAL/USD,USD,1,0.0025,0.01,BAL/USD,0.01
 ftx,BALUSDT,crypto,BAL/USDT,USDT,1,0.0025,0.01,BAL/USDT,0.01
 ftx,BALBEARUSD,crypto,BALBEAR/USD,USD,1,0.00000001,10000.0,BALBEAR/USD,10000.0
 ftx,BALBEARUSDT,crypto,BALBEAR/USDT,USDT,1,0.00000001,10000.0,BALBEAR/USDT,10000.0
-ftx,BALBULLUSD,crypto,BALBULL/USD,USD,1,0.000025,1.0,BALBULL/USD,1.0
-ftx,BALBULLUSDT,crypto,BALBULL/USDT,USDT,1,0.000025,1.0,BALBULL/USDT,1.0
+ftx,BALBULLUSD,crypto,BALBULL/USD,USD,1,0.000005,10.0,BALBULL/USD,10.0
+ftx,BALBULLUSDT,crypto,BALBULL/USDT,USDT,1,0.000005,10.0,BALBULL/USDT,10.0
 ftx,BALHALFUSD,crypto,BALHALF/USD,USD,1,0.5,0.00001,BALHALF/USD,0.00001
 ftx,BALHEDGEUSD,crypto,BALHEDGE/USD,USD,1,0.01,0.001,BALHEDGE/USD,0.001
 ftx,BANDUSD,crypto,BAND/USD,USD,1,0.001,0.1,BAND/USD,0.1
@@ -2879,16 +2884,19 @@ ftx,BBUSD,crypto,BB/USD,USD,1,0.0025,0.1,BB/USD,0.1
 ftx,BCHBTC,crypto,BCH/BTC,BTC,1,0.000001,0.001,BCH/BTC,0.001
 ftx,BCHUSD,crypto,BCH/USD,USD,1,0.025,0.001,BCH/USD,0.001
 ftx,BCHUSDT,crypto,BCH/USDT,USDT,1,0.025,0.001,BCH/USDT,0.001
-ftx,BCHBEARUSD,crypto,BCHBEAR/USD,USD,1,0.00000005,1000.0,BCHBEAR/USD,1000.0
-ftx,BCHBEARUSDT,crypto,BCHBEAR/USDT,USDT,1,0.00000005,1000.0,BCHBEAR/USDT,1000.0
-ftx,BCHBULLUSD,crypto,BCHBULL/USD,USD,1,0.00001,10.0,BCHBULL/USD,10.0
-ftx,BCHBULLUSDT,crypto,BCHBULL/USDT,USDT,1,0.00001,10.0,BCHBULL/USDT,10.0
+ftx,BCHBEARUSD,crypto,BCHBEAR/USD,USD,1,0.0000001,1000.0,BCHBEAR/USD,1000.0
+ftx,BCHBEARUSDT,crypto,BCHBEAR/USDT,USDT,1,0.0000001,1000.0,BCHBEAR/USDT,1000.0
+ftx,BCHBULLUSD,crypto,BCHBULL/USD,USD,1,0.0000005,100.0,BCHBULL/USD,100.0
+ftx,BCHBULLUSDT,crypto,BCHBULL/USDT,USDT,1,0.0000005,100.0,BCHBULL/USDT,100.0
 ftx,BCHHALFUSD,crypto,BCHHALF/USD,USD,1,0.5,0.00001,BCHHALF/USD,0.00001
-ftx,BCHHEDGEUSD,crypto,BCHHEDGE/USD,USD,1,0.025,0.001,BCHHEDGE/USD,0.001
+ftx,BCHHEDGEUSD,crypto,BCHHEDGE/USD,USD,1,0.05,0.0001,BCHHEDGE/USD,0.0001
 ftx,BEARUSD,crypto,BEAR/USD,USD,1,0.00000005,1000.0,BEAR/USD,1000.0
 ftx,BEARUSDT,crypto,BEAR/USDT,USDT,1,0.00000005,1000.0,BEAR/USDT,1000.0
 ftx,BEARSHITUSD,crypto,BEARSHIT/USD,USD,1,0.00000001,10000.0,BEARSHIT/USD,10000.0
+ftx,BICOUSD,crypto,BICO/USD,USD,1,0.0001,1.0,BICO/USD,1.0
 ftx,BILIUSD,crypto,BILI/USD,USD,1,0.005,0.05,BILI/USD,0.05
+ftx,BITUSD,crypto,BIT/USD,USD,1,0.00025,1.0,BIT/USD,1.0
+ftx,BITOUSD,crypto,BITO/USD,USD,1,0.005,0.01,BITO/USD,0.01
 ftx,BITWUSD,crypto,BITW/USD,USD,1,0.005,0.01,BITW/USD,0.01
 ftx,BLTUSD,crypto,BLT/USD,USD,1,0.0001,1.0,BLT/USD,1.0
 ftx,BNBBTC,crypto,BNB/BTC,BTC,1,0.0000001,0.01,BNB/BTC,0.01
@@ -2896,18 +2904,19 @@ ftx,BNBUSD,crypto,BNB/USD,USD,1,0.001,0.01,BNB/USD,0.01
 ftx,BNBUSDT,crypto,BNB/USDT,USDT,1,0.001,0.01,BNB/USDT,0.01
 ftx,BNBBEARUSD,crypto,BNBBEAR/USD,USD,1,0.00000001,10000000.0,BNBBEAR/USD,10000000.0
 ftx,BNBBEARUSDT,crypto,BNBBEAR/USDT,USDT,1,0.00000001,10000000.0,BNBBEAR/USDT,10000000.0
-ftx,BNBBULLUSD,crypto,BNBBULL/USD,USD,1,0.25,0.0001,BNBBULL/USD,0.0001
-ftx,BNBBULLUSDT,crypto,BNBBULL/USDT,USDT,1,0.25,0.0001,BNBBULL/USDT,0.0001
+ftx,BNBBULLUSD,crypto,BNBBULL/USD,USD,1,0.1,0.0001,BNBBULL/USD,0.0001
+ftx,BNBBULLUSDT,crypto,BNBBULL/USDT,USDT,1,0.1,0.0001,BNBBULL/USDT,0.0001
 ftx,BNBHALFUSD,crypto,BNBHALF/USD,USD,1,2.5,0.000001,BNBHALF/USD,0.000001
 ftx,BNBHEDGEUSD,crypto,BNBHEDGE/USD,USD,1,0.001,0.01,BNBHEDGE/USD,0.01
 ftx,BNTUSD,crypto,BNT/USD,USD,1,0.0005,0.1,BNT/USD,0.1
 ftx,BNTXUSD,crypto,BNTX/USD,USD,1,0.01,0.01,BNTX/USD,0.01
+ftx,BOBAUSD,crypto,BOBA/USD,USD,1,0.0005,0.1,BOBA/USD,0.1
 ftx,BRZUSD,crypto,BRZ/USD,USD,1,0.00001,1.0,BRZ/USD,1.0
 ftx,BRZUSDT,crypto,BRZ/USDT,USDT,1,0.00001,1.0,BRZ/USDT,1.0
 ftx,BSVBEARUSD,crypto,BSVBEAR/USD,USD,1,0.00000001,10000.0,BSVBEAR/USD,10000.0
 ftx,BSVBEARUSDT,crypto,BSVBEAR/USDT,USDT,1,0.00000001,10000.0,BSVBEAR/USDT,10000.0
-ftx,BSVBULLUSD,crypto,BSVBULL/USD,USD,1,0.00000001,1000.0,BSVBULL/USD,1000.0
-ftx,BSVBULLUSDT,crypto,BSVBULL/USDT,USDT,1,0.00000001,1000.0,BSVBULL/USDT,1000.0
+ftx,BSVBULLUSD,crypto,BSVBULL/USD,USD,1,0.00000001,10000.0,BSVBULL/USD,10000.0
+ftx,BSVBULLUSDT,crypto,BSVBULL/USDT,USDT,1,0.00000001,10000.0,BSVBULL/USDT,10000.0
 ftx,BSVHALFUSD,crypto,BSVHALF/USD,USD,1,0.25,0.00001,BSVHALF/USD,0.00001
 ftx,BSVHEDGEUSD,crypto,BSVHEDGE/USD,USD,1,0.025,0.001,BSVHEDGE/USD,0.001
 ftx,BTCBRZ,crypto,BTC/BRZ,BRZ,1,5.0,0.0001,BTC/BRZ,0.0001
@@ -2915,9 +2924,9 @@ ftx,BTCEUR,crypto,BTC/EUR,EUR,1,1.0,0.0001,BTC/EUR,0.0001
 ftx,BTCTRYB,crypto,BTC/TRYB,TRYB,1,0.1,0.0001,BTC/TRYB,0.0001
 ftx,BTCUSD,crypto,BTC/USD,USD,1,1.0,0.0001,BTC/USD,0.0001
 ftx,BTCUSDT,crypto,BTC/USDT,USDT,1,1.0,0.0001,BTC/USDT,0.0001
-ftx,BULLUSD,crypto,BULL/USD,USD,1,2.5,0.00001,BULL/USD,0.00001
-ftx,BULLUSDT,crypto,BULL/USDT,USDT,1,2.5,0.00001,BULL/USDT,0.00001
-ftx,BULLSHITUSD,crypto,BULLSHIT/USD,USD,1,0.025,0.001,BULLSHIT/USD,0.001
+ftx,BULLUSD,crypto,BULL/USD,USD,1,1.0,0.0001,BULL/USD,0.0001
+ftx,BULLUSDT,crypto,BULL/USDT,USDT,1,1.0,0.0001,BULL/USDT,0.0001
+ftx,BULLSHITUSD,crypto,BULLSHIT/USD,USD,1,0.005,0.01,BULLSHIT/USD,0.01
 ftx,BVOLBTC,crypto,BVOL/BTC,BTC,1,0.00005,0.0001,BVOL/BTC,0.0001
 ftx,BVOLUSD,crypto,BVOL/USD,USD,1,0.05,0.0001,BVOL/USD,0.0001
 ftx,BVOLUSDT,crypto,BVOL/USDT,USDT,1,0.05,0.0001,BVOL/USDT,0.0001
@@ -2937,10 +2946,10 @@ ftx,COMPUSD,crypto,COMP/USD,USD,1,0.01,0.0001,COMP/USD,0.0001
 ftx,COMPUSDT,crypto,COMP/USDT,USDT,1,0.01,0.0001,COMP/USDT,0.0001
 ftx,COMPBEARUSD,crypto,COMPBEAR/USD,USD,1,0.00000001,10000.0,COMPBEAR/USD,10000.0
 ftx,COMPBEARUSDT,crypto,COMPBEAR/USDT,USDT,1,0.00000001,10000.0,COMPBEAR/USDT,10000.0
-ftx,COMPBULLUSD,crypto,COMPBULL/USD,USD,1,0.0005,0.1,COMPBULL/USD,0.1
-ftx,COMPBULLUSDT,crypto,COMPBULL/USDT,USDT,1,0.0005,0.1,COMPBULL/USDT,0.1
-ftx,COMPHALFUSD,crypto,COMPHALF/USD,USD,1,1.0,0.00001,COMPHALF/USD,0.00001
-ftx,COMPHEDGEUSD,crypto,COMPHEDGE/USD,USD,1,0.005,0.001,COMPHEDGE/USD,0.001
+ftx,COMPBULLUSD,crypto,COMPBULL/USD,USD,1,0.000025,1.0,COMPBULL/USD,1.0
+ftx,COMPBULLUSDT,crypto,COMPBULL/USDT,USDT,1,0.000025,1.0,COMPBULL/USDT,1.0
+ftx,COMPHALFUSD,crypto,COMPHALF/USD,USD,1,0.5,0.00001,COMPHALF/USD,0.00001
+ftx,COMPHEDGEUSD,crypto,COMPHEDGE/USD,USD,1,0.01,0.001,COMPHEDGE/USD,0.001
 ftx,CONVUSD,crypto,CONV/USD,USD,1,0.00001,10.0,CONV/USD,10.0
 ftx,COPEUSD,crypto,COPE/USD,USD,1,0.00025,1.0,COPE/USD,1.0
 ftx,CQTUSD,crypto,CQT/USD,USD,1,0.0001,1.0,CQT/USD,1.0
@@ -2963,11 +2972,12 @@ ftx,DAIUSDT,crypto,DAI/USDT,USDT,1,0.0001,0.1,DAI/USDT,0.1
 ftx,DAWNUSD,crypto,DAWN/USD,USD,1,0.0005,0.1,DAWN/USD,0.1
 ftx,DEFIBEARUSD,crypto,DEFIBEAR/USD,USD,1,0.00000025,100.0,DEFIBEAR/USD,100.0
 ftx,DEFIBEARUSDT,crypto,DEFIBEAR/USDT,USDT,1,0.00000025,100.0,DEFIBEAR/USDT,100.0
-ftx,DEFIBULLUSD,crypto,DEFIBULL/USD,USD,1,0.025,0.001,DEFIBULL/USD,0.001
-ftx,DEFIBULLUSDT,crypto,DEFIBULL/USDT,USDT,1,0.025,0.001,DEFIBULL/USDT,0.001
+ftx,DEFIBULLUSD,crypto,DEFIBULL/USD,USD,1,0.0025,0.01,DEFIBULL/USD,0.01
+ftx,DEFIBULLUSDT,crypto,DEFIBULL/USDT,USDT,1,0.0025,0.01,DEFIBULL/USDT,0.01
 ftx,DEFIHALFUSD,crypto,DEFIHALF/USD,USD,1,1.0,0.00001,DEFIHALF/USD,0.00001
-ftx,DEFIHEDGEUSD,crypto,DEFIHEDGE/USD,USD,1,0.01,0.001,DEFIHEDGE/USD,0.001
+ftx,DEFIHEDGEUSD,crypto,DEFIHEDGE/USD,USD,1,0.025,0.001,DEFIHEDGE/USD,0.001
 ftx,DENTUSD,crypto,DENT/USD,USD,1,0.000001,100.0,DENT/USD,100.0
+ftx,DFLUSD,crypto,DFL/USD,USD,1,0.000005,10.0,DFL/USD,10.0
 ftx,DKNGUSD,crypto,DKNG/USD,USD,1,0.005,0.01,DKNG/USD,0.01
 ftx,DMGUSD,crypto,DMG/USD,USD,1,0.001,0.1,DMG/USD,0.1
 ftx,DMGUSDT,crypto,DMG/USDT,USDT,1,0.001,0.1,DMG/USDT,0.1
@@ -2976,25 +2986,29 @@ ftx,DOGEBTC,crypto,DOGE/BTC,BTC,1,0.00000001,1.0,DOGE/BTC,1.0
 ftx,DOGEUSD,crypto,DOGE/USD,USD,1,0.0000005,1.0,DOGE/USD,1.0
 ftx,DOGEUSDT,crypto,DOGE/USDT,USDT,1,0.0000005,1.0,DOGE/USDT,1.0
 ftx,DOGEBEAR2021USD,crypto,DOGEBEAR2021/USD,USD,1,0.0025,0.01,DOGEBEAR2021/USD,0.01
-ftx,DOGEBULLUSD,crypto,DOGEBULL/USD,USD,1,0.025,0.001,DOGEBULL/USD,0.001
-ftx,DOGEHALFUSD,crypto,DOGEHALF/USD,USD,1,10.0,0.000001,DOGEHALF/USD,0.000001
+ftx,DOGEBULLUSD,crypto,DOGEBULL/USD,USD,1,0.001,0.01,DOGEBULL/USD,0.01
+ftx,DOGEHALFUSD,crypto,DOGEHALF/USD,USD,1,5.0,0.000001,DOGEHALF/USD,0.000001
 ftx,DOGEHEDGEUSD,crypto,DOGEHEDGE/USD,USD,1,0.00005,0.1,DOGEHEDGE/USD,0.1
+ftx,DOTBTC,crypto,DOT/BTC,BTC,1,0.00000001,0.1,DOT/BTC,0.1
+ftx,DOTUSD,crypto,DOT/USD,USD,1,0.001,0.1,DOT/USD,0.1
+ftx,DOTUSDT,crypto,DOT/USDT,USDT,1,0.001,0.1,DOT/USDT,0.1
 ftx,DRGNBEARUSD,crypto,DRGNBEAR/USD,USD,1,0.00000001,10000.0,DRGNBEAR/USD,10000.0
-ftx,DRGNBULLUSD,crypto,DRGNBULL/USD,USD,1,0.005,0.01,DRGNBULL/USD,0.01
+ftx,DRGNBULLUSD,crypto,DRGNBULL/USD,USD,1,0.0005,0.1,DRGNBULL/USD,0.1
 ftx,DRGNHALFUSD,crypto,DRGNHALF/USD,USD,1,1.0,0.00001,DRGNHALF/USD,0.00001
 ftx,DRGNHEDGEUSD,crypto,DRGNHEDGE/USD,USD,1,0.005,0.001,DRGNHEDGE/USD,0.001
 ftx,DYDXUSD,crypto,DYDX/USD,USD,1,0.001,0.1,DYDX/USD,0.1
 ftx,EDENUSD,crypto,EDEN/USD,USD,1,0.0005,0.1,EDEN/USD,0.1
 ftx,EMBUSD,crypto,EMB/USD,USD,1,0.00001,10.0,EMB/USD,10.0
 ftx,ENJUSD,crypto,ENJ/USD,USD,1,0.00005,1.0,ENJ/USD,1.0
+ftx,ENSUSD,crypto,ENS/USD,USD,1,0.005,0.01,ENS/USD,0.01
 ftx,EOSBEARUSD,crypto,EOSBEAR/USD,USD,1,0.00000001,10000.0,EOSBEAR/USD,10000.0
 ftx,EOSBEARUSDT,crypto,EOSBEAR/USDT,USDT,1,0.00000001,10000.0,EOSBEAR/USDT,10000.0
-ftx,EOSBULLUSD,crypto,EOSBULL/USD,USD,1,0.00000025,100.0,EOSBULL/USD,100.0
-ftx,EOSBULLUSDT,crypto,EOSBULL/USDT,USDT,1,0.00000025,100.0,EOSBULL/USDT,100.0
+ftx,EOSBULLUSD,crypto,EOSBULL/USD,USD,1,0.00000001,1000.0,EOSBULL/USD,1000.0
+ftx,EOSBULLUSDT,crypto,EOSBULL/USDT,USDT,1,0.00000001,1000.0,EOSBULL/USDT,1000.0
 ftx,EOSHALFUSD,crypto,EOSHALF/USD,USD,1,0.5,0.00001,EOSHALF/USD,0.00001
-ftx,EOSHEDGEUSD,crypto,EOSHEDGE/USD,USD,1,0.01,0.001,EOSHEDGE/USD,0.001
+ftx,EOSHEDGEUSD,crypto,EOSHEDGE/USD,USD,1,0.025,0.001,EOSHEDGE/USD,0.001
 ftx,ETCBEARUSD,crypto,ETCBEAR/USD,USD,1,0.00000001,1000000.0,ETCBEAR/USD,1000000.0
-ftx,ETCBULLUSD,crypto,ETCBULL/USD,USD,1,0.0025,0.01,ETCBULL/USD,0.01
+ftx,ETCBULLUSD,crypto,ETCBULL/USD,USD,1,0.0001,0.1,ETCBULL/USD,0.1
 ftx,ETCHALFUSD,crypto,ETCHALF/USD,USD,1,1.0,0.00001,ETCHALF/USD,0.00001
 ftx,ETCHEDGEUSD,crypto,ETCHEDGE/USD,USD,1,0.0025,0.01,ETCHEDGE/USD,0.01
 ftx,ETHBRZ,crypto,ETH/BRZ,BRZ,1,1.0,0.0001,ETH/BRZ,0.0001
@@ -3004,14 +3018,14 @@ ftx,ETHUSD,crypto,ETH/USD,USD,1,0.1,0.001,ETH/USD,0.001
 ftx,ETHUSDT,crypto,ETH/USDT,USDT,1,0.1,0.001,ETH/USDT,0.001
 ftx,ETHBEARUSD,crypto,ETHBEAR/USD,USD,1,0.00000001,1000000.0,ETHBEAR/USD,1000000.0
 ftx,ETHBEARUSDT,crypto,ETHBEAR/USDT,USDT,1,0.00000001,1000000.0,ETHBEAR/USDT,1000000.0
-ftx,ETHBULLUSD,crypto,ETHBULL/USD,USD,1,0.5,0.0001,ETHBULL/USD,0.0001
-ftx,ETHBULLUSDT,crypto,ETHBULL/USDT,USDT,1,0.5,0.0001,ETHBULL/USDT,0.0001
+ftx,ETHBULLUSD,crypto,ETHBULL/USD,USD,1,0.1,0.0001,ETHBULL/USD,0.0001
+ftx,ETHBULLUSDT,crypto,ETHBULL/USDT,USDT,1,0.1,0.0001,ETHBULL/USDT,0.0001
 ftx,ETHEUSD,crypto,ETHE/USD,USD,1,0.001,0.1,ETHE/USD,0.1
 ftx,ETHHALFUSD,crypto,ETHHALF/USD,USD,1,1.0,0.00001,ETHHALF/USD,0.00001
 ftx,ETHHEDGEUSD,crypto,ETHHEDGE/USD,USD,1,0.0025,0.01,ETHHEDGE/USD,0.01
 ftx,EURUSD,crypto,EUR/USD,USD,1,0.0001,1.0,EUR/USD,1.0
-ftx,EXCHBEARUSD,crypto,EXCHBEAR/USD,USD,1,0.00000025,100.0,EXCHBEAR/USD,100.0
-ftx,EXCHBULLUSD,crypto,EXCHBULL/USD,USD,1,2.5,0.00001,EXCHBULL/USD,0.00001
+ftx,EXCHBEARUSD,crypto,EXCHBEAR/USD,USD,1,0.0000001,1000.0,EXCHBEAR/USD,1000.0
+ftx,EXCHBULLUSD,crypto,EXCHBULL/USD,USD,1,1.0,0.00001,EXCHBULL/USD,0.00001
 ftx,EXCHHALFUSD,crypto,EXCHHALF/USD,USD,1,1.0,0.00001,EXCHHALF/USD,0.00001
 ftx,EXCHHEDGEUSD,crypto,EXCHHEDGE/USD,USD,1,0.01,0.001,EXCHHEDGE/USD,0.001
 ftx,FBUSD,crypto,FB/USD,USD,1,0.01,0.01,FB/USD,0.01
@@ -3024,17 +3038,21 @@ ftx,FTTBTC,crypto,FTT/BTC,BTC,1,0.0000001,0.1,FTT/BTC,0.1
 ftx,FTTUSD,crypto,FTT/USD,USD,1,0.001,0.1,FTT/USD,0.1
 ftx,FTTUSDT,crypto,FTT/USDT,USDT,1,0.001,0.1,FTT/USDT,0.1
 ftx,GALUSD,crypto,GAL/USD,USD,1,0.001,0.1,GAL/USD,0.1
+ftx,GALAUSD,crypto,GALA/USD,USD,1,0.000025,10.0,GALA/USD,10.0
 ftx,GBPUSD,crypto,GBP/USD,USD,1,0.0001,1.0,GBP/USD,1.0
 ftx,GBTCUSD,crypto,GBTC/USD,USD,1,0.005,0.01,GBTC/USD,0.01
 ftx,GDXUSD,crypto,GDX/USD,USD,1,0.005,0.01,GDX/USD,0.01
 ftx,GDXJUSD,crypto,GDXJ/USD,USD,1,0.005,0.01,GDXJ/USD,0.01
+ftx,GENEUSD,crypto,GENE/USD,USD,1,0.001,0.1,GENE/USD,0.1
 ftx,GLDUSD,crypto,GLD/USD,USD,1,0.025,0.01,GLD/USD,0.01
 ftx,GLXYUSD,crypto,GLXY/USD,USD,1,0.001,0.1,GLXY/USD,0.1
 ftx,GMEUSD,crypto,GME/USD,USD,1,0.025,0.01,GME/USD,0.01
+ftx,GODSUSD,crypto,GODS/USD,USD,1,0.0005,0.1,GODS/USD,0.1
+ftx,GOGUSD,crypto,GOG/USD,USD,1,0.00025,1.0,GOG/USD,1.0
 ftx,GOOGLUSD,crypto,GOOGL/USD,USD,1,0.1,0.001,GOOGL/USD,0.001
 ftx,GRTUSD,crypto,GRT/USD,USD,1,0.00005,1.0,GRT/USD,1.0
-ftx,GRTBEARUSD,crypto,GRTBEAR/USD,USD,1,0.00000025,100.0,GRTBEAR/USD,100.0
-ftx,GRTBULLUSD,crypto,GRTBULL/USD,USD,1,0.0001,0.1,GRTBULL/USD,0.1
+ftx,GRTBEARUSD,crypto,GRTBEAR/USD,USD,1,0.00000005,1000.0,GRTBEAR/USD,1000.0
+ftx,GRTBULLUSD,crypto,GRTBULL/USD,USD,1,0.000005,10.0,GRTBULL/USD,10.0
 ftx,GTUSD,crypto,GT/USD,USD,1,0.0005,0.1,GT/USD,0.1
 ftx,HALFUSD,crypto,HALF/USD,USD,1,1.0,0.00001,HALF/USD,0.00001
 ftx,HALFSHITUSD,crypto,HALFSHIT/USD,USD,1,1.0,0.00001,HALFSHIT/USD,0.00001
@@ -3048,50 +3066,56 @@ ftx,HNTUSDT,crypto,HNT/USDT,USDT,1,0.0001,0.1,HNT/USDT,0.1
 ftx,HOLYUSD,crypto,HOLY/USD,USD,1,0.0001,1.0,HOLY/USD,1.0
 ftx,HOODUSD,crypto,HOOD/USD,USD,1,0.005,0.01,HOOD/USD,0.01
 ftx,HTUSD,crypto,HT/USD,USD,1,0.0005,0.1,HT/USD,0.1
-ftx,HTBEARUSD,crypto,HTBEAR/USD,USD,1,0.0000025,10.0,HTBEAR/USD,10.0
-ftx,HTBULLUSD,crypto,HTBULL/USD,USD,1,0.0005,0.1,HTBULL/USD,0.1
+ftx,HTBEARUSD,crypto,HTBEAR/USD,USD,1,0.0000005,100.0,HTBEAR/USD,100.0
+ftx,HTBULLUSD,crypto,HTBULL/USD,USD,1,0.00025,0.1,HTBULL/USD,0.1
 ftx,HTHALFUSD,crypto,HTHALF/USD,USD,1,0.5,0.00001,HTHALF/USD,0.00001
-ftx,HTHEDGEUSD,crypto,HTHEDGE/USD,USD,1,0.05,0.0001,HTHEDGE/USD,0.0001
+ftx,HTHEDGEUSD,crypto,HTHEDGE/USD,USD,1,0.025,0.001,HTHEDGE/USD,0.001
 ftx,HUMUSD,crypto,HUM/USD,USD,1,0.000025,10.0,HUM/USD,10.0
 ftx,HXROUSD,crypto,HXRO/USD,USD,1,0.00005,1.0,HXRO/USD,1.0
 ftx,HXROUSDT,crypto,HXRO/USDT,USDT,1,0.00005,1.0,HXRO/USDT,1.0
-ftx,IBVOLBTC,crypto,IBVOL/BTC,BTC,1,0.00005,0.00001,IBVOL/BTC,0.00001
-ftx,IBVOLUSD,crypto,IBVOL/USD,USD,1,0.5,0.00001,IBVOL/USD,0.00001
-ftx,IBVOLUSDT,crypto,IBVOL/USDT,USDT,1,0.5,0.00001,IBVOL/USDT,0.00001
+ftx,IBVOLBTC,crypto,IBVOL/BTC,BTC,1,0.00005,0.0001,IBVOL/BTC,0.0001
+ftx,IBVOLUSD,crypto,IBVOL/USD,USD,1,0.25,0.0001,IBVOL/USD,0.0001
+ftx,IBVOLUSDT,crypto,IBVOL/USDT,USDT,1,0.25,0.0001,IBVOL/USDT,0.0001
+ftx,IMXUSD,crypto,IMX/USD,USD,1,0.0005,0.1,IMX/USD,0.1
 ftx,INTERUSD,crypto,INTER/USD,USD,1,0.001,0.1,INTER/USD,0.1
+ftx,JETUSD,crypto,JET/USD,USD,1,0.00005,1.0,JET/USD,1.0
+ftx,JOEUSD,crypto,JOE/USD,USD,1,0.00025,1.0,JOE/USD,1.0
 ftx,JSTUSD,crypto,JST/USD,USD,1,0.000005,10.0,JST/USD,10.0
 ftx,KINUSD,crypto,KIN/USD,USD,1,0.00000001,10000.0,KIN/USD,10000.0
 ftx,KNCUSD,crypto,KNC/USD,USD,1,0.0001,0.1,KNC/USD,0.1
 ftx,KNCUSDT,crypto,KNC/USDT,USDT,1,0.0001,0.1,KNC/USDT,0.1
-ftx,KNCBEARUSD,crypto,KNCBEAR/USD,USD,1,0.00000025,100.0,KNCBEAR/USD,100.0
-ftx,KNCBEARUSDT,crypto,KNCBEAR/USDT,USDT,1,0.00000025,100.0,KNCBEAR/USDT,100.0
-ftx,KNCBULLUSD,crypto,KNCBULL/USD,USD,1,0.0001,0.1,KNCBULL/USD,0.1
-ftx,KNCBULLUSDT,crypto,KNCBULL/USDT,USDT,1,0.0001,0.1,KNCBULL/USDT,0.1
+ftx,KNCBEARUSD,crypto,KNCBEAR/USD,USD,1,0.0000001,1000.0,KNCBEAR/USD,1000.0
+ftx,KNCBEARUSDT,crypto,KNCBEAR/USDT,USDT,1,0.0000001,1000.0,KNCBEAR/USDT,1000.0
+ftx,KNCBULLUSD,crypto,KNCBULL/USD,USD,1,0.00001,1.0,KNCBULL/USD,1.0
+ftx,KNCBULLUSDT,crypto,KNCBULL/USDT,USDT,1,0.00001,1.0,KNCBULL/USDT,1.0
 ftx,KNCHALFUSD,crypto,KNCHALF/USD,USD,1,0.5,0.00001,KNCHALF/USD,0.00001
 ftx,KNCHEDGEUSD,crypto,KNCHEDGE/USD,USD,1,0.025,0.001,KNCHEDGE/USD,0.001
+ftx,KSHIBUSD,crypto,KSHIB/USD,USD,1,0.000005,10.0,KSHIB/USD,10.0
+ftx,KSOSUSD,crypto,KSOS/USD,USD,1,0.0000005,100.0,KSOS/USD,100.0
 ftx,LEOUSD,crypto,LEO/USD,USD,1,0.0001,1.0,LEO/USD,1.0
-ftx,LEOBEARUSD,crypto,LEOBEAR/USD,USD,1,0.005,0.01,LEOBEAR/USD,0.01
-ftx,LEOBULLUSD,crypto,LEOBULL/USD,USD,1,0.25,0.0001,LEOBULL/USD,0.0001
+ftx,LEOBEARUSD,crypto,LEOBEAR/USD,USD,1,0.001,0.1,LEOBEAR/USD,0.1
+ftx,LEOBULLUSD,crypto,LEOBULL/USD,USD,1,0.5,0.0001,LEOBULL/USD,0.0001
 ftx,LEOHALFUSD,crypto,LEOHALF/USD,USD,1,0.5,0.00001,LEOHALF/USD,0.00001
 ftx,LEOHEDGEUSD,crypto,LEOHEDGE/USD,USD,1,0.1,0.0001,LEOHEDGE/USD,0.0001
 ftx,LINAUSD,crypto,LINA/USD,USD,1,0.000005,10.0,LINA/USD,10.0
 ftx,LINKBTC,crypto,LINK/BTC,BTC,1,0.00000005,0.1,LINK/BTC,0.1
 ftx,LINKUSD,crypto,LINK/USD,USD,1,0.0005,0.1,LINK/USD,0.1
 ftx,LINKUSDT,crypto,LINK/USDT,USDT,1,0.0005,0.1,LINK/USDT,0.1
-ftx,LINKBEARUSD,crypto,LINKBEAR/USD,USD,1,0.00000001,10000000.0,LINKBEAR/USD,10000000.0
-ftx,LINKBEARUSDT,crypto,LINKBEAR/USDT,USDT,1,0.00000001,10000000.0,LINKBEAR/USDT,10000000.0
-ftx,LINKBULLUSD,crypto,LINKBULL/USD,USD,1,0.0005,0.1,LINKBULL/USD,0.1
-ftx,LINKBULLUSDT,crypto,LINKBULL/USDT,USDT,1,0.0005,0.1,LINKBULL/USDT,0.1
+ftx,LINKBEARUSD,crypto,LINKBEAR/USD,USD,1,0.00000001,100000000.0,LINKBEAR/USD,100000000.0
+ftx,LINKBEARUSDT,crypto,LINKBEAR/USDT,USDT,1,0.00000001,100000000.0,LINKBEAR/USDT,100000000.0
+ftx,LINKBULLUSD,crypto,LINKBULL/USD,USD,1,0.0001,1.0,LINKBULL/USD,1.0
+ftx,LINKBULLUSDT,crypto,LINKBULL/USDT,USDT,1,0.0001,1.0,LINKBULL/USDT,1.0
 ftx,LINKHALFUSD,crypto,LINKHALF/USD,USD,1,1.0,0.00001,LINKHALF/USD,0.00001
 ftx,LINKHEDGEUSD,crypto,LINKHEDGE/USD,USD,1,0.001,0.01,LINKHEDGE/USD,0.01
+ftx,LOOKSUSD,crypto,LOOKS/USD,USD,1,0.00025,1.0,LOOKS/USD,1.0
 ftx,LRCUSD,crypto,LRC/USD,USD,1,0.00005,1.0,LRC/USD,1.0
 ftx,LTCBTC,crypto,LTC/BTC,BTC,1,0.00000025,0.01,LTC/BTC,0.01
 ftx,LTCUSD,crypto,LTC/USD,USD,1,0.005,0.01,LTC/USD,0.01
 ftx,LTCUSDT,crypto,LTC/USDT,USDT,1,0.005,0.01,LTC/USDT,0.01
-ftx,LTCBEARUSD,crypto,LTCBEAR/USD,USD,1,0.0000005,100.0,LTCBEAR/USD,100.0
-ftx,LTCBEARUSDT,crypto,LTCBEAR/USDT,USDT,1,0.0000005,100.0,LTCBEAR/USDT,100.0
-ftx,LTCBULLUSD,crypto,LTCBULL/USD,USD,1,0.00005,1.0,LTCBULL/USD,1.0
-ftx,LTCBULLUSDT,crypto,LTCBULL/USDT,USDT,1,0.00005,1.0,LTCBULL/USDT,1.0
+ftx,LTCBEARUSD,crypto,LTCBEAR/USD,USD,1,0.00000025,100.0,LTCBEAR/USD,100.0
+ftx,LTCBEARUSDT,crypto,LTCBEAR/USDT,USDT,1,0.00000025,100.0,LTCBEAR/USDT,100.0
+ftx,LTCBULLUSD,crypto,LTCBULL/USD,USD,1,0.000005,10.0,LTCBULL/USD,10.0
+ftx,LTCBULLUSDT,crypto,LTCBULL/USDT,USDT,1,0.000005,10.0,LTCBULL/USDT,10.0
 ftx,LTCHALFUSD,crypto,LTCHALF/USD,USD,1,0.5,0.00001,LTCHALF/USD,0.00001
 ftx,LTCHEDGEUSD,crypto,LTCHEDGE/USD,USD,1,0.05,0.0001,LTCHEDGE/USD,0.0001
 ftx,LUAUSD,crypto,LUA/USD,USD,1,0.00025,0.1,LUA/USD,0.1
@@ -3103,37 +3127,41 @@ ftx,MATHUSD,crypto,MATH/USD,USD,1,0.0001,0.1,MATH/USD,0.1
 ftx,MATHUSDT,crypto,MATH/USDT,USDT,1,0.0001,0.1,MATH/USDT,0.1
 ftx,MATICBTC,crypto,MATIC/BTC,BTC,1,0.00000001,10.0,MATIC/BTC,10.0
 ftx,MATICUSD,crypto,MATIC/USD,USD,1,0.000001,10.0,MATIC/USD,10.0
-ftx,MATICBEAR2021USD,crypto,MATICBEAR2021/USD,USD,1,0.00001,1.0,MATICBEAR2021/USD,1.0
-ftx,MATICBULLUSD,crypto,MATICBULL/USD,USD,1,0.0001,0.1,MATICBULL/USD,0.1
+ftx,MATICBEAR2021USD,crypto,MATICBEAR2021/USD,USD,1,0.0000001,100.0,MATICBEAR2021/USD,100.0
+ftx,MATICBULLUSD,crypto,MATICBULL/USD,USD,1,0.00005,1.0,MATICBULL/USD,1.0
 ftx,MATICHALFUSD,crypto,MATICHALF/USD,USD,1,5.0,0.000001,MATICHALF/USD,0.000001
-ftx,MATICHEDGEUSD,crypto,MATICHEDGE/USD,USD,1,0.00005,0.1,MATICHEDGE/USD,0.1
+ftx,MATICHEDGEUSD,crypto,MATICHEDGE/USD,USD,1,0.00001,1.0,MATICHEDGE/USD,1.0
+ftx,MBSUSD,crypto,MBS/USD,USD,1,0.00005,1.0,MBS/USD,1.0
+ftx,MCBUSD,crypto,MCB/USD,USD,1,0.005,0.01,MCB/USD,0.01
 ftx,MEDIAUSD,crypto,MEDIA/USD,USD,1,0.1,0.01,MEDIA/USD,0.01
 ftx,MERUSD,crypto,MER/USD,USD,1,0.0001,1.0,MER/USD,1.0
 ftx,MIDBEARUSD,crypto,MIDBEAR/USD,USD,1,0.00000005,1000.0,MIDBEAR/USD,1000.0
-ftx,MIDBULLUSD,crypto,MIDBULL/USD,USD,1,0.05,0.001,MIDBULL/USD,0.001
+ftx,MIDBULLUSD,crypto,MIDBULL/USD,USD,1,0.01,0.001,MIDBULL/USD,0.001
 ftx,MIDHALFUSD,crypto,MIDHALF/USD,USD,1,1.0,0.00001,MIDHALF/USD,0.00001
 ftx,MIDHEDGEUSD,crypto,MIDHEDGE/USD,USD,1,0.01,0.001,MIDHEDGE/USD,0.001
 ftx,MKRUSD,crypto,MKR/USD,USD,1,0.5,0.001,MKR/USD,0.001
 ftx,MKRUSDT,crypto,MKR/USDT,USDT,1,0.5,0.001,MKR/USDT,0.001
-ftx,MKRBEARUSD,crypto,MKRBEAR/USD,USD,1,0.0000001,100.0,MKRBEAR/USD,100.0
-ftx,MKRBULLUSD,crypto,MKRBULL/USD,USD,1,0.025,0.001,MKRBULL/USD,0.001
+ftx,MKRBEARUSD,crypto,MKRBEAR/USD,USD,1,0.00000005,1000.0,MKRBEAR/USD,1000.0
+ftx,MKRBULLUSD,crypto,MKRBULL/USD,USD,1,0.005,0.01,MKRBULL/USD,0.01
 ftx,MNGOUSD,crypto,MNGO/USD,USD,1,0.000025,10.0,MNGO/USD,10.0
 ftx,MOBUSD,crypto,MOB/USD,USD,1,0.0005,0.5,MOB/USD,0.5
 ftx,MOBUSDT,crypto,MOB/USDT,USDT,1,0.0005,0.5,MOB/USDT,0.5
 ftx,MRNAUSD,crypto,MRNA/USD,USD,1,0.025,0.005,MRNA/USD,0.005
+ftx,MSOLUSD,crypto,MSOL/USD,USD,1,0.025,0.01,MSOL/USD,0.01
 ftx,MSTRUSD,crypto,MSTR/USD,USD,1,0.25,0.005,MSTR/USD,0.005
 ftx,MTAUSD,crypto,MTA/USD,USD,1,0.0005,1.0,MTA/USD,1.0
 ftx,MTAUSDT,crypto,MTA/USDT,USDT,1,0.0005,1.0,MTA/USDT,1.0
 ftx,MTLUSD,crypto,MTL/USD,USD,1,0.0005,0.1,MTL/USD,0.1
+ftx,NEXOUSD,crypto,NEXO/USD,USD,1,0.00025,1.0,NEXO/USD,1.0
 ftx,NFLXUSD,crypto,NFLX/USD,USD,1,0.01,0.01,NFLX/USD,0.01
 ftx,NIOUSD,crypto,NIO/USD,USD,1,0.05,0.005,NIO/USD,0.005
 ftx,NOKUSD,crypto,NOK/USD,USD,1,0.0005,0.1,NOK/USD,0.1
 ftx,NVDAUSD,crypto,NVDA/USD,USD,1,0.05,0.0025,NVDA/USD,0.0025
 ftx,OKBUSD,crypto,OKB/USD,USD,1,0.0005,0.1,OKB/USD,0.1
-ftx,OKBBEARUSD,crypto,OKBBEAR/USD,USD,1,0.00000001,100000.0,OKBBEAR/USD,100000.0
+ftx,OKBBEARUSD,crypto,OKBBEAR/USD,USD,1,0.00000001,1000000.0,OKBBEAR/USD,1000000.0
 ftx,OKBBULLUSD,crypto,OKBBULL/USD,USD,1,0.005,0.01,OKBBULL/USD,0.01
 ftx,OKBHALFUSD,crypto,OKBHALF/USD,USD,1,1.0,0.00001,OKBHALF/USD,0.00001
-ftx,OKBHEDGEUSD,crypto,OKBHEDGE/USD,USD,1,0.0025,0.01,OKBHEDGE/USD,0.01
+ftx,OKBHEDGEUSD,crypto,OKBHEDGE/USD,USD,1,0.001,0.01,OKBHEDGE/USD,0.01
 ftx,OMGUSD,crypto,OMG/USD,USD,1,0.0005,0.5,OMG/USD,0.5
 ftx,ORBSUSD,crypto,ORBS/USD,USD,1,0.000025,10.0,ORBS/USD,10.0
 ftx,OXYUSD,crypto,OXY/USD,USD,1,0.0001,1.0,OXY/USD,1.0
@@ -3145,21 +3173,28 @@ ftx,PAXGBULLUSD,crypto,PAXGBULL/USD,USD,1,1.0,0.00001,PAXGBULL/USD,0.00001
 ftx,PAXGHALFUSD,crypto,PAXGHALF/USD,USD,1,0.25,0.00001,PAXGHALF/USD,0.00001
 ftx,PAXGHEDGEUSD,crypto,PAXGHEDGE/USD,USD,1,0.5,0.00001,PAXGHEDGE/USD,0.00001
 ftx,PENNUSD,crypto,PENN/USD,USD,1,0.01,0.01,PENN/USD,0.01
+ftx,PEOPLEUSD,crypto,PEOPLE/USD,USD,1,0.00001,10.0,PEOPLE/USD,10.0
 ftx,PERPUSD,crypto,PERP/USD,USD,1,0.001,0.1,PERP/USD,0.1
 ftx,PFEUSD,crypto,PFE/USD,USD,1,0.01,0.01,PFE/USD,0.01
 ftx,POLISUSD,crypto,POLIS/USD,USD,1,0.000025,0.1,POLIS/USD,0.1
+ftx,PORTUSD,crypto,PORT/USD,USD,1,0.0005,0.1,PORT/USD,0.1
+ftx,PRISMUSD,crypto,PRISM/USD,USD,1,0.000005,10.0,PRISM/USD,10.0
 ftx,PRIVBEARUSD,crypto,PRIVBEAR/USD,USD,1,0.00001,10.0,PRIVBEAR/USD,10.0
-ftx,PRIVBULLUSD,crypto,PRIVBULL/USD,USD,1,0.01,0.001,PRIVBULL/USD,0.001
-ftx,PRIVHALFUSD,crypto,PRIVHALF/USD,USD,1,1.0,0.00001,PRIVHALF/USD,0.00001
-ftx,PRIVHEDGEUSD,crypto,PRIVHEDGE/USD,USD,1,0.025,0.001,PRIVHEDGE/USD,0.001
+ftx,PRIVBULLUSD,crypto,PRIVBULL/USD,USD,1,0.001,0.01,PRIVBULL/USD,0.01
+ftx,PRIVHALFUSD,crypto,PRIVHALF/USD,USD,1,0.5,0.00001,PRIVHALF/USD,0.00001
+ftx,PRIVHEDGEUSD,crypto,PRIVHEDGE/USD,USD,1,0.05,0.0001,PRIVHEDGE/USD,0.0001
 ftx,PROMUSD,crypto,PROM/USD,USD,1,0.005,0.01,PROM/USD,0.01
 ftx,PSGUSD,crypto,PSG/USD,USD,1,0.0025,0.1,PSG/USD,0.1
+ftx,PTUUSD,crypto,PTU/USD,USD,1,0.00025,1.0,PTU/USD,1.0
 ftx,PUNDIXUSD,crypto,PUNDIX/USD,USD,1,0.0005,0.1,PUNDIX/USD,0.1
 ftx,PYPLUSD,crypto,PYPL/USD,USD,1,0.025,0.005,PYPL/USD,0.005
+ftx,QIUSD,crypto,QI/USD,USD,1,0.00001,10.0,QI/USD,10.0
 ftx,RAMPUSD,crypto,RAMP/USD,USD,1,0.0001,1.0,RAMP/USD,1.0
 ftx,RAYUSD,crypto,RAY/USD,USD,1,0.0001,1.0,RAY/USD,1.0
+ftx,REALUSD,crypto,REAL/USD,USD,1,0.0025,0.1,REAL/USD,0.1
 ftx,REEFUSD,crypto,REEF/USD,USD,1,0.000005,10.0,REEF/USD,10.0
 ftx,RENUSD,crypto,REN/USD,USD,1,0.00005,1.0,REN/USD,1.0
+ftx,RNDRUSD,crypto,RNDR/USD,USD,1,0.0005,0.1,RNDR/USD,0.1
 ftx,ROOKUSD,crypto,ROOK/USD,USD,1,0.05,0.001,ROOK/USD,0.001
 ftx,ROOKUSDT,crypto,ROOK/USDT,USDT,1,0.05,0.001,ROOK/USDT,0.001
 ftx,RSRUSD,crypto,RSR/USD,USD,1,0.000005,10.0,RSR/USD,10.0
@@ -3169,6 +3204,7 @@ ftx,SANDUSD,crypto,SAND/USD,USD,1,0.00005,1.0,SAND/USD,1.0
 ftx,SECOUSD,crypto,SECO/USD,USD,1,0.0001,1.0,SECO/USD,1.0
 ftx,SHIBUSD,crypto,SHIB/USD,USD,1,0.00000001,100000.0,SHIB/USD,100000.0
 ftx,SKLUSD,crypto,SKL/USD,USD,1,0.0001,1.0,SKL/USD,1.0
+ftx,SLNDUSD,crypto,SLND/USD,USD,1,0.001,0.1,SLND/USD,0.1
 ftx,SLPUSD,crypto,SLP/USD,USD,1,0.000025,10.0,SLP/USD,10.0
 ftx,SLRSUSD,crypto,SLRS/USD,USD,1,0.00001,1.0,SLRS/USD,1.0
 ftx,SLVUSD,crypto,SLV/USD,USD,1,0.0025,0.1,SLV/USD,0.1
@@ -3177,52 +3213,58 @@ ftx,SNYUSD,crypto,SNY/USD,USD,1,0.0001,1.0,SNY/USD,1.0
 ftx,SOLBTC,crypto,SOL/BTC,BTC,1,0.00000025,0.01,SOL/BTC,0.01
 ftx,SOLUSD,crypto,SOL/USD,USD,1,0.0025,0.01,SOL/USD,0.01
 ftx,SOLUSDT,crypto,SOL/USDT,USDT,1,0.0025,0.01,SOL/USDT,0.01
+ftx,SOSUSD,crypto,SOS/USD,USD,1,0.00000001,100000.0,SOS/USD,100000.0
+ftx,SPELLUSD,crypto,SPELL/USD,USD,1,0.0000025,100.0,SPELL/USD,100.0
 ftx,SPYUSD,crypto,SPY/USD,USD,1,0.01,0.001,SPY/USD,0.001
 ftx,SQUSD,crypto,SQ/USD,USD,1,0.025,0.005,SQ/USD,0.005
 ftx,SRMUSD,crypto,SRM/USD,USD,1,0.001,1.0,SRM/USD,1.0
 ftx,SRMUSDT,crypto,SRM/USDT,USDT,1,0.001,1.0,SRM/USDT,1.0
+ftx,STARSUSD,crypto,STARS/USD,USD,1,0.00005,1.0,STARS/USD,1.0
 ftx,STEPUSD,crypto,STEP/USD,USD,1,0.0005,0.1,STEP/USD,0.1
+ftx,STETHUSD,crypto,STETH/USD,USD,1,0.5,0.0001,STETH/USD,0.0001
 ftx,STMXUSD,crypto,STMX/USD,USD,1,0.000005,10.0,STMX/USD,10.0
 ftx,STORJUSD,crypto,STORJ/USD,USD,1,0.0005,0.1,STORJ/USD,0.1
+ftx,STSOLUSD,crypto,STSOL/USD,USD,1,0.025,0.01,STSOL/USD,0.01
 ftx,SUNUSD,crypto,SUN/USD,USD,1,0.0000005,0.001,SUN/USD,0.001
 ftx,SUSHIBTC,crypto,SUSHI/BTC,BTC,1,0.00000001,0.5,SUSHI/BTC,0.5
 ftx,SUSHIUSD,crypto,SUSHI/USD,USD,1,0.0001,0.5,SUSHI/USD,0.5
 ftx,SUSHIUSDT,crypto,SUSHI/USDT,USDT,1,0.0001,0.5,SUSHI/USDT,0.5
 ftx,SUSHIBEARUSD,crypto,SUSHIBEAR/USD,USD,1,0.00000001,10000000.0,SUSHIBEAR/USD,10000000.0
-ftx,SUSHIBULLUSD,crypto,SUSHIBULL/USD,USD,1,0.00000005,1000.0,SUSHIBULL/USD,1000.0
+ftx,SUSHIBULLUSD,crypto,SUSHIBULL/USD,USD,1,0.00000001,10000.0,SUSHIBULL/USD,10000.0
 ftx,SXPBTC,crypto,SXP/BTC,BTC,1,0.00000001,0.1,SXP/BTC,0.1
 ftx,SXPUSD,crypto,SXP/USD,USD,1,0.0005,0.1,SXP/USD,0.1
 ftx,SXPUSDT,crypto,SXP/USDT,USDT,1,0.0005,0.1,SXP/USDT,0.1
-ftx,SXPBEARUSD,crypto,SXPBEAR/USD,USD,1,0.00000001,1000000.0,SXPBEAR/USD,1000000.0
-ftx,SXPBULLUSD,crypto,SXPBULL/USD,USD,1,0.0000025,10.0,SXPBULL/USD,10.0
+ftx,SXPBEARUSD,crypto,SXPBEAR/USD,USD,1,0.00000001,10000000.0,SXPBEAR/USD,10000000.0
+ftx,SXPBULLUSD,crypto,SXPBULL/USD,USD,1,0.0000001,1000.0,SXPBULL/USD,1000.0
 ftx,SXPHALFUSD,crypto,SXPHALF/USD,USD,1,0.5,0.00001,SXPHALF/USD,0.00001
 ftx,SXPHALFUSDT,crypto,SXPHALF/USDT,USDT,1,0.5,0.00001,SXPHALF/USDT,0.00001
 ftx,SXPHEDGEUSD,crypto,SXPHEDGE/USD,USD,1,0.25,0.0001,SXPHEDGE/USD,0.0001
 ftx,THETABEARUSD,crypto,THETABEAR/USD,USD,1,0.00000001,10000000.0,THETABEAR/USD,10000000.0
-ftx,THETABULLUSD,crypto,THETABULL/USD,USD,1,0.05,0.001,THETABULL/USD,0.001
-ftx,THETAHALFUSD,crypto,THETAHALF/USD,USD,1,2.5,0.000001,THETAHALF/USD,0.000001
-ftx,THETAHEDGEUSD,crypto,THETAHEDGE/USD,USD,1,0.001,0.01,THETAHEDGE/USD,0.01
+ftx,THETABULLUSD,crypto,THETABULL/USD,USD,1,0.0005,0.1,THETABULL/USD,0.1
+ftx,THETAHALFUSD,crypto,THETAHALF/USD,USD,1,1.0,0.00001,THETAHALF/USD,0.00001
+ftx,THETAHEDGEUSD,crypto,THETAHEDGE/USD,USD,1,0.0025,0.01,THETAHEDGE/USD,0.01
 ftx,TLMUSD,crypto,TLM/USD,USD,1,0.00005,1.0,TLM/USD,1.0
 ftx,TLRYUSD,crypto,TLRY/USD,USD,1,0.001,0.1,TLRY/USD,0.1
 ftx,TOMOUSD,crypto,TOMO/USD,USD,1,0.00005,0.1,TOMO/USD,0.1
 ftx,TOMOUSDT,crypto,TOMO/USDT,USDT,1,0.00005,0.1,TOMO/USDT,0.1
-ftx,TOMOBEAR2021USD,crypto,TOMOBEAR2021/USD,USD,1,0.005,0.01,TOMOBEAR2021/USD,0.01
-ftx,TOMOBULLUSD,crypto,TOMOBULL/USD,USD,1,0.00000025,100.0,TOMOBULL/USD,100.0
+ftx,TOMOBEAR2021USD,crypto,TOMOBEAR2021/USD,USD,1,0.0025,0.01,TOMOBEAR2021/USD,0.01
+ftx,TOMOBULLUSD,crypto,TOMOBULL/USD,USD,1,0.00000001,10000.0,TOMOBULL/USD,10000.0
 ftx,TOMOHALFUSD,crypto,TOMOHALF/USD,USD,1,1.0,0.00001,TOMOHALF/USD,0.00001
 ftx,TOMOHEDGEUSD,crypto,TOMOHEDGE/USD,USD,1,0.001,0.01,TOMOHEDGE/USD,0.01
+ftx,TONCOINUSD,crypto,TONCOIN/USD,USD,1,0.0005,0.1,TONCOIN/USD,0.1
 ftx,TRUUSD,crypto,TRU/USD,USD,1,0.000025,1.0,TRU/USD,1.0
 ftx,TRUUSDT,crypto,TRU/USDT,USDT,1,0.000025,1.0,TRU/USDT,1.0
 ftx,TRXBTC,crypto,TRX/BTC,BTC,1,0.00000001,1.0,TRX/BTC,1.0
 ftx,TRXUSD,crypto,TRX/USD,USD,1,0.0000025,1.0,TRX/USD,1.0
 ftx,TRXUSDT,crypto,TRX/USDT,USDT,1,0.0000025,1.0,TRX/USDT,1.0
-ftx,TRXBEARUSD,crypto,TRXBEAR/USD,USD,1,0.00000001,1000000.0,TRXBEAR/USD,1000000.0
-ftx,TRXBULLUSD,crypto,TRXBULL/USD,USD,1,0.0001,0.1,TRXBULL/USD,0.1
+ftx,TRXBEARUSD,crypto,TRXBEAR/USD,USD,1,0.00000001,100000.0,TRXBEAR/USD,100000.0
+ftx,TRXBULLUSD,crypto,TRXBULL/USD,USD,1,0.000025,1.0,TRXBULL/USD,1.0
 ftx,TRXHALFUSD,crypto,TRXHALF/USD,USD,1,1.0,0.00001,TRXHALF/USD,0.00001
-ftx,TRXHEDGEUSD,crypto,TRXHEDGE/USD,USD,1,0.005,0.001,TRXHEDGE/USD,0.001
+ftx,TRXHEDGEUSD,crypto,TRXHEDGE/USD,USD,1,0.01,0.001,TRXHEDGE/USD,0.001
 ftx,TRYBUSD,crypto,TRYB/USD,USD,1,0.00001,0.1,TRYB/USD,0.1
 ftx,TRYBBEARUSD,crypto,TRYBBEAR/USD,USD,1,1.0,0.00001,TRYBBEAR/USD,0.00001
-ftx,TRYBBULLUSD,crypto,TRYBBULL/USD,USD,1,2.5,0.00001,TRYBBULL/USD,0.00001
-ftx,TRYBHALFUSD,crypto,TRYBHALF/USD,USD,1,0.5,0.00001,TRYBHALF/USD,0.00001
+ftx,TRYBBULLUSD,crypto,TRYBBULL/USD,USD,1,0.5,0.0001,TRYBBULL/USD,0.0001
+ftx,TRYBHALFUSD,crypto,TRYBHALF/USD,USD,1,0.25,0.00001,TRYBHALF/USD,0.00001
 ftx,TRYBHEDGEUSD,crypto,TRYBHEDGE/USD,USD,1,0.5,0.00001,TRYBHEDGE/USD,0.00001
 ftx,TSLABTC,crypto,TSLA/BTC,BTC,1,0.0000025,0.01,TSLA/BTC,0.01
 ftx,TSLADOGE,crypto,TSLA/DOGE,DOGE,1,1.0,0.01,TSLA/DOGE,0.01
@@ -3236,8 +3278,8 @@ ftx,UBXTUSDT,crypto,UBXT/USDT,USDT,1,0.000005,1.0,UBXT/USDT,1.0
 ftx,UNIBTC,crypto,UNI/BTC,BTC,1,0.00000005,0.05,UNI/BTC,0.05
 ftx,UNIUSD,crypto,UNI/USD,USD,1,0.001,0.1,UNI/USD,0.1
 ftx,UNIUSDT,crypto,UNI/USDT,USDT,1,0.001,0.05,UNI/USDT,0.05
-ftx,UNISWAPBEARUSD,crypto,UNISWAPBEAR/USD,USD,1,0.0000025,10.0,UNISWAPBEAR/USD,10.0
-ftx,UNISWAPBULLUSD,crypto,UNISWAPBULL/USD,USD,1,0.5,0.0001,UNISWAPBULL/USD,0.0001
+ftx,UNISWAPBEARUSD,crypto,UNISWAPBEAR/USD,USD,1,0.000005,10.0,UNISWAPBEAR/USD,10.0
+ftx,UNISWAPBULLUSD,crypto,UNISWAPBULL/USD,USD,1,0.05,0.001,UNISWAPBULL/USD,0.001
 ftx,USDTUSD,crypto,USDT/USD,USD,1,0.0001,0.01,USDT/USD,0.01
 ftx,USDTBEARUSD,crypto,USDTBEAR/USD,USD,1,1.0,0.00001,USDTBEAR/USD,0.00001
 ftx,USDTBULLUSD,crypto,USDTBULL/USD,USD,1,2.5,0.00001,USDTBULL/USD,0.00001
@@ -3246,13 +3288,14 @@ ftx,USDTHEDGEUSD,crypto,USDTHEDGE/USD,USD,1,0.5,0.00001,USDTHEDGE/USD,0.00001
 ftx,USOUSD,crypto,USO/USD,USD,1,0.005,0.01,USO/USD,0.01
 ftx,VETBEARUSD,crypto,VETBEAR/USD,USD,1,0.00000001,10000.0,VETBEAR/USD,10000.0
 ftx,VETBEARUSDT,crypto,VETBEAR/USDT,USDT,1,0.00000001,10000.0,VETBEAR/USDT,10000.0
-ftx,VETBULLUSD,crypto,VETBULL/USD,USD,1,0.0005,0.1,VETBULL/USD,0.1
-ftx,VETBULLUSDT,crypto,VETBULL/USDT,USDT,1,0.0005,0.1,VETBULL/USDT,0.1
+ftx,VETBULLUSD,crypto,VETBULL/USD,USD,1,0.000025,1.0,VETBULL/USD,1.0
+ftx,VETBULLUSDT,crypto,VETBULL/USDT,USDT,1,0.000025,1.0,VETBULL/USDT,1.0
 ftx,VETHEDGEUSD,crypto,VETHEDGE/USD,USD,1,0.5,0.00001,VETHEDGE/USD,0.00001
 ftx,VGXUSD,crypto,VGX/USD,USD,1,0.00025,1.0,VGX/USD,1.0
 ftx,WAVESUSD,crypto,WAVES/USD,USD,1,0.0005,0.5,WAVES/USD,0.5
 ftx,WBTCBTC,crypto,WBTC/BTC,BTC,1,0.0001,0.0001,WBTC/BTC,0.0001
 ftx,WBTCUSD,crypto,WBTC/USD,USD,1,1.0,0.0001,WBTC/USD,0.0001
+ftx,WNDRUSD,crypto,WNDR/USD,USD,1,0.00025,1.0,WNDR/USD,1.0
 ftx,WRXUSD,crypto,WRX/USD,USD,1,0.00001,1.0,WRX/USD,1.0
 ftx,WRXUSDT,crypto,WRX/USDT,USDT,1,0.00001,1.0,WRX/USDT,1.0
 ftx,XAUTUSD,crypto,XAUT/USD,USD,1,0.1,0.0001,XAUT/USD,0.0001
@@ -3261,28 +3304,88 @@ ftx,XAUTBEARUSD,crypto,XAUTBEAR/USD,USD,1,1.0,0.00001,XAUTBEAR/USD,0.00001
 ftx,XAUTBULLUSD,crypto,XAUTBULL/USD,USD,1,2.5,0.00001,XAUTBULL/USD,0.00001
 ftx,XAUTHALFUSD,crypto,XAUTHALF/USD,USD,1,0.25,0.00001,XAUTHALF/USD,0.00001
 ftx,XAUTHEDGEUSD,crypto,XAUTHEDGE/USD,USD,1,0.5,0.00001,XAUTHEDGE/USD,0.00001
-ftx,XLMBEARUSD,crypto,XLMBEAR/USD,USD,1,0.000025,1.0,XLMBEAR/USD,1.0
-ftx,XLMBULLUSD,crypto,XLMBULL/USD,USD,1,0.0005,0.1,XLMBULL/USD,0.1
+ftx,XLMBEARUSD,crypto,XLMBEAR/USD,USD,1,0.00001,10.0,XLMBEAR/USD,10.0
+ftx,XLMBULLUSD,crypto,XLMBULL/USD,USD,1,0.00005,1.0,XLMBULL/USD,1.0
 ftx,XRPBTC,crypto,XRP/BTC,BTC,1,0.00000001,1.0,XRP/BTC,1.0
 ftx,XRPUSD,crypto,XRP/USD,USD,1,0.000025,1.0,XRP/USD,1.0
 ftx,XRPUSDT,crypto,XRP/USDT,USDT,1,0.000025,1.0,XRP/USDT,1.0
 ftx,XRPBEARUSD,crypto,XRPBEAR/USD,USD,1,0.00000001,1000000.0,XRPBEAR/USD,1000000.0
 ftx,XRPBEARUSDT,crypto,XRPBEAR/USDT,USDT,1,0.00000001,1000000.0,XRPBEAR/USDT,1000000.0
-ftx,XRPBULLUSD,crypto,XRPBULL/USD,USD,1,0.0000025,10.0,XRPBULL/USD,10.0
-ftx,XRPBULLUSDT,crypto,XRPBULL/USDT,USDT,1,0.0000025,10.0,XRPBULL/USDT,10.0
+ftx,XRPBULLUSD,crypto,XRPBULL/USD,USD,1,0.00000025,100.0,XRPBULL/USD,100.0
+ftx,XRPBULLUSDT,crypto,XRPBULL/USDT,USDT,1,0.00000025,100.0,XRPBULL/USDT,100.0
 ftx,XRPHALFUSD,crypto,XRPHALF/USD,USD,1,1.0,0.00001,XRPHALF/USD,0.00001
 ftx,XRPHEDGEUSD,crypto,XRPHEDGE/USD,USD,1,0.005,0.001,XRPHEDGE/USD,0.001
 ftx,XTZBEARUSD,crypto,XTZBEAR/USD,USD,1,0.00000001,100000.0,XTZBEAR/USD,100000.0
 ftx,XTZBEARUSDT,crypto,XTZBEAR/USDT,USDT,1,0.00000001,100000.0,XTZBEAR/USDT,100000.0
-ftx,XTZBULLUSD,crypto,XTZBULL/USD,USD,1,0.0001,0.1,XTZBULL/USD,0.1
-ftx,XTZBULLUSDT,crypto,XTZBULL/USDT,USDT,1,0.0001,0.1,XTZBULL/USDT,0.1
+ftx,XTZBULLUSD,crypto,XTZBULL/USD,USD,1,0.0000025,10.0,XTZBULL/USD,10.0
+ftx,XTZBULLUSDT,crypto,XTZBULL/USDT,USDT,1,0.0000025,10.0,XTZBULL/USDT,10.0
 ftx,XTZHALFUSD,crypto,XTZHALF/USD,USD,1,0.5,0.00001,XTZHALF/USD,0.00001
 ftx,XTZHEDGEUSD,crypto,XTZHEDGE/USD,USD,1,0.005,0.001,XTZHEDGE/USD,0.001
 ftx,YFIBTC,crypto,YFI/BTC,BTC,1,0.00025,0.001,YFI/BTC,0.001
 ftx,YFIUSD,crypto,YFI/USD,USD,1,5.0,0.001,YFI/USD,0.001
 ftx,YFIUSDT,crypto,YFI/USDT,USDT,1,5.0,0.001,YFI/USDT,0.001
 ftx,YFIIUSD,crypto,YFII/USD,USD,1,0.25,0.001,YFII/USD,0.001
-ftx,ZECBEARUSD,crypto,ZECBEAR/USD,USD,1,0.00025,0.1,ZECBEAR/USD,0.1
-ftx,ZECBULLUSD,crypto,ZECBULL/USD,USD,1,0.0001,0.1,ZECBULL/USD,0.1
+ftx,ZECBEARUSD,crypto,ZECBEAR/USD,USD,1,0.000025,1.0,ZECBEAR/USD,1.0
+ftx,ZECBULLUSD,crypto,ZECBULL/USD,USD,1,0.000025,1.0,ZECBULL/USD,1.0
 ftx,ZMUSD,crypto,ZM/USD,USD,1,0.01,0.01,ZM/USD,0.01
 ftx,ZRXUSD,crypto,ZRX/USD,USD,1,0.00025,1.0,ZRX/USD,1.0
+
+ftxus,AAVEUSD,crypto,AAVE/USD,USD,1,0.01,0.01,AAVE/USD,0.01
+ftxus,AAVEUSDT,crypto,AAVE/USDT,USDT,1,0.01,0.01,AAVE/USDT,0.01
+ftxus,AUDUSD,crypto,AUD/USD,USD,1,0.0001,1.0,AUD/USD,1.0
+ftxus,BATUSD,crypto,BAT/USD,USD,1,0.000025,1.0,BAT/USD,1.0
+ftxus,BCHBTC,crypto,BCH/BTC,BTC,1,0.000001,0.001,BCH/BTC,0.001
+ftxus,BCHUSD,crypto,BCH/USD,USD,1,0.025,0.001,BCH/USD,0.001
+ftxus,BCHUSDT,crypto,BCH/USDT,USDT,1,0.025,0.001,BCH/USDT,0.001
+ftxus,BRZUSD,crypto,BRZ/USD,USD,1,0.00001,1.0,BRZ/USD,1.0
+ftxus,BRZUSDT,crypto,BRZ/USDT,USDT,1,0.00001,1.0,BRZ/USDT,1.0
+ftxus,BTCAUD,crypto,BTC/AUD,AUD,1,1.0,0.0001,BTC/AUD,0.0001
+ftxus,BTCBRZ,crypto,BTC/BRZ,BRZ,1,5.0,0.0001,BTC/BRZ,0.0001
+ftxus,BTCEUR,crypto,BTC/EUR,EUR,1,1.0,0.0001,BTC/EUR,0.0001
+ftxus,BTCUSD,crypto,BTC/USD,USD,1,1.0,0.0001,BTC/USD,0.0001
+ftxus,BTCUSDT,crypto,BTC/USDT,USDT,1,1.0,0.0001,BTC/USDT,0.0001
+ftxus,CADUSD,crypto,CAD/USD,USD,1,0.0001,1.0,CAD/USD,1.0
+ftxus,CUSDTUSD,crypto,CUSDT/USD,USD,1,0.0000025,1.0,CUSDT/USD,1.0
+ftxus,CUSDTUSDT,crypto,CUSDT/USDT,USDT,1,0.0000025,1.0,CUSDT/USDT,1.0
+ftxus,DAIUSD,crypto,DAI/USD,USD,1,0.0001,0.1,DAI/USD,0.1
+ftxus,DAIUSDT,crypto,DAI/USDT,USDT,1,0.0001,0.1,DAI/USDT,0.1
+ftxus,DOGEBTC,crypto,DOGE/BTC,BTC,1,0.00000001,1.0,DOGE/BTC,1.0
+ftxus,DOGEUSD,crypto,DOGE/USD,USD,1,0.0000005,1.0,DOGE/USD,1.0
+ftxus,DOGEUSDT,crypto,DOGE/USDT,USDT,1,0.0000005,1.0,DOGE/USDT,1.0
+ftxus,ETHAUD,crypto,ETH/AUD,AUD,1,0.1,0.001,ETH/AUD,0.001
+ftxus,ETHBTC,crypto,ETH/BTC,BTC,1,0.0000025,0.001,ETH/BTC,0.001
+ftxus,ETHEUR,crypto,ETH/EUR,EUR,1,0.1,0.001,ETH/EUR,0.001
+ftxus,ETHUSD,crypto,ETH/USD,USD,1,0.1,0.001,ETH/USD,0.001
+ftxus,ETHUSDT,crypto,ETH/USDT,USDT,1,0.1,0.001,ETH/USDT,0.001
+ftxus,EURUSD,crypto,EUR/USD,USD,1,0.0001,1.0,EUR/USD,1.0
+ftxus,GBPUSD,crypto,GBP/USD,USD,1,0.0001,1.0,GBP/USD,1.0
+ftxus,GRTUSD,crypto,GRT/USD,USD,1,0.00005,1.0,GRT/USD,1.0
+ftxus,KSHIBUSD,crypto,KSHIB/USD,USD,1,0.000005,10.0,KSHIB/USD,10.0
+ftxus,LINKBTC,crypto,LINK/BTC,BTC,1,0.00000005,0.1,LINK/BTC,0.1
+ftxus,LINKUSD,crypto,LINK/USD,USD,1,0.0005,0.1,LINK/USD,0.1
+ftxus,LINKUSDT,crypto,LINK/USDT,USDT,1,0.0005,0.1,LINK/USDT,0.1
+ftxus,LTCBTC,crypto,LTC/BTC,BTC,1,0.00000025,0.01,LTC/BTC,0.01
+ftxus,LTCUSD,crypto,LTC/USD,USD,1,0.005,0.01,LTC/USD,0.01
+ftxus,LTCUSDT,crypto,LTC/USDT,USDT,1,0.005,0.01,LTC/USDT,0.01
+ftxus,MATICBTC,crypto,MATIC/BTC,BTC,1,0.00000001,10.0,MATIC/BTC,10.0
+ftxus,MATICUSD,crypto,MATIC/USD,USD,1,0.000001,10.0,MATIC/USD,10.0
+ftxus,MKRUSD,crypto,MKR/USD,USD,1,0.5,0.001,MKR/USD,0.001
+ftxus,PAXGUSD,crypto,PAXG/USD,USD,1,0.1,0.0001,PAXG/USD,0.0001
+ftxus,PAXGUSDT,crypto,PAXG/USDT,USDT,1,0.1,0.0001,PAXG/USDT,0.0001
+ftxus,SHIBUSD,crypto,SHIB/USD,USD,1,0.00000001,100000.0,SHIB/USD,100000.0
+ftxus,SOLBTC,crypto,SOL/BTC,BTC,1,0.00000025,0.01,SOL/BTC,0.01
+ftxus,SOLUSD,crypto,SOL/USD,USD,1,0.0025,0.01,SOL/USD,0.01
+ftxus,SOLUSDT,crypto,SOL/USDT,USDT,1,0.0025,0.01,SOL/USDT,0.01
+ftxus,SUSHIBTC,crypto,SUSHI/BTC,BTC,1,0.00000001,0.5,SUSHI/BTC,0.5
+ftxus,SUSHIUSD,crypto,SUSHI/USD,USD,1,0.0001,0.5,SUSHI/USD,0.5
+ftxus,SUSHIUSDT,crypto,SUSHI/USDT,USDT,1,0.0001,0.5,SUSHI/USDT,0.5
+ftxus,TRXUSD,crypto,TRX/USD,USD,1,0.0000025,1.0,TRX/USD,1.0
+ftxus,TRXUSDT,crypto,TRX/USDT,USDT,1,0.0000025,1.0,TRX/USDT,1.0
+ftxus,UNIBTC,crypto,UNI/BTC,BTC,1,0.00000005,0.05,UNI/BTC,0.05
+ftxus,UNIUSD,crypto,UNI/USD,USD,1,0.001,0.1,UNI/USD,0.1
+ftxus,UNIUSDT,crypto,UNI/USDT,USDT,1,0.001,0.05,UNI/USDT,0.05
+ftxus,USDTUSD,crypto,USDT/USD,USD,1,0.0001,0.01,USDT/USD,0.01
+ftxus,WBTCUSD,crypto,WBTC/USD,USD,1,1.0,0.0001,WBTC/USD,0.0001
+ftxus,YFIBTC,crypto,YFI/BTC,BTC,1,0.00025,0.001,YFI/BTC,0.001
+ftxus,YFIUSD,crypto,YFI/USD,USD,1,5.0,0.001,YFI/USD,0.001
+ftxus,YFIUSDT,crypto,YFI/USDT,USDT,1,5.0,0.001,YFI/USDT,0.001

Dockerfile

@@ -8,11 +8,8 @@ FROM quantconnect/lean:foundation
 
 MAINTAINER QuantConnect <contact@quantconnect.com>
 
-#Install Python Tool for Visual Studio Debugger for remote python debugging
-RUN pip install ptvsd
-
-#Install PyDev Debugger for Pycharm for remote python debugging
-RUN pip install pydevd-pycharm~=201.8538.36
+#Install debugpy and PyDevD for remote python debugging
+RUN pip install --no-cache-dir debugpy~=1.5.1 pydevd-pycharm~=201.8538.36
 
 # Install vsdbg for remote C# debugging in Visual Studio and Visual Studio Code
 RUN wget https://aka.ms/getvsdbgsh -O - 2>/dev/null | /bin/sh /dev/stdin -v 16.9.20122.2 -l /root/vsdbg

DockerfileLeanFoundation

@@ -29,10 +29,10 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
 
 # Install IB Gateway: Installs to /root/ibgateway
 RUN mkdir -p /root/ibgateway && \
-    wget https://cdn.quantconnect.com/interactive/ibgateway-latest-standalone-linux-x64.v985.1j.sh && \
-    chmod 777 ibgateway-latest-standalone-linux-x64.v985.1j.sh && \
-    ./ibgateway-latest-standalone-linux-x64.v985.1j.sh -q -dir /root/ibgateway && \
-    rm ibgateway-latest-standalone-linux-x64.v985.1j.sh
+    wget https://cdn.quantconnect.com/interactive/ibgateway-latest-standalone-linux-x64.v10.12.2d.sh && \
+    chmod 777 ibgateway-latest-standalone-linux-x64.v10.12.2d.sh && \
+    ./ibgateway-latest-standalone-linux-x64.v10.12.2d.sh -q -dir /root/ibgateway && \
+    rm ibgateway-latest-standalone-linux-x64.v10.12.2d.sh
 
 # Install dotnet 5 sdk & runtime
 RUN wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb && \
@@ -53,9 +53,9 @@ ENV PATH="/opt/miniconda3/bin:${PATH}"
 RUN wget https://cdn.quantconnect.com/miniconda/${CONDA} && \
     bash ${CONDA} -b -p /opt/miniconda3 && rm -rf ${CONDA} && \
     ln -s /opt/miniconda3/lib/libpython3.6m.so /usr/lib/libpython3.6m.so && \
-    conda update -y conda pip && \
-    pip install --upgrade --no-cache-dir pip && \
-    conda install -y python=3.6.8 && conda clean -y --all
+    conda install -y conda=4.10.3 && \
+    pip install --upgrade --no-cache-dir pip==21.2.2 && \
+    conda install -y python=3.6.8 && conda install -y pip=21.2.2 && conda clean -y --all
 
 # Avoid pip install read timeouts
 ENV PIP_DEFAULT_TIMEOUT=120
@@ -167,7 +167,7 @@ RUN pip install --no-cache-dir      \
     ruptures==1.1.3                 \
     simpy==4.0.1                    \
     scikit-learn-extra==0.2.0       \
-    ray==1.5.1
+    ray==1.9.1
 
 # feature_selector has overly strict dependency version ranges
 # We already installed close-enough versions of all of its dependencies above
@@ -259,10 +259,10 @@ RUN wget https://cdn.quantconnect.com/tigramite/tigramite-4.1.zip && \
     python setup.py install && cd .. && rm -rf tigramite-4.1 && rm tigramite-4.1.zip
 
 # Install H2O: https://www.h2o.ai/download/
-RUN wget https://cdn.quantconnect.com/h2o/h2o-3.30.0.3.zip && \
-    unzip -q h2o-3.30.0.3.zip && \
-    pip install h2o-3.30.0.3/python/h2o-3.30.0.3-py2.py3-none-any.whl && \
-    rm -rf h2o-3.30.0.3 && rm h2o-3.30.0.3.zip
+RUN wget https://cdn.quantconnect.com/h2o/h2o-3.34.0.7.zip && \
+    unzip -q h2o-3.34.0.7.zip && \
+    pip install h2o-3.34.0.7-py2.py3-none-any.whl && \
+    rm h2o-3.34.0.7.zip h2o-3.34.0.7-py2.py3-none-any.whl
 
 # Remove black-listed packages
 RUN pip uninstall -y s3transfer

DockerfileLeanFoundationARM

@@ -27,12 +27,12 @@ RUN apt-get update && apt-fast install -y git libgtk2.0.0 cmake bzip2 curl unzip
 # We update the install script so it doesn't use the bundled JVM
 # The bundled JVM doesn't work on ARM64, so we update it to use the JVM installed in the previous command
 RUN mkdir -p /root/ibgateway && \
-    wget https://cdn.quantconnect.com/interactive/ibgateway-latest-standalone-linux-x64.v985.1j.sh && \
+    wget https://cdn.quantconnect.com/interactive/ibgateway-latest-standalone-linux-x64.v10.12.2d.sh && \
     java_patch_version=$(java -version 2>&1 | head -n 1 | cut -d'_' -f2 | cut -d'"' -f1) && \
-    bbe -e 's|# INSTALL4J_JAVA_HOME_OVERRIDE=|INSTALL4J_JAVA_HOME_OVERRIDE="/usr/lib/jvm/java-1.8.0-openjdk-arm64"|' -e "s|-lt \"152\"|-lt \"$java_patch_version\"|" -e "s|-gt \"152\"|-gt \"$java_patch_version\"|" ibgateway-latest-standalone-linux-x64.v985.1j.sh > ibgateway-stable-standalone-linux-custom-jvm.sh && \
+    bbe -e 's|# INSTALL4J_JAVA_HOME_OVERRIDE=|INSTALL4J_JAVA_HOME_OVERRIDE="/usr/lib/jvm/java-1.8.0-openjdk-arm64"|' -e "s|-lt \"152\"|-lt \"$java_patch_version\"|" -e "s|-gt \"152\"|-gt \"$java_patch_version\"|" ibgateway-latest-standalone-linux-x64.v10.12.2d.sh > ibgateway-stable-standalone-linux-custom-jvm.sh && \
     chmod 777 ibgateway-stable-standalone-linux-custom-jvm.sh && \
     ./ibgateway-stable-standalone-linux-custom-jvm.sh -q -dir /root/ibgateway && \
-    rm ibgateway-latest-standalone-linux-x64.v985.1j.sh ibgateway-stable-standalone-linux-custom-jvm.sh
+    rm ibgateway-latest-standalone-linux-x64.v10.12.2d.sh ibgateway-stable-standalone-linux-custom-jvm.sh
 
 # Install dotnet 5 sdk & runtime
 # The .deb packages don't support ARM, the install script does
@@ -54,9 +54,9 @@ RUN wget -O Miniforge3.sh https://github.com/conda-forge/miniforge/releases/late
     rm Miniforge3.sh && \
     ln -s /opt/miniforge3/lib/libpython3.6m.so /usr/lib/libpython3.6m.so && \
     conda config --append channels https://repo.anaconda.com/pkgs/main && \
-    conda update -y conda pip && \
-    pip install --upgrade --no-cache-dir pip && \
-    conda install -y python=3.6.7 && conda clean -y --all
+    conda install -y conda=4.10.3 && \
+    pip install --upgrade --no-cache-dir pip==21.2.2 && \
+    conda install -y python=3.6.7 && conda install -y pip=21.2.2 && conda clean -y --all
 
 # Avoid pip install read timeouts
 ENV PIP_DEFAULT_TIMEOUT=120
@@ -179,10 +179,10 @@ RUN wget https://cdn.quantconnect.com/tigramite/tigramite-4.1.zip && \
     python setup.py install && cd .. && rm -rf tigramite-4.1 && rm tigramite-4.1.zip
 
 # Install H2O: https://www.h2o.ai/download/
-RUN wget https://cdn.quantconnect.com/h2o/h2o-3.30.0.3.zip && \
-    unzip -q h2o-3.30.0.3.zip && \
-    pip install h2o-3.30.0.3/python/h2o-3.30.0.3-py2.py3-none-any.whl && \
-    rm -rf h2o-3.30.0.3 && rm h2o-3.30.0.3.zip
+RUN wget https://cdn.quantconnect.com/h2o/h2o-3.34.0.7.zip && \
+    unzip -q h2o-3.34.0.7.zip && \
+    pip install h2o-3.34.0.7-py2.py3-none-any.whl && \
+    rm h2o-3.34.0.7.zip h2o-3.34.0.7-py2.py3-none-any.whl
 
 # Remove black-listed packages
 RUN pip uninstall -y s3transfer

Engine/DataFeeds/ApiDataProvider.cs

@@ -42,6 +42,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         private readonly DataPricesList _dataPrices;
         private readonly Api.Api _api;
         private readonly bool _subscribedToEquityMapAndFactorFiles;
+        private readonly bool _subscribedToFutureMapAndFactorFiles;
         private volatile bool _invalidSecurityTypeLog;
 
         /// <summary>
@@ -65,10 +66,18 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             _dataPrices = _api.ReadDataPrices(_organizationId);
             var organization = _api.ReadOrganization(_organizationId);
 
-            // Determine if the user is subscribed to map and factor files (Data product Id 37)
-            if (organization.Products.Where(x => x.Type == ProductType.Data).Any(x => x.Items.Any(x => x.Id == 37)))
+            foreach (var productItem in organization.Products.Where(x => x.Type == ProductType.Data).SelectMany(product => product.Items))
             {
-                _subscribedToEquityMapAndFactorFiles = true;
+                if (productItem.Id == 37)
+                {
+                    // Determine if the user is subscribed to Equity map and factor files (Data product Id 37)
+                    _subscribedToEquityMapAndFactorFiles = true;
+                }
+                else if (productItem.Id == 137)
+                {
+                    // Determine if the user is subscribed to Future map and factor files (Data product Id 137)
+                    _subscribedToFutureMapAndFactorFiles = true;
+                }
             }
 
             // Verify user has agreed to data provider agreements
@@ -154,23 +163,41 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             // Some security types can't be downloaded, lets attempt to extract that information
             if (LeanData.TryParseSecurityType(filePath, out SecurityType securityType) && _unsupportedSecurityType.Contains(securityType))
             {
-                if (!_invalidSecurityTypeLog)
+                // we do support future auxiliary data (map and factor files)
+                if (securityType != SecurityType.Future || !IsAuxiliaryData(filePath))
                 {
-                    // let's log this once. Will still use any existing data on disk
-                    _invalidSecurityTypeLog = true;
-                    Log.Error($"ApiDataProvider(): does not support security types: {string.Join(", ", _unsupportedSecurityType)}");
+                    if (!_invalidSecurityTypeLog)
+                    {
+                        // let's log this once. Will still use any existing data on disk
+                        _invalidSecurityTypeLog = true;
+                        Log.Error($"ApiDataProvider(): does not support security types: {string.Join(", ", _unsupportedSecurityType)}");
+                    }
+                    return false;
                 }
-                return false;
             }
 
             // Only download if it doesn't exist or is out of date.
             // Files are only "out of date" for non date based files (hour, daily, margins, etc.) because this data is stored all in one file
             var shouldDownload = !File.Exists(filePath) || filePath.IsOutOfDate();
 
-            // Final check; If we want to download and the request requires equity data we need to be sure they are subscribed to map and factor files
-            if (shouldDownload && (securityType == SecurityType.Equity || securityType == SecurityType.Option || IsEquitiesAux(filePath)))
+            if (shouldDownload)
             {
-                CheckMapFactorFileSubscription();
+                if (securityType == SecurityType.Future)
+                {
+                    if (!_subscribedToFutureMapAndFactorFiles)
+                    {
+                        throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
+                            "to download Future auxiliary data from QuantConnect. " +
+                            "Please visit https://www.quantconnect.com/datasets/quantconnect-us-futures-security-master for details.");
+                    }
+                }
+                // Final check; If we want to download and the request requires equity data we need to be sure they are subscribed to map and factor files
+                else if (!_subscribedToEquityMapAndFactorFiles && (securityType == SecurityType.Equity || securityType == SecurityType.Option || IsAuxiliaryData(filePath)))
+                {
+                    throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
+                        "to download Equity data from QuantConnect. " +
+                        "Please visit https://www.quantconnect.com/datasets/quantconnect-security-master for details.");
+                }
             }
 
             return shouldDownload;
@@ -198,29 +225,16 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         }
 
         /// <summary>
-        /// Helper method to determine if this filepath is Equity Aux data
+        /// Helper method to determine if this filepath is auxiliary data
         /// </summary>
-        /// <param name="filepath"></param>
-        /// <returns>True if this file is EquitiesAux</returns>
-        private static bool IsEquitiesAux(string filepath)
+        /// <param name="filepath">The target file path</param>
+        /// <returns>True if this file is of auxiliary data</returns>
+        private static bool IsAuxiliaryData(string filepath)
         {
             return filepath.Contains("map_files", StringComparison.InvariantCulture)
                 || filepath.Contains("factor_files", StringComparison.InvariantCulture)
                 || filepath.Contains("fundamental", StringComparison.InvariantCulture)
                 || filepath.Contains("shortable", StringComparison.InvariantCulture);
         }
-
-        /// <summary>
-        /// Helper to check map and factor file subscription, throws if not subscribed.
-        /// </summary>
-        private void CheckMapFactorFileSubscription()
-        {
-            if(!_subscribedToEquityMapAndFactorFiles)
-            {
-                throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
-                    "to download Equity data from QuantConnect. " + 
-                    "Please visit https://www.quantconnect.com/datasets/quantconnect-security-master for details.");
-            }
-        }
     }
 }

Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactory.cs

@@ -31,6 +31,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
     {
         private readonly Func<SubscriptionRequest, IEnumerable<DateTime>> _tradableDaysProvider;
         private readonly IMapFileProvider _mapFileProvider;
+        private readonly IDataCacheProvider _dataCacheProvider;
         private readonly bool _isLiveMode;
 
         /// <summary>
@@ -38,13 +39,10 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
         /// </summary>
         /// <param name="isLiveMode">True for live mode, false otherwise</param>
         /// <param name="mapFileProvider">Used for resolving the correct map files</param>
-        /// <param name="factorFileProvider">Used for getting factor files</param>
-        /// <param name="tradableDaysProvider">Function used to provide the tradable dates to be enumerator.
-        /// Specify null to default to <see cref="SubscriptionRequest.TradableDays"/></param>
-        public BaseDataSubscriptionEnumeratorFactory(bool isLiveMode, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, Func<SubscriptionRequest, IEnumerable<DateTime>> tradableDaysProvider = null)
+        /// <param name="dataCacheProvider">The cache provider instance to use</param>
+        public BaseDataSubscriptionEnumeratorFactory(bool isLiveMode, IMapFileProvider mapFileProvider, IDataCacheProvider dataCacheProvider)
+            : this(isLiveMode, dataCacheProvider)
         {
-            _isLiveMode = isLiveMode;
-            _tradableDaysProvider = tradableDaysProvider ?? (request => request.TradableDays);
             _mapFileProvider = mapFileProvider;
         }
 
@@ -52,12 +50,12 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
         /// Initializes a new instance of the <see cref="BaseDataSubscriptionEnumeratorFactory"/> class
         /// </summary>
         /// <param name="isLiveMode">True for live mode, false otherwise</param>
-        /// <param name="tradableDaysProvider">Function used to provide the tradable dates to be enumerator.
-        /// Specify null to default to <see cref="SubscriptionRequest.TradableDays"/></param>
-        public BaseDataSubscriptionEnumeratorFactory(bool isLiveMode, Func<SubscriptionRequest, IEnumerable<DateTime>> tradableDaysProvider = null)
+        /// <param name="dataCacheProvider">The cache provider instance to use</param>
+        public BaseDataSubscriptionEnumeratorFactory(bool isLiveMode, IDataCacheProvider dataCacheProvider)
         {
             _isLiveMode = isLiveMode;
-            _tradableDaysProvider = tradableDaysProvider ?? (request => request.TradableDays);
+            _dataCacheProvider = dataCacheProvider;
+            _tradableDaysProvider = (request => request.TradableDays);
         }
 
         /// <summary>
@@ -73,28 +71,19 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
             // It has the added benefit of caching any zip files that we request from the filesystem, and reading
             // files contained within the zip file, which the SingleEntryDataCacheProvider does not support.
             var sourceFactory = request.Configuration.GetBaseDataInstance();
-            using (var dataCacheProvider = new ZipDataCacheProvider(dataProvider))
+            foreach (var date in _tradableDaysProvider(request))
             {
-                foreach (var date in _tradableDaysProvider(request))
+                if (sourceFactory.RequiresMapping() && _mapFileProvider != null)
                 {
-                    if (sourceFactory.RequiresMapping() && _mapFileProvider != null)
-                    {
-                        request.Configuration.MappedSymbol = GetMappedSymbol(request.Configuration, date);
-                    }
+                    request.Configuration.MappedSymbol = GetMappedSymbol(request.Configuration, date);
+                }
 
-                    var source = sourceFactory.GetSource(request.Configuration, date, _isLiveMode);
-                    var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, request.Configuration, date, _isLiveMode, sourceFactory, dataProvider);
-                    var entriesForDate = factory.Read(source);
-                    foreach (var entry in entriesForDate)
-                    {
-                        // Fix for Daily/Hour options cases when reading in all equity data from daily/hourly file
-                        if (entry.Time.Date != date)
-                        {
-                            continue;
-                        }
-
-                        yield return entry;
-                    }
+                var source = sourceFactory.GetSource(request.Configuration, date, _isLiveMode);
+                var factory = SubscriptionDataSourceReader.ForSource(source, _dataCacheProvider, request.Configuration, date, _isLiveMode, sourceFactory, dataProvider);
+                var entriesForDate = factory.Read(source);
+                foreach (var entry in entriesForDate)
+                {
+                    yield return entry;
                 }
             }
         }

Engine/DataFeeds/Enumerators/Factories/FuturesChainUniverseSubscriptionEnumeratorFactory.cs

@@ -16,7 +16,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using QuantConnect.Data;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Interfaces;
@@ -32,15 +31,18 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
         private readonly bool _isLiveMode;
 
         private readonly IDataQueueUniverseProvider _symbolUniverse;
+        private readonly IDataCacheProvider _dataCacheProvider;
         private readonly ITimeProvider _timeProvider;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="FuturesChainUniverseSubscriptionEnumeratorFactory"/> class
         /// </summary>
         /// <param name="enumeratorConfigurator">Function used to configure the sub-enumerators before sync (fill-forward/filter/ect...)</param>
-        public FuturesChainUniverseSubscriptionEnumeratorFactory(Func<SubscriptionRequest, IEnumerator<BaseData>, IEnumerator<BaseData>> enumeratorConfigurator)
+        /// <param name="dataCacheProvider">The cache provider instance to use</param>
+        public FuturesChainUniverseSubscriptionEnumeratorFactory(Func<SubscriptionRequest, IEnumerator<BaseData>, IEnumerator<BaseData>> enumeratorConfigurator, IDataCacheProvider dataCacheProvider)
         {
             _isLiveMode = false;
+            _dataCacheProvider = dataCacheProvider;
             _enumeratorConfigurator = enumeratorConfigurator;
         }
 
@@ -67,36 +69,28 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
         {
             if (_isLiveMode)
             {
-                var subscriptionConfiguration = GetSubscriptionConfigurations(request).First();
-                var subscriptionRequest = new SubscriptionRequest(request, configuration: subscriptionConfiguration);
+                var subscriptionRequest = new SubscriptionRequest(request, configuration: GetSubscriptionConfiguration(request));
 
                 return new DataQueueFuturesChainUniverseDataCollectionEnumerator(subscriptionRequest, _symbolUniverse, _timeProvider);
             }
             else
             {
-                var factory = new BaseDataSubscriptionEnumeratorFactory(_isLiveMode);
+                var factory = new BaseDataSubscriptionEnumeratorFactory(_isLiveMode, _dataCacheProvider);
 
-                var enumerators = GetSubscriptionConfigurations(request)
-                    .Select(c => new SubscriptionRequest(request, configuration: c))
-                    .Select(sr => _enumeratorConfigurator(request, factory.CreateEnumerator(sr, dataProvider)));
+                var newRequest = new SubscriptionRequest(request, configuration: GetSubscriptionConfiguration(request));
+                var enumerator = _enumeratorConfigurator(request, factory.CreateEnumerator(newRequest, dataProvider));
 
-                var sync = new SynchronizingEnumerator(enumerators);
-                return new FuturesChainUniverseDataCollectionAggregatorEnumerator(sync, request.Security.Symbol);
+                return new FuturesChainUniverseDataCollectionAggregatorEnumerator(enumerator, request.Security.Symbol);
             }
         }
 
-        private static IEnumerable<SubscriptionDataConfig> GetSubscriptionConfigurations(SubscriptionRequest request)
+        private static SubscriptionDataConfig GetSubscriptionConfiguration(SubscriptionRequest request)
         {
             var config = request.Configuration;
             var resolution = config.Resolution;
 
-            var configurations = new List<SubscriptionDataConfig>
-            {
-                // rewrite the primary to be fill forward
-                new SubscriptionDataConfig(config, resolution: resolution, fillForward: true)
-            };
-
-            return configurations;
+            // rewrite the primary to be fill forward
+            return new SubscriptionDataConfig(config, resolution: resolution, fillForward: true);
         }
     }
-}
+}

Engine/DataFeeds/Enumerators/Factories/SubscriptionDataReaderSubscriptionEnumeratorFactory.cs

@@ -35,7 +35,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
         private readonly bool _isLiveMode;
         private readonly IResultHandler _resultHandler;
         private readonly IFactorFileProvider _factorFileProvider;
-        private readonly ZipDataCacheProvider _zipDataCacheProvider;
+        private readonly IDataCacheProvider _dataCacheProvider;
         private readonly ConcurrentDictionary<Symbol, string> _numericalPrecisionLimitedWarnings;
         private readonly int _numericalPrecisionLimitedWarningsMaxCount = 10;
         private readonly ConcurrentDictionary<Symbol, string> _startDateLimitedWarnings;
@@ -50,14 +50,14 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
         /// <param name="resultHandler">The result handler for the algorithm</param>
         /// <param name="mapFileProvider">The map file provider</param>
         /// <param name="factorFileProvider">The factor file provider</param>
-        /// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
+        /// <param name="cacheProvider">Provider used to get data when it is not present on disk</param>
         /// <param name="tradableDaysProvider">Function used to provide the tradable dates to be enumerator.
         /// Specify null to default to <see cref="SubscriptionRequest.TradableDays"/></param>
         /// <param name="enablePriceScaling">Applies price factor</param>
         public SubscriptionDataReaderSubscriptionEnumeratorFactory(IResultHandler resultHandler,
             IMapFileProvider mapFileProvider,
             IFactorFileProvider factorFileProvider,
-            IDataProvider dataProvider,
+            IDataCacheProvider cacheProvider,
             Func<SubscriptionRequest, IEnumerable<DateTime>> tradableDaysProvider = null,
             bool enablePriceScaling = true
             )
@@ -65,7 +65,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
             _resultHandler = resultHandler;
             _mapFileProvider = mapFileProvider;
             _factorFileProvider = factorFileProvider;
-            _zipDataCacheProvider = new ZipDataCacheProvider(dataProvider, isDataEphemeral: false);
+            _dataCacheProvider = cacheProvider;
             _numericalPrecisionLimitedWarnings = new ConcurrentDictionary<Symbol, string>();
             _startDateLimitedWarnings = new ConcurrentDictionary<Symbol, string>();
             _isLiveMode = false;
@@ -88,7 +88,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
                 _factorFileProvider,
                 _tradableDaysProvider(request),
                 _isLiveMode,
-                 _zipDataCacheProvider,
+                _dataCacheProvider,
                 dataProvider
                 );
 
@@ -159,8 +159,6 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
 
                 _resultHandler.DebugMessage(message);
             }
-
-            _zipDataCacheProvider?.DisposeSafely();
         }
     }
 }

Engine/DataFeeds/Enumerators/FilterEnumerator.cs

@@ -0,0 +1,84 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Enumerator that allow applying a filtering function
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    public class FilterEnumerator<T> : IEnumerator<T>
+    {
+        private readonly IEnumerator<T> _enumerator;
+        private readonly Func<T, bool> _filter;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="enumerator">The underlying enumerator to filter on</param>
+        /// <param name="filter">The filter to apply</param>
+        public FilterEnumerator(IEnumerator<T> enumerator, Func<T, bool> filter)
+        {
+            _enumerator = enumerator;
+            _filter = filter;
+        }
+
+        #region Implementation of IDisposable
+
+        public void Dispose()
+        {
+            _enumerator.Dispose();
+        }
+
+        #endregion
+
+        #region Implementation of IEnumerator
+
+        public bool MoveNext()
+        {
+            // run the enumerator until it passes the specified filter
+            while (_enumerator.MoveNext())
+            {
+                if (_filter(_enumerator.Current))
+                {
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+
+        public T Current
+        {
+            get { return _enumerator.Current; }
+        }
+
+        object IEnumerator.Current
+        {
+            get { return _enumerator.Current; }
+        }
+
+        #endregion
+    }
+}

Engine/DataFeeds/Enumerators/LiveSubscriptionEnumerator.cs

@@ -29,13 +29,20 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
     /// </summary>
     public class LiveSubscriptionEnumerator : IEnumerator<BaseData>
     {
+        private BaseData _current;
         private readonly Symbol _requestedSymbol;
         private SubscriptionDataConfig _currentConfig;
         private IEnumerator<BaseData> _previousEnumerator;
         private IEnumerator<BaseData> _underlyingEnumerator;
 
-        public BaseData Current => _underlyingEnumerator.Current;
+        /// <summary>
+        /// The current data object instance
+        /// </summary>
+        public BaseData Current => _current;
 
+        /// <summary>
+        /// The current data object instance
+        /// </summary>
         object IEnumerator.Current => Current;
 
         /// <summary>
@@ -73,9 +80,14 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
             }
 
             var result = _underlyingEnumerator.MoveNext();
-            if (Current != null)
+            _current = _underlyingEnumerator.Current;
+
+            if (_current != null && _current.Symbol != _requestedSymbol)
             {
-                Current.Symbol = _requestedSymbol;
+                // if we've done some mapping at this layer let's clone the underlying and set the requested symbol,
+                // don't trust the IDQH implementations for data uniqueness, since the configuration could be shared
+                _current = _current.Clone();
+                _current.Symbol = _requestedSymbol;
             }
 
             return result;

Engine/DataFeeds/FileSystemDataFeed.cs

@@ -18,7 +18,6 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using QuantConnect.Data;
-using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Market;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Interfaces;
@@ -44,6 +43,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         private IMapFileProvider _mapFileProvider;
         private IFactorFileProvider _factorFileProvider;
         private IDataProvider _dataProvider;
+        private IDataCacheProvider _cacheProvider;
         private SubscriptionCollection _subscriptions;
         private CancellationTokenSource _cancellationTokenSource = new CancellationTokenSource();
         private SubscriptionDataReaderSubscriptionEnumeratorFactory _subscriptionFactory;
@@ -74,11 +74,12 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             _timeProvider = dataFeedTimeProvider.FrontierTimeProvider;
             _subscriptions = subscriptionManager.DataFeedSubscriptions;
             _cancellationTokenSource = new CancellationTokenSource();
+            _cacheProvider = new ZipDataCacheProvider(dataProvider, isDataEphemeral: false);
             _subscriptionFactory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(
                 _resultHandler,
                 _mapFileProvider,
                 _factorFileProvider,
-                _dataProvider,
+                _cacheProvider,
                 enablePriceScaling: false);
 
             IsActive = true;
@@ -150,13 +151,19 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             {
                 factory = new OptionChainUniverseSubscriptionEnumeratorFactory((req) =>
                 {
-                    var underlyingFactory = new BaseDataSubscriptionEnumeratorFactory(false, _mapFileProvider, _factorFileProvider);
+                    if (!req.Configuration.SecurityType.IsOption())
+                    {
+                        var enumerator = _subscriptionFactory.CreateEnumerator(req, _dataProvider);
+                        enumerator = new FilterEnumerator<BaseData>(enumerator, data => data.DataType != MarketDataType.Auxiliary);
+                        return ConfigureEnumerator(req, true, enumerator);
+                    }
+                    var underlyingFactory = new BaseDataSubscriptionEnumeratorFactory(false, _mapFileProvider, _cacheProvider);
                     return ConfigureEnumerator(req, true, underlyingFactory.CreateEnumerator(req, _dataProvider));
                 });
             }
             if (request.Universe is FuturesChainUniverse)
             {
-                factory = new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e));
+                factory = new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e), _cacheProvider);
             }
 
             // define our data enumerator

Engine/DataFeeds/LiveTradingDataFeed.cs

@@ -192,12 +192,6 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                 IEnumerator<BaseData> enumerator;
                 if (!_channelProvider.ShouldStreamSubscription(request.Configuration))
                 {
-                    if (!Quandl.IsAuthCodeSet)
-                    {
-                        // we're not using the SubscriptionDataReader, so be sure to set the auth token here
-                        Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
-                    }
-
                     if (!Tiingo.IsAuthCodeSet)
                     {
                         // we're not using the SubscriptionDataReader, so be sure to set the auth token here

Engine/DataFeeds/SingleEntryDataCacheProvider.cs

@@ -14,11 +14,14 @@
  *
 */
 
+using System;
 using System.IO;
 using Ionic.Zip;
-using QuantConnect.Interfaces;
-using QuantConnect.Logging;
+using System.Linq;
 using QuantConnect.Util;
+using QuantConnect.Logging;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
 
 namespace QuantConnect.Lean.Engine.DataFeeds
 {
@@ -89,6 +92,22 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             //
         }
 
+        /// <summary>
+        /// Returns a list of zip entries in a provided zip file
+        /// </summary>
+        public List<string> GetZipEntries(string zipFile)
+        {
+            var stream = _dataProvider.Fetch(zipFile);
+            if (stream == null)
+            {
+                throw new ArgumentException($"Failed to create source stream {zipFile}");
+            }
+            var entryNames = Compression.GetZipEntryFileNames(stream).ToList();
+            stream.DisposeSafely();
+
+            return entryNames;
+        }
+
         /// <summary>
         /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
         /// </summary>

Engine/DataFeeds/SubscriptionDataReader.cs

@@ -21,7 +21,6 @@ using QuantConnect.Data;
 using System.Globalization;
 using QuantConnect.Logging;
 using QuantConnect.Interfaces;
-using QuantConnect.Data.Custom;
 using System.Collections.Generic;
 using QuantConnect.Configuration;
 using QuantConnect.Data.Auxiliary;
@@ -188,16 +187,6 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                 return;
             }
 
-            //If its quandl set the access token in data factory:
-            var quandl = _dataFactory as Quandl;
-            if (quandl != null)
-            {
-                if (!Quandl.IsAuthCodeSet)
-                {
-                    Quandl.SetAuthCode(Config.Get("quandl-auth-token"));
-                }
-            }
-
             // If Tiingo data, set the access token in data factory
             var tiingo = _dataFactory as TiingoPrice;
             if (tiingo != null)

Engine/DataFeeds/SubscriptionDataSourceReader.cs

@@ -55,7 +55,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                     break;
 
                 case FileFormat.ZipEntryName:
-                    reader = new ZipEntryNameSubscriptionDataSourceReader(dataProvider, config, date, isLiveMode);
+                    reader = new ZipEntryNameSubscriptionDataSourceReader(dataCacheProvider, config, date, isLiveMode);
                     break;
 
                 case FileFormat.Index:

Engine/DataFeeds/TimeSliceFactory.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -450,19 +450,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             OptionContract contract;
             if (!chain.Contracts.TryGetValue(baseData.Symbol, out contract))
             {
-                var underlyingSymbol = baseData.Symbol.Underlying;
-                contract = new OptionContract(baseData.Symbol, underlyingSymbol)
-                {
-                    Time = baseData.EndTime,
-                    LastPrice = security.Close,
-                    Volume = (long)security.Volume,
-                    BidPrice = security.BidPrice,
-                    BidSize = (long)security.BidSize,
-                    AskPrice = security.AskPrice,
-                    AskSize = (long)security.AskSize,
-                    OpenInterest = security.OpenInterest,
-                    UnderlyingLastPrice = chain.Underlying.Price
-                };
+                contract = OptionContract.Create(baseData, security, chain.Underlying.Price);
 
                 chain.Contracts[baseData.Symbol] = contract;
 

Engine/DataFeeds/ZipDataCacheProvider.cs

@@ -15,15 +15,15 @@
 
 using System;
 using System.IO;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
-using QuantConnect.Logging;
-using System.Linq;
-using System.Threading;
 using Ionic.Zip;
 using Ionic.Zlib;
-using QuantConnect.Interfaces;
+using System.Linq;
+using System.Threading;
 using QuantConnect.Util;
+using QuantConnect.Logging;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
 
 namespace QuantConnect.Lean.Engine.DataFeeds
 {
@@ -89,6 +89,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                                 }
                                 else
                                 {
+                                    existingZip.Refresh();
                                     stream = CreateEntryStream(existingZip, entryName, filename);
                                 }
                             }
@@ -173,6 +174,26 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             }
         }
 
+        /// <summary>
+        /// Returns a list of zip entries in a provided zip file
+        /// </summary>
+        public List<string> GetZipEntries(string zipFile)
+        {
+            if (!_zipFileCache.TryGetValue(zipFile, out var cachedZip))
+            {
+                if (!Cache(zipFile, out cachedZip))
+                {
+                    throw new ArgumentException($"Failed to get zip entries from {zipFile}");
+                }
+            }
+
+            lock (cachedZip)
+            {
+                cachedZip.Refresh();
+                return cachedZip.EntryCache.Keys.ToList();
+            }
+        }
+
         /// <summary>
         /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
         /// </summary>
@@ -461,8 +482,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             public void WriteEntry(string entryName, byte[] content)
             {
                 Interlocked.Increment(ref _modified);
-                // we refresh our cache time when modified
-                _dateCached = new ReferenceWrapper<DateTime>(DateTime.UtcNow);
+                Refresh();
 
                 // If the entry already exists remove it 
                 if (_zipFile.ContainsEntry(entryName))
@@ -476,6 +496,14 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                 EntryCache.Add(entryName, new ZipEntryCache { Entry = newEntry, Modified = true });
             }
 
+            /// <summary>
+            /// We refresh our cache time when used to avoid it being clean up
+            /// </summary>
+            public void Refresh()
+            {
+                _dateCached = new ReferenceWrapper<DateTime>(DateTime.UtcNow);
+            }
+
             /// <summary>
             /// Dispose of the ZipFile
             /// </summary>
@@ -497,7 +525,6 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                     _zipFile.Save(tempFileName);
                 }
 
-                EntryCache.Clear();
                 _zipFile?.DisposeSafely();
                 _dataStream?.DisposeSafely();
 

Engine/DataFeeds/ZipEntryNameSubscriptionDataSourceReader.cs

@@ -14,10 +14,7 @@
 */
 
 using System;
-using Ionic.Zip;
-using System.Linq;
 using QuantConnect.Data;
-using QuantConnect.Util;
 using QuantConnect.Interfaces;
 using System.Collections.Generic;
 
@@ -28,7 +25,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
     /// </summary>
     public class ZipEntryNameSubscriptionDataSourceReader : ISubscriptionDataSourceReader
     {
-        private readonly IDataProvider _dataProvider;
+        private readonly IDataCacheProvider _dataProvider;
         private readonly SubscriptionDataConfig _config;
         private readonly DateTime _date;
         private readonly bool _isLiveMode;
@@ -47,13 +44,13 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         /// <param name="config">The subscription's configuration</param>
         /// <param name="date">The date this factory was produced to read data for</param>
         /// <param name="isLiveMode">True if we're in live mode, false for backtesting</param>
-        public ZipEntryNameSubscriptionDataSourceReader(IDataProvider dataProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
+        public ZipEntryNameSubscriptionDataSourceReader(IDataCacheProvider dataProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
         {
             _date = date;
             _config = config;
             _isLiveMode = isLiveMode;
             _dataProvider = dataProvider;
-            _factory = _factory = config.GetBaseDataInstance();
+            _factory = config.GetBaseDataInstance();
         }
 
         /// <summary>
@@ -66,16 +63,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             List<string> entryNames;
             try
             {
-                var stream = _dataProvider.Fetch(source.Source);
-                if (stream == null)
-                {
-                    OnInvalidSource(source, new ArgumentException($"Failed to create source stream {source.Source}"));
-                    yield break;
-                }
-                entryNames = Compression.GetZipEntryFileNames(stream).ToList();
-                stream.DisposeSafely();
+                entryNames = _dataProvider.GetZipEntries(source.Source);
             }
-            catch (ZipException err)
+            catch (Exception err)
             {
                 OnInvalidSource(source, err);
                 yield break;

Engine/HistoricalData/SubscriptionDataReaderHistoryProvider.cs

@@ -15,11 +15,9 @@
 */
 
 using System;
-using System.Collections;
 using System.Collections.Generic;
 using NodaTime;
 using QuantConnect.Data;
-using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Market;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Interfaces;
@@ -192,58 +190,5 @@ namespace QuantConnect.Lean.Engine.HistoricalData
         {
             return null;
         }
-
-        private class FilterEnumerator<T> : IEnumerator<T>
-        {
-            private readonly IEnumerator<T> _enumerator;
-            private readonly Func<T, bool> _filter;
-
-            public FilterEnumerator(IEnumerator<T> enumerator, Func<T, bool> filter)
-            {
-                _enumerator = enumerator;
-                _filter = filter;
-            }
-
-            #region Implementation of IDisposable
-
-            public void Dispose()
-            {
-                _enumerator.Dispose();
-            }
-
-            #endregion
-
-            #region Implementation of IEnumerator
-
-            public bool MoveNext()
-            {
-                // run the enumerator until it passes the specified filter
-                while (_enumerator.MoveNext())
-                {
-                    if (_filter(_enumerator.Current))
-                    {
-                        return true;
-                    }
-                }
-                return false;
-            }
-
-            public void Reset()
-            {
-                _enumerator.Reset();
-            }
-
-            public T Current
-            {
-                get { return _enumerator.Current; }
-            }
-
-            object IEnumerator.Current
-            {
-                get { return _enumerator.Current; }
-            }
-
-            #endregion
-        }
     }
 }

Engine/QuantConnect.Lean.Engine.csproj

@@ -43,7 +43,7 @@
     <Message Text="SelectedOptimization $(SelectedOptimization)" Importance="high" />
   </Target>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="fasterflect" Version="3.0.0" />
     <PackageReference Include="FSharp.Core" Version="4.5.2" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />

Indicators/QuantConnect.Indicators.csproj

@@ -32,7 +32,7 @@
     <Message Text="SelectedOptimization $(SelectedOptimization)" Importance="high" />
   </Target>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>

Launcher/config.json

@@ -24,7 +24,7 @@
   // engine
   "data-folder": "../../../Data/",
 
-  // debugging configuration - options for debugging-method LocalCmdLine, VisualStudio, PTVSD, PyCharm
+  // debugging configuration - options for debugging-method LocalCmdLine, VisualStudio, Debugpy, PyCharm
   "debugging": false,
   "debugging-method": "LocalCmdline",
 
@@ -152,9 +152,9 @@
   "tt-order-routing-port": "",
   "tt-log-fix-messages": false,
 
-  // Required to access data from Quandl
-  // To get your access token go to https://www.quandl.com/account/api
-  "quandl-auth-token": "",
+  // Required to access data from Nasdaq
+  // To get your access token go to https://data.nasdaq.com/account/profile
+  "nasdaq-auth-token": "",
 
   // Required to access data from Tiingo
   // To get your access token go to https://www.tiingo.com
@@ -194,6 +194,11 @@
   "ftx-api-secret": "",
   "ftx-api-key": "",
 
+  // FTX.US configuration
+  "ftxus-account-tier": "Tier1", // accept "Tier1", "Tier2", "Tier3", "Tier4", "Tier5", "Tier6", "Tier7", "Tier8", "Tier9", "VIP1", "VIP2", "MM1", "MM2", "MM3"
+  "ftxus-api-secret": "",
+  "ftxus-api-key": "",
+
   // parameters to set in the algorithm (the below are just samples)
   "parameters": {
     // Intrinio account user and password
@@ -431,6 +436,8 @@
       "transaction-handler": "QuantConnect.Lean.Engine.TransactionHandlers.BrokerageTransactionHandler",
       "history-provider": "BrokerageHistoryProvider"
     },
+
+    // defines the 'live-ftx' environment
     "live-ftx": {
       "live-mode": true,
 
@@ -443,6 +450,21 @@
       "real-time-handler": "QuantConnect.Lean.Engine.RealTime.LiveTradingRealTimeHandler",
       "transaction-handler": "QuantConnect.Lean.Engine.TransactionHandlers.BrokerageTransactionHandler",
       "history-provider": "BrokerageHistoryProvider"
+    },
+
+    // defines the 'live-ftxus' environment
+    "live-ftxus": {
+      "live-mode": true,
+
+      // real brokerage implementations require the BrokerageTransactionHandler
+      "live-mode-brokerage": "QuantConnect.FTXBrokerage.FTXUSBrokerage",
+      "data-queue-handler": [ "QuantConnect.FTXBrokerage.FTXUSBrokerage" ],
+      "setup-handler": "QuantConnect.Lean.Engine.Setup.BrokerageSetupHandler",
+      "result-handler": "QuantConnect.Lean.Engine.Results.LiveTradingResultHandler",
+      "data-feed-handler": "QuantConnect.Lean.Engine.DataFeeds.LiveTradingDataFeed",
+      "real-time-handler": "QuantConnect.Lean.Engine.RealTime.LiveTradingRealTimeHandler",
+      "transaction-handler": "QuantConnect.Lean.Engine.TransactionHandlers.BrokerageTransactionHandler",
+      "history-provider": "BrokerageHistoryProvider"
     }
   }
 }

Report/QuantConnect.Report.csproj

@@ -42,7 +42,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Deedle" Version="2.1.0" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">

Research/QuantConnect.Research.csproj

@@ -32,7 +32,7 @@
     <Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

Tests/Algorithm/AlgorithmAddDataTests.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -16,6 +16,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using Python.Runtime;
 using Newtonsoft.Json;
 using NodaTime;
 using NUnit.Framework;
@@ -26,10 +27,8 @@ using QuantConnect.Configuration;
 using QuantConnect.Data;
 using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Consolidators;
-using QuantConnect.Data.Custom;
 using QuantConnect.Data.Custom.IconicTypes;
 using QuantConnect.Data.Market;
-using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Securities;
 using QuantConnect.Tests.Engine.DataFeeds;
@@ -136,10 +135,10 @@ namespace QuantConnect.Tests.Algorithm
             var bitcoinSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Type == typeof(Bitcoin));
             Assert.AreEqual(bitcoinSubscription.Type, typeof(Bitcoin));
 
-            // Add a quandl subscription
-            qcAlgorithm.AddData<Quandl>("EURCAD");
-            var quandlSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Type == typeof(Quandl));
-            Assert.AreEqual(quandlSubscription.Type, typeof(Quandl));
+            // Add a unlinkedData subscription
+            qcAlgorithm.AddData<UnlinkedData>("EURCAD");
+            var unlinkedDataSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Type == typeof(UnlinkedData));
+            Assert.AreEqual(unlinkedDataSubscription.Type, typeof(UnlinkedData));
         }
 
         [Test]
@@ -556,7 +555,7 @@ namespace QuantConnect.Tests.Algorithm
 
             // Initialize contains the statements:
             // self.AddData(Nifty, "NIFTY")
-            // self.AddData(QuandlFuture, "SCF/CME_CL1_ON", Resolution.Daily)
+            // self.AddData(CustomPythonData, "IBM", Resolution.Daily)
             qcAlgorithm.Initialize();
 
             var niftySubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Symbol.Value == "NIFTY");
@@ -565,11 +564,13 @@ namespace QuantConnect.Tests.Algorithm
             var niftyFactory = (BaseData)ObjectActivator.GetActivator(niftySubscription.Type).Invoke(new object[] { niftySubscription.Type });
             Assert.DoesNotThrow(() => niftyFactory.GetSource(niftySubscription, DateTime.UtcNow, false));
 
-            var quandlSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Symbol.Value == "SCF/CME_CL1_ON");
-            Assert.IsNotNull(quandlSubscription);
+            var customDataSubscription = qcAlgorithm.SubscriptionManager.Subscriptions.FirstOrDefault(x => x.Symbol.Value == "IBM");
+            Assert.IsNotNull(customDataSubscription);
+            Assert.IsTrue(customDataSubscription.IsCustomData);
+            Assert.AreEqual("custom_data.CustomPythonData", customDataSubscription.Type.ToString());
 
-            var quandlFactory = (BaseData)ObjectActivator.GetActivator(quandlSubscription.Type).Invoke(new object[] { quandlSubscription.Type });
-            Assert.DoesNotThrow(() => quandlFactory.GetSource(quandlSubscription, DateTime.UtcNow, false));
+            var customDataFactory = (BaseData)ObjectActivator.GetActivator(customDataSubscription.Type).Invoke(new object[] { customDataSubscription.Type });
+            Assert.DoesNotThrow(() => customDataFactory.GetSource(customDataSubscription, DateTime.UtcNow, false));
         }
 
         [Test]
@@ -580,14 +581,14 @@ namespace QuantConnect.Tests.Algorithm
 
             // Initialize contains the statements:
             // self.AddData(Nifty, "NIFTY")
-            // self.AddData(QuandlFuture, "SCF/CME_CL1_ON", Resolution.Daily)
+            // self.AddData(CustomPythonData, "IBM", Resolution.Daily)
             qcAlgorithm.Initialize();
 
             var niftyConsolidator = new DynamicDataConsolidator(TimeSpan.FromDays(2));
             Assert.DoesNotThrow(() => qcAlgorithm.SubscriptionManager.AddConsolidator("NIFTY", niftyConsolidator));
 
-            var quandlConsolidator = new DynamicDataConsolidator(TimeSpan.FromDays(2));
-            Assert.DoesNotThrow(() => qcAlgorithm.SubscriptionManager.AddConsolidator("SCF/CME_CL1_ON", quandlConsolidator));
+            var customDataConsolidator = new DynamicDataConsolidator(TimeSpan.FromDays(2));
+            Assert.DoesNotThrow(() => qcAlgorithm.SubscriptionManager.AddConsolidator("IBM", customDataConsolidator));
         }
 
         [Test]
@@ -607,10 +608,10 @@ namespace QuantConnect.Tests.Algorithm
             const string ticker = "ticker";
             var algorithm = Algorithm();
 
-            var security = algorithm.AddData<Quandl>(ticker);
+            var security = algorithm.AddData<UnlinkedData>(ticker);
             Assert.AreEqual(ticker.ToUpperInvariant(), security.Symbol.Value);
-            Assert.AreEqual($"{ticker.ToUpperInvariant()}.{typeof(Quandl).Name}", security.Symbol.ID.Symbol);
-            Assert.AreEqual(SecurityIdentifier.GenerateBaseSymbol(typeof(Quandl), ticker), security.Symbol.ID.Symbol);
+            Assert.AreEqual($"{ticker.ToUpperInvariant()}.{typeof(UnlinkedData).Name}", security.Symbol.ID.Symbol);
+            Assert.AreEqual(SecurityIdentifier.GenerateBaseSymbol(typeof(UnlinkedData), ticker), security.Symbol.ID.Symbol);
         }
 
         [Test]
@@ -619,7 +620,7 @@ namespace QuantConnect.Tests.Algorithm
             var algorithm = Algorithm();
 
             Symbol cachedSymbol;
-            var security = algorithm.AddData<Quandl>("ticker");
+            var security = algorithm.AddData<UnlinkedData>("ticker");
             var symbolCacheAlias = security.Symbol.ID.Symbol;
 
             Assert.IsTrue(SymbolCache.TryGetSymbol(symbolCacheAlias, out cachedSymbol));
@@ -632,16 +633,16 @@ namespace QuantConnect.Tests.Algorithm
             const string ticker = "ticker";
             var algorithm = Algorithm();
 
-            var security1 = algorithm.AddData<Quandl>(ticker);
+            var security1 = algorithm.AddData<UnlinkedData>(ticker);
             var security2 = algorithm.AddData<Bitcoin>(ticker);
 
-            var quandl = algorithm.Securities[security1.Symbol];
-            Assert.AreSame(security1, quandl);
+            var unlinkedData = algorithm.Securities[security1.Symbol];
+            Assert.AreSame(security1, unlinkedData);
 
             var bitcoin = algorithm.Securities[security2.Symbol];
             Assert.AreSame(security2, bitcoin);
 
-            Assert.AreNotSame(quandl, bitcoin);
+            Assert.AreNotSame(unlinkedData, bitcoin);
         }
 
         private static SubscriptionDataConfig GetMatchingSubscription(QCAlgorithm algorithm, Symbol symbol, Type type)

Tests/Algorithm/AlgorithmAddSecurityTests.cs

@@ -17,15 +17,18 @@
 using System;
 using NUnit.Framework;
 using QuantConnect.Algorithm;
-using QuantConnect.Data.Custom.AlphaStreams;
-using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Securities;
+using System.Collections.Generic;
 using QuantConnect.Securities.Cfd;
 using QuantConnect.Securities.Crypto;
 using QuantConnect.Securities.Equity;
 using QuantConnect.Securities.Forex;
 using QuantConnect.Securities.Future;
 using QuantConnect.Securities.Option;
+using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Tests.Engine.DataFeeds;
+using QuantConnect.Securities.IndexOption;
+using QuantConnect.Data.Custom.AlphaStreams;
 using Index = QuantConnect.Securities.Index.Index;
 
 namespace QuantConnect.Tests.Algorithm
@@ -80,6 +83,9 @@ namespace QuantConnect.Tests.Algorithm
                     case SecurityType.Index:
                         var index = (Index)security;
                         break;
+                    case SecurityType.IndexOption:
+                        var indexOption = (IndexOption)security;
+                        break;
                     case SecurityType.Crypto:
                         var crypto = (Crypto)security;
                         break;
@@ -102,7 +108,7 @@ namespace QuantConnect.Tests.Algorithm
         {
             get
             {
-                return new[]
+                var result = new List<TestCaseData>()
                 {
                     new TestCaseData(Symbols.SPY, null),
                     new TestCaseData(Symbols.EURUSD, null),
@@ -117,6 +123,21 @@ namespace QuantConnect.Tests.Algorithm
                     new TestCaseData(Symbol.Create("CustomData", SecurityType.Base, Market.Binance), null),
                     new TestCaseData(Symbol.Create("CustomData2", SecurityType.Base, Market.COMEX), null)
                 };
+
+                foreach (var market in Market.SupportedMarkets())
+                {
+                    foreach (var kvp in SymbolPropertiesDatabase.FromDataFolder().GetSymbolPropertiesList(market))
+                    {
+                        var securityDatabaseKey = kvp.Key;
+                        if (securityDatabaseKey.SecurityType != SecurityType.FutureOption)
+                        {
+                            result.Add(new TestCaseData(Symbol.Create(securityDatabaseKey.Symbol, securityDatabaseKey.SecurityType,
+                                securityDatabaseKey.Market), null));
+                        }
+                    }
+                }
+
+                return result.ToArray();
             }
         }
     }

Tests/Algorithm/AlgorithmWarmupTests.cs

@@ -26,6 +26,7 @@ using QuantConnect.Packets;
 using QuantConnect.Indicators;
 using QuantConnect.Tests.Engine.DataFeeds;
 using Python.Runtime;
+using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Lean.Engine.HistoricalData;
 using QuantConnect.Util;
@@ -109,6 +110,30 @@ namespace QuantConnect.Tests.Algorithm
             Assert.GreaterOrEqual(_algorithm.WarmUpDataCount, estimateExpectedDataCount);
         }
 
+        [Test]
+        public void WarmUpInternalSubscriptionsHistoryRequest()
+        {
+            var algo = new AlgorithmStub(new MockDataFeed())
+            {
+                HistoryProvider = new SubscriptionDataReaderHistoryProvider()
+            };
+
+            algo.SetStartDate(2013, 10, 08);
+            algo.AddCfd("DE30EUR", Resolution.Second, Market.Oanda);
+            algo.SetWarmup(10);
+            algo.PostInitialize();
+            algo.OnEndOfTimeStep();
+            algo.DataManager.UniverseSelection.EnsureCurrencyDataFeeds(SecurityChanges.None);
+
+            var result = algo.GetWarmupHistoryRequests();
+
+            foreach (var historyRequest in result)
+            {
+                Assert.AreEqual(Resolution.Second, historyRequest.Resolution);
+                Assert.AreEqual(TimeSpan.FromSeconds(10), historyRequest.EndTimeUtc - historyRequest.StartTimeUtc);
+            }
+        }
+
         [Test]
         public void WarmUpPythonIndicatorProperly()
         {

Tests/Common/Brokerages/FTXBrokerageModelTests.cs

@@ -37,8 +37,8 @@ namespace QuantConnect.Tests.Common.Brokerages
         [SetUp]
         public void Init()
         {
-            _brokerageModel = new();
-            _symbol = Symbol.Create("ETHUSD", SecurityType.Crypto, Market.FTX);
+            _brokerageModel = GetBrokerageModel();
+            _symbol = Symbol.Create("ETHUSD", SecurityType.Crypto, Market);
         }
 
         protected Crypto Security =>
@@ -60,10 +60,12 @@ namespace QuantConnect.Tests.Common.Brokerages
                 RegisteredSecurityDataTypesProvider.Null
             );
 
+        protected virtual string Market => QuantConnect.Market.FTX;
+
         [Test]
         public void GetCashBuyingPowerModelTest()
         {
-            var model = new FTXBrokerageModel(AccountType.Cash);
+            var model = GetBrokerageModel(AccountType.Cash);
             Assert.IsInstanceOf<CashBuyingPowerModel>(model.GetBuyingPowerModel(Security));
             Assert.AreEqual(1, model.GetLeverage(Security));
         }
@@ -71,33 +73,36 @@ namespace QuantConnect.Tests.Common.Brokerages
         [Test]
         public void GetSecurityMarginModelTest()
         {
-            var model = new FTXBrokerageModel(AccountType.Margin);
+            var model = GetBrokerageModel(AccountType.Margin);
             Assert.IsInstanceOf<SecurityMarginModel>(model.GetBuyingPowerModel(Security));
             Assert.AreEqual(3M, model.GetLeverage(Security));
         }
 
         [Test]
-        public void GetFeeModelTest()
+        public virtual void GetFeeModelTest()
         {
             Assert.IsInstanceOf<FTXFeeModel>(_brokerageModel.GetFeeModel(Security));
         }
 
         [TestCase(SecurityType.Crypto)]
-        public void ShouldReturnFTXMarket(SecurityType securityType)
+        public void ShouldReturnProperMarket(SecurityType securityType)
         {
-            Assert.AreEqual(Market.FTX, _brokerageModel.DefaultMarkets[securityType]);
+            Assert.AreEqual(Market, _brokerageModel.DefaultMarkets[securityType]);
         }
 
         [TestCase(0.01, true)]
         [TestCase(0.00005, false)]
         public void CanSubmitOrder_WhenQuantityIsLargeEnough(decimal orderQuantity, bool isValidOrderQuantity)
         {
-            BrokerageMessageEvent message;
-            var order = new Mock<Order>();
+            var order = new Mock<Order>
+            {
+                Object =
+                {
+                    Quantity = orderQuantity
+                }
+            };
 
-            order.Object.Quantity = orderQuantity;
-
-            Assert.AreEqual(isValidOrderQuantity, _brokerageModel.CanSubmitOrder(TestsHelpers.GetSecurity(market: Market.FTX), order.Object, out message));
+            Assert.AreEqual(isValidOrderQuantity, _brokerageModel.CanSubmitOrder(TestsHelpers.GetSecurity(market: Market), order.Object, out _));
         }
 
         [Test]
@@ -190,5 +195,7 @@ namespace QuantConnect.Tests.Common.Brokerages
             Assert.AreEqual(false, _brokerageModel.CanSubmitOrder(security, order.Object, out var message));
             Assert.NotNull(message);
         }
+
+        protected virtual FTXBrokerageModel GetBrokerageModel(AccountType accountType = AccountType.Margin) => new(accountType);
     }
 }

Tests/Common/Brokerages/FTXUSBrokerageModelTests.cs

@@ -0,0 +1,35 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using NUnit.Framework;
+using QuantConnect.Brokerages;
+using QuantConnect.Orders.Fees;
+
+namespace QuantConnect.Tests.Common.Brokerages
+{
+    [TestFixture]
+    public class FTXUSBrokerageModelTests : FTXBrokerageModelTests
+    {
+        protected override FTXBrokerageModel GetBrokerageModel(AccountType accountType = AccountType.Margin) => new FTXUSBrokerageModel(accountType);
+
+        protected override string Market => QuantConnect.Market.FTXUS;
+
+        [Test]
+        public override void GetFeeModelTest()
+        {
+            Assert.IsInstanceOf<FTXUSFeeModel>(GetBrokerageModel().GetFeeModel(Security));
+        }
+    }
+}

Tests/Common/Data/Custom/QuandlTests.cs

@@ -1,51 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using System.Linq;
-using NodaTime;
-using NUnit.Framework;
-using QuantConnect.Algorithm.CSharp;
-using QuantConnect.Data;
-using QuantConnect.Data.Custom;
-using QuantConnect.Lean.Engine.DataFeeds;
-using QuantConnect.Lean.Engine.DataFeeds.Transport;
-
-namespace QuantConnect.Tests.Common.Data.Custom
-{
-    [TestFixture, Parallelizable(ParallelScope.All)]
-    public class QuandlTests
-    {
-        [Test]
-        public void QuandlDownloadDoesNotThrow()
-        {
-            Quandl.SetAuthCode("WyAazVXnq7ATy_fefTqm");
-            RemoteFileSubscriptionStreamReader.SetDownloadProvider(new Api.Api());
-            var data = new HistoryAlgorithm.QuandlFuture();
-
-            const string ticker = "CHRIS/CME_SP1";
-            var date = new DateTime(2018, 8, 31);
-
-            var config = new SubscriptionDataConfig(typeof(HistoryAlgorithm.QuandlFuture), Symbol.Create(ticker, SecurityType.Base, QuantConnect.Market.USA), Resolution.Daily, DateTimeZone.Utc, DateTimeZone.Utc, false, false, false, true);
-            var source = data.GetSource(config, date, false);
-            var dataCacheProvider = new SingleEntryDataCacheProvider(new DefaultDataProvider());
-            var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, config, date, false, data, TestGlobals.DataProvider);
-
-            var rows = factory.Read(source).ToList();
-
-            Assert.IsTrue(rows.Count > 0);
-        }
-    }
-}

Tests/Common/Data/SliceTests.cs

@@ -19,7 +19,6 @@ using System.Linq;
 using NUnit.Framework;
 using Python.Runtime;
 using QuantConnect.Data;
-using QuantConnect.Data.Custom;
 using QuantConnect.Data.Custom.IconicTypes;
 using QuantConnect.Data.Market;
 using QuantConnect.Indicators;
@@ -35,17 +34,17 @@ namespace QuantConnect.Tests.Common.Data
         {
             var now = DateTime.UtcNow;
             var tradeBar = new TradeBar { Symbol = Symbols.SPY, Time = now };
-            var quandl = new Quandl { Symbol = Symbols.SPY, Time = now };
+            var unlinkedData = new UnlinkedData { Symbol = Symbols.SPY, Time = now };
             var quoteBar = new QuoteBar { Symbol = Symbols.SPY, Time = now };
             var tick = new Tick(now, Symbols.SPY, 1.1m, 2.1m) {TickType = TickType.Trade};
             var openInterest = new OpenInterest(now, Symbols.SPY, 1);
             var split = new Split(Symbols.SPY, now, 1, 1, SplitType.SplitOccurred);
             var delisting = new Delisting(Symbols.SPY, now, 1, DelistingType.Delisted);
 
-            var slice = new Slice(now, new BaseData[] {quoteBar, tradeBar, quandl, tick, split, delisting, openInterest });
+            var slice = new Slice(now, new BaseData[] {quoteBar, tradeBar, unlinkedData, tick, split, delisting, openInterest });
 
             Assert.AreEqual(slice.Get(typeof(TradeBar))[Symbols.SPY], tradeBar);
-            Assert.AreEqual(slice.Get(typeof(Quandl))[Symbols.SPY], quandl);
+            Assert.AreEqual(slice.Get(typeof(UnlinkedData))[Symbols.SPY], unlinkedData);
             Assert.AreEqual(slice.Get(typeof(QuoteBar))[Symbols.SPY], quoteBar);
             Assert.AreEqual(slice.Get(typeof(Tick))[Symbols.SPY], tick);
             Assert.AreEqual(slice.Get(typeof(Split))[Symbols.SPY], split);
@@ -156,22 +155,22 @@ namespace QuantConnect.Tests.Common.Data
         public void AccessesCustomGenericallyByTypeOtherTypesPresent()
         {
             var tradeBar = new TradeBar { Symbol = Symbols.SPY, Time = DateTime.Now };
-            var quandlSpy = new Quandl { Symbol = Symbols.SPY, Time = DateTime.Now };
-            Slice slice = new Slice(DateTime.Now, new BaseData[] { quandlSpy, tradeBar });
+            var unlinkedDataSpy = new UnlinkedData { Symbol = Symbols.SPY, Time = DateTime.Now };
+            var slice = new Slice(DateTime.Now, new BaseData[] { unlinkedDataSpy, tradeBar });
 
-            DataDictionary<Quandl> quandlData = slice.Get<Quandl>();
-            Assert.AreEqual(1, quandlData.Count);
+            var unlinkedData = slice.Get<UnlinkedData>();
+            Assert.AreEqual(1, unlinkedData.Count);
         }
 
         [Test]
         public void AccessesCustomGenericallyByType()
         {
-            Quandl quandlSpy = new Quandl { Symbol = Symbols.SPY, Time = DateTime.Now };
-            Quandl quandlAapl = new Quandl { Symbol = Symbols.AAPL, Time = DateTime.Now };
-            Slice slice = new Slice(DateTime.Now, new[] { quandlSpy, quandlAapl });
+            var unlinkedDataSpy = new UnlinkedData { Symbol = Symbols.SPY, Time = DateTime.Now };
+            var unlinkedDataAapl = new UnlinkedData { Symbol = Symbols.AAPL, Time = DateTime.Now };
+            var slice = new Slice(DateTime.Now, new[] { unlinkedDataSpy, unlinkedDataAapl });
 
-            DataDictionary<Quandl> quandlData = slice.Get<Quandl>();
-            Assert.AreEqual(2, quandlData.Count);
+            var unlinkedData = slice.Get<UnlinkedData>();
+            Assert.AreEqual(2, unlinkedData.Count);
         }
 
         [Test]
@@ -199,12 +198,12 @@ namespace QuantConnect.Tests.Common.Data
         [Test]
         public void AccessesGenericallyByTypeAndSymbol()
         {
-            Quandl quandlSpy = new Quandl { Symbol = Symbols.SPY, Time = DateTime.Now };
-            Quandl quandlAapl = new Quandl { Symbol = Symbols.AAPL, Time = DateTime.Now };
-            Slice slice = new Slice(DateTime.Now, new[] { quandlSpy, quandlAapl });
+            var unlinkedDataSpy = new UnlinkedData { Symbol = Symbols.SPY, Time = DateTime.Now };
+            var unlinkedDataAapl = new UnlinkedData { Symbol = Symbols.AAPL, Time = DateTime.Now };
+            var slice = new Slice(DateTime.Now, new[] { unlinkedDataSpy, unlinkedDataAapl });
 
-            Quandl quandlData = slice.Get<Quandl>(Symbols.SPY);
-            Assert.AreEqual(quandlSpy, quandlData);
+            var unlinkedData = slice.Get<UnlinkedData>(Symbols.SPY);
+            Assert.AreEqual(unlinkedDataSpy, unlinkedData);
         }
 
         [Test]
@@ -217,11 +216,11 @@ namespace QuantConnect.Tests.Common.Data
 from AlgorithmImports import *
 
 def Test(slice):
-    data = slice.Get(Quandl)
+    data = slice.Get(UnlinkedData)
     return data").GetAttr("Test");
-                var quandlSpy = new Quandl { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10 };
-                var quandlAapl = new Quandl { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11 };
-                var slice = new Slice(DateTime.Now, new[] { quandlSpy, quandlAapl });
+                var unlinkedDataSpy = new UnlinkedData { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10 };
+                var unlinkedDataAapl = new UnlinkedData { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11 };
+                var slice = new Slice(DateTime.Now, new[] { unlinkedDataSpy, unlinkedDataAapl });
 
                 var data = test(new PythonSlice(slice));
                 Assert.AreEqual(2, (int)data.Count);
@@ -242,8 +241,8 @@ from AlgorithmImports import *
 def Test(slice):
     for dataPoint in slice:
         return dataPoint").GetAttr("Test");
-                var quandlAapl = new Quandl { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11 };
-                var slice = new Slice(DateTime.Now, new[] { quandlAapl });
+                var unlinkedDataAapl = new UnlinkedData { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11 };
+                var slice = new Slice(DateTime.Now, new[] { unlinkedDataAapl });
 
                 var data = test(new PythonSlice(slice)) as PyObject;
                 var keyValuePair = data.As<KeyValuePair<Symbol, BaseData>>();
@@ -263,13 +262,13 @@ from AlgorithmImports import *
 from QuantConnect.Tests import *
 
 def Test(slice):
-    data = slice.Get(Quandl)
+    data = slice.Get(UnlinkedData)
     value = data[Symbols.AAPL].Value
     if value != 11:
         raise Exception('Unexpected value')").GetAttr("Test");
-                var quandlSpy = new Quandl { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10 };
-                var quandlAapl = new Quandl { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11 };
-                var slice = new Slice(DateTime.Now, new[] { quandlSpy, quandlAapl });
+                var unlinkedDataSpy = new UnlinkedData { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10 };
+                var unlinkedDataAapl = new UnlinkedData { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11 };
+                var slice = new Slice(DateTime.Now, new[] { unlinkedDataSpy, unlinkedDataAapl });
 
                 Assert.DoesNotThrow(() => test(new PythonSlice(slice)));
             }
@@ -286,13 +285,13 @@ from AlgorithmImports import *
 from QuantConnect.Tests import *
 
 def Test(slice):
-    data = slice.Get(Quandl, Symbols.AAPL)
+    data = slice.Get(UnlinkedData, Symbols.AAPL)
     value = data.Value
     if value != 11:
         raise Exception('Unexpected value')").GetAttr("Test");
-                var quandlSpy = new Quandl { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10 };
-                var quandlAapl = new Quandl { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11 };
-                var slice = new Slice(DateTime.Now, new[] { quandlSpy, quandlAapl });
+                var unlinkedDataSpy = new UnlinkedData { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10 };
+                var unlinkedDataAapl = new UnlinkedData { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11 };
+                var slice = new Slice(DateTime.Now, new[] { unlinkedDataSpy, unlinkedDataAapl });
 
                 Assert.DoesNotThrow(() => test(new PythonSlice(slice)));
             }
@@ -312,9 +311,9 @@ def Test(slice):
     return data").GetAttr("Test");
                 var TradeBarSpy = new TradeBar { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 8 };
                 var TradeBarAapl = new TradeBar { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 9 };
-                var quandlSpy = new Quandl { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10 };
-                var quandlAapl = new Quandl { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11 };
-                var slice = new Slice(DateTime.Now, new BaseData[] { quandlSpy, TradeBarAapl, quandlAapl, TradeBarSpy });
+                var unlinkedDataSpy = new UnlinkedData { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10 };
+                var unlinkedDataAapl = new UnlinkedData { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11 };
+                var slice = new Slice(DateTime.Now, new BaseData[] { unlinkedDataSpy, TradeBarAapl, unlinkedDataAapl, TradeBarSpy });
 
                 var data = test(new PythonSlice(slice));
                 Assert.AreEqual(2, (int)data.Count);
@@ -341,10 +340,10 @@ def Test(slice):
                 var now = DateTime.UtcNow;
                 var TradeBarSpy = new TradeBar { Symbol = Symbols.SPY, Time = now, Value = 8 };
                 var TradeBarAapl = new TradeBar { Symbol = Symbols.AAPL, Time = now, Value = 9 };
-                var quandlSpy = new Quandl { Symbol = Symbols.SPY, Time = now, Value = 10 };
-                var quandlAapl = new Quandl { Symbol = Symbols.AAPL, Time = now, Value = 11 };
+                var unlinkedDataSpy = new UnlinkedData { Symbol = Symbols.SPY, Time = now, Value = 10 };
+                var unlinkedDataAapl = new UnlinkedData { Symbol = Symbols.AAPL, Time = now, Value = 11 };
                 var openInterest = new OpenInterest(now, Symbols.AAPL, 33);
-                var slice = new Slice(now, new BaseData[] { quandlSpy, TradeBarAapl, quandlAapl, TradeBarSpy, openInterest });
+                var slice = new Slice(now, new BaseData[] { unlinkedDataSpy, TradeBarAapl, unlinkedDataAapl, TradeBarSpy, openInterest });
 
                 Assert.DoesNotThrow(() => test(new PythonSlice(slice)));
             }
@@ -367,9 +366,9 @@ def Test(slice):
         raise Exception('Unexpected value')").GetAttr("Test");
                 var TradeBarSpy = new TradeBar { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 8 };
                 var TradeBarAapl = new TradeBar { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 9 };
-                var quandlSpy = new Quandl { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10 };
-                var quandlAapl = new Quandl { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11 };
-                var slice = new Slice(DateTime.Now, new BaseData[] { quandlSpy, TradeBarAapl, quandlAapl, TradeBarSpy });
+                var unlinkedDataSpy = new UnlinkedData { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10 };
+                var unlinkedDataAapl = new UnlinkedData { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11 };
+                var slice = new Slice(DateTime.Now, new BaseData[] { unlinkedDataSpy, TradeBarAapl, unlinkedDataAapl, TradeBarSpy });
 
                 Assert.DoesNotThrow(() => test(new PythonSlice(slice)));
             }
@@ -392,9 +391,9 @@ def Test(slice):
         raise Exception('Unexpected value')").GetAttr("Test");
                 var TradeBarSpy = new TradeBar { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 8 };
                 var TradeBarAapl = new TradeBar { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 9 };
-                var quandlSpy = new Quandl { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10 };
-                var quandlAapl = new Quandl { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11 };
-                var slice = new Slice(DateTime.Now, new BaseData[] { quandlSpy, TradeBarAapl, quandlAapl, TradeBarSpy });
+                var unlinkedDataSpy = new UnlinkedData { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10 };
+                var unlinkedDataAapl = new UnlinkedData { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11 };
+                var slice = new Slice(DateTime.Now, new BaseData[] { unlinkedDataSpy, TradeBarAapl, unlinkedDataAapl, TradeBarSpy });
 
                 Assert.DoesNotThrow(() => test(new PythonSlice(slice)));
             }
@@ -419,10 +418,10 @@ def Test(slice):
         count += 1
     if count != 2:
         raise Exception('Unexpected value')").GetAttr("Test");
-                var quandlSpy = new IndexedLinkedData { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10 };
+                var indexedLinkedDataSpy = new IndexedLinkedData { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10 };
                 var tradeBarAapl = new TradeBar { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 9 };
-                var quandlAapl = new IndexedLinkedData { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11 };
-                var slice = new Slice(DateTime.Now, new BaseData[] { quandlSpy, tradeBarAapl, quandlAapl });
+                var indexedLinkedDataAapl = new IndexedLinkedData { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11 };
+                var slice = new Slice(DateTime.Now, new BaseData[] { indexedLinkedDataSpy, tradeBarAapl, indexedLinkedDataAapl });
 
                 Assert.DoesNotThrow(() => test(new PythonSlice(slice)));
             }
@@ -466,15 +465,15 @@ def Test(slice):
 from AlgorithmImports import *
 
 def Test(slice):
-    data = slice.Get(Quandl)
+    data = slice.Get(UnlinkedData)
     count = 0
     for singleData in data:
         count += 1
     if count != 2:
         raise Exception('Unexpected value')").GetAttr("Test");
-                var quandlSpy = new Quandl { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10 };
-                var quandlAapl = new Quandl { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11 };
-                var slice = new Slice(DateTime.Now, new[] { quandlSpy, quandlAapl });
+                var unlinkedDataSpy = new UnlinkedData { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10 };
+                var unlinkedDataAapl = new UnlinkedData { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11 };
+                var slice = new Slice(DateTime.Now, new[] { unlinkedDataSpy, unlinkedDataAapl });
 
                 Assert.DoesNotThrow(() => test(new PythonSlice(slice)));
             }
@@ -1061,10 +1060,10 @@ def Test(slice, symbol):
         private Slice GetSlice()
         {
             SymbolCache.Clear();
-            var quandlSpy = new IndexedLinkedData { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10 };
+            var indexedLinkedDataSpy = new IndexedLinkedData { Symbol = Symbols.SPY, Time = DateTime.Now, Value = 10 };
             var tradeBarAapl = new TradeBar { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 9 };
-            var quandlAapl = new IndexedLinkedData { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11 };
-            return new Slice(DateTime.Now, new BaseData[] { quandlSpy, tradeBarAapl, quandlAapl });
+            var indexedLinkedDataAapl = new IndexedLinkedData { Symbol = Symbols.AAPL, Time = DateTime.Now, Value = 11 };
+            return new Slice(DateTime.Now, new BaseData[] { indexedLinkedDataSpy, tradeBarAapl, indexedLinkedDataAapl });
         }
 
         private PythonSlice GetPythonSlice() => new PythonSlice(GetSlice());

Tests/Common/Orders/Fees/FTXFeeTests.cs

@@ -29,11 +29,16 @@ namespace QuantConnect.Tests.Common.Orders.Fees
     {
         private Crypto _xrpusdt;
         private Crypto _ethusd;
-        private readonly IFeeModel _feeModel = new FTXFeeModel();
+        private IFeeModel _feeModel;
+
+        protected decimal TakerFee;
+        protected decimal MakerFee;
 
         [SetUp]
         public void Initialize()
         {
+            _feeModel = GetFeeModel();
+            SetBrokerageFees();
             var spdb = SymbolPropertiesDatabase.FromDataFolder();
             var tz = TimeZones.Utc;
             _xrpusdt = new Crypto(
@@ -57,6 +62,12 @@ namespace QuantConnect.Tests.Common.Orders.Fees
             _ethusd.SetMarketPrice(new Tick(DateTime.UtcNow, _ethusd.Symbol, 100, 100));
         }
 
+        protected virtual void SetBrokerageFees()
+        {
+            MakerFee = 0.02m;
+            TakerFee = 0.07m;
+        }
+
         [TestCase(-1)]
         [TestCase(1)]
         public void ReturnsTakerFeeInQuoteCurrency(decimal quantity)
@@ -116,7 +127,7 @@ namespace QuantConnect.Tests.Common.Orders.Fees
 
             Assert.AreEqual(_ethusd.QuoteCurrency.Symbol, fee.Value.Currency);
             // 100 (price) * 0.0007 (taker fee, in quote currency)
-            Assert.AreEqual(0.07m, fee.Value.Amount);
+            Assert.AreEqual(TakerFee, fee.Value.Amount);
         }
 
         [Test]
@@ -154,7 +165,7 @@ namespace QuantConnect.Tests.Common.Orders.Fees
 
             Assert.AreEqual(_ethusd.QuoteCurrency.Symbol, fee.Value.Currency);
             // 0.0002 (maker fee, in quote currency)
-            Assert.AreEqual(0.02, fee.Value.Amount);
+            Assert.AreEqual(MakerFee, fee.Value.Amount);
         }
 
         [Test]
@@ -192,7 +203,7 @@ namespace QuantConnect.Tests.Common.Orders.Fees
 
             Assert.AreEqual(_ethusd.BaseCurrencySymbol, fee.Value.Currency);
             // 0.0002 (maker fee, in base currency)
-            Assert.AreEqual(0.0002, fee.Value.Amount);
+            Assert.AreEqual(MakerFee / 100, fee.Value.Amount);
         }
 
         [Test]
@@ -207,7 +218,9 @@ namespace QuantConnect.Tests.Common.Orders.Fees
 
             Assert.AreEqual("USDT", fee.Value.Currency);
             // 100 (price) * 0.0007 (taker fee)
-            Assert.AreEqual(0.07m, fee.Value.Amount);
+            Assert.AreEqual(TakerFee, fee.Value.Amount);
         }
+
+        protected virtual FTXFeeModel GetFeeModel() => new();
     }
 }