Install NetCoreDbg

Drop Watchlist

Remove breakpoints from lean

Prep install of NetCoreDbg

Cleanup

Algorithm.CSharp/AddRemoveOptionUniverseRegressionAlgorithm.cs

@@ -40,8 +40,8 @@ namespace QuantConnect.Algorithm.CSharp
         private int _expectedContractIndex;
         private readonly List<Symbol> _expectedContracts = new List<Symbol>
         {
-            SymbolRepresentation.ParseOptionTickerOSI("GOOG  151224P00750000"),
             SymbolRepresentation.ParseOptionTickerOSI("GOOG  151224P00747500"),
+            SymbolRepresentation.ParseOptionTickerOSI("GOOG  151224P00750000"),
             SymbolRepresentation.ParseOptionTickerOSI("GOOG  151224P00752500")
         };
 
@@ -109,6 +109,11 @@ namespace QuantConnect.Algorithm.CSharp
                 var googOptionChain = AddOption(UnderlyingTicker);
                 googOptionChain.SetFilter(u =>
                 {
+                    // we added the universe at 10, the universe selection data should not be from before
+                    if (u.Underlying.EndTime.Hour < 10)
+                    {
+                        throw new Exception($"Unexpected underlying data point {u.Underlying.EndTime} {u.Underlying}");
+                    }
                     // find first put above market price
                     return u.IncludeWeeklys()
                         .Strikes(+1, +1)
@@ -231,7 +236,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Treynor Ratio", "0"},
             {"Total Fees", "$6.00"},
             {"Estimated Strategy Capacity", "$2000.00"},
-            {"Lowest Capacity Asset", "GOOCV 305RBQ2BZBZT2|GOOCV VP83T1ZUHROL"},
+            {"Lowest Capacity Asset", "GOOCV 305RBR0BSWIX2|GOOCV VP83T1ZUHROL"},
             {"Fitness Score", "0"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
@@ -251,7 +256,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "1e7b3e90918777b9dbf46353a96f3329"}
+            {"OrderListHash", "550a99c482106defd8ba15f48183768e"}
         };
     }
 }

Algorithm.CSharp/BasicTemplateIndiaAlgorithm.cs

@@ -33,13 +33,13 @@ namespace QuantConnect.Algorithm.CSharp
         /// </summary>
         public override void Initialize()
         {
-            SetStartDate(2003, 10, 07);  //Set Start Date
-            SetEndDate(2003, 10, 11);    //Set End Date
+            SetStartDate(2019, 1, 23);  //Set Start Date
+            SetEndDate(2019, 10, 31);    //Set End Date
             SetCash(100000);             //Set Strategy Cash
 
             // Find more symbols here: http://quantconnect.com/data
             // Equities Resolutions: Tick, Second, Minute, Hour, Daily.
-            AddEquity("UNIONBANK", Resolution.Second, Market.India);
+            AddEquity("YESBANK", Resolution.Minute, Market.India);
 
             //Set Order Prperties as per the requirements for order placement
             DefaultOrderProperties = new IndiaOrderProperties(exchange: Exchange.NSE);
@@ -58,7 +58,7 @@ namespace QuantConnect.Algorithm.CSharp
         {
             if (!Portfolio.Invested)
             {
-                var marketTicket = MarketOrder("UNIONBANK", 1);
+                var marketTicket = MarketOrder("YESBANK", 1);
             }
         }
 
@@ -73,12 +73,12 @@ namespace QuantConnect.Algorithm.CSharp
         /// <summary>
         /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
         /// </summary>
-        public bool CanRunLocally { get; } = false;
+        public bool CanRunLocally { get; } = true;
 
         /// <summary>
         /// This is used by the regression test system to indicate which languages this algorithm is written in.
         /// </summary>
-        public Language[] Languages { get; } = { Language.CSharp };
+        public Language[] Languages { get; } = { Language.CSharp, Language.Python };
 
         /// <summary>
         /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm

Algorithm.CSharp/IndiaDataRegressionAlgorithm.cs

@@ -0,0 +1,201 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using QuantConnect.Interfaces;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Regression algorithm demonstrating use of map files with India data
+    /// </summary>
+    /// <meta name="tag" content="using data" />
+    /// <meta name="tag" content="India data" />
+    /// <meta name="tag" content="regression test" />
+    /// <meta name="tag" content="rename event" />
+    /// <meta name="tag" content="map" />
+    /// <meta name="tag" content="mapping" />
+    /// <meta name="tag" content="map files" />
+    public class IndiaDataRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        private Symbol _mappingSymbol, _splitAndDividendSymbol;
+        private bool _initialMapping;
+        private bool _executionMapping;
+        private bool _receivedWarningEvent;
+        private bool _receivedOccurredEvent;
+
+        /// <summary>
+        /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
+        /// </summary>
+        public override void Initialize()
+        {
+            SetStartDate(2004, 5, 20);  //Set Start Date
+            SetEndDate(2016, 7, 26);    //Set End Date
+            _mappingSymbol = AddEquity("3MINDIA", Resolution.Daily, Market.India).Symbol;
+            _splitAndDividendSymbol = AddEquity("CCCL", Resolution.Daily, Market.India).Symbol;
+        }
+
+        /// <summary>
+        /// Raises the data event.
+        /// </summary>
+        /// <param name="data">Data.</param>
+        public void OnData(Dividends data)
+        {
+            if (data.ContainsKey(_splitAndDividendSymbol))
+            {
+                var dividend = data[_splitAndDividendSymbol];
+                if (Time.Date == new DateTime(2010, 06, 15) &&
+                    (dividend.Price != 0.5m || dividend.ReferencePrice != 88.8m || dividend.Distribution != 0.5m))
+                {
+                    throw new Exception("Did not receive expected dividend values");
+                }
+            }
+        }
+
+        /// <summary>
+        /// Raises the data event.
+        /// </summary>
+        /// <param name="data">Data.</param>
+        public void OnData(Splits data)
+        {
+            if (data.ContainsKey(_splitAndDividendSymbol))
+            {
+                var split = data[_splitAndDividendSymbol];
+                if (split.Type == SplitType.Warning)
+                {
+                    _receivedWarningEvent = true;
+                }
+                else if (split.Type == SplitType.SplitOccurred)
+                {
+                    _receivedOccurredEvent = true;
+                    if (split.Price != 421m || split.ReferencePrice != 421m || split.SplitFactor != 0.2m)
+                    {
+                        throw new Exception("Did not receive expected split values");
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Checks the symbol change event
+        /// </summary>
+        public override void OnData(Slice slice)
+        {
+            if (slice.SymbolChangedEvents.ContainsKey(_mappingSymbol))
+            {
+                var mappingEvent = slice.SymbolChangedEvents.Single(x => x.Key.SecurityType == SecurityType.Equity).Value;
+                Log($"{Time} - Ticker changed from: {mappingEvent.OldSymbol} to {mappingEvent.NewSymbol}");
+                if (Time.Date == new DateTime(1999, 01, 01))
+                {
+                    _initialMapping = true;
+                }
+                else if (Time.Date == new DateTime(2004, 06, 15))
+                {
+                    if (mappingEvent.NewSymbol == "3MINDIA"
+                        && mappingEvent.OldSymbol == "BIRLA3M")
+                    {
+                        _executionMapping = true;
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Final step of the algorithm
+        /// </summary>
+        public override void OnEndOfAlgorithm()
+        {
+            if (_initialMapping)
+            {
+                throw new Exception("The ticker generated the initial rename event");
+            }
+            if (!_executionMapping)
+            {
+                throw new Exception("The ticker did not rename throughout the course of its life even though it should have");
+            }
+            if (!_receivedOccurredEvent)
+            {
+                throw new Exception("Did not receive expected split event");
+            }
+            if (!_receivedWarningEvent)
+            {
+                throw new Exception("Did not receive expected split warning event");
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public Language[] Languages { get; } = { Language.CSharp, Language.Python };
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Trades", "0"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "0%"},
+            {"Drawdown", "0%"},
+            {"Expectancy", "0"},
+            {"Net Profit", "0%"},
+            {"Sharpe Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "0%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "-0.427"},
+            {"Tracking Error", "0.158"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "$0.00"},
+            {"Estimated Strategy Capacity", "$0"},
+            {"Lowest Capacity Asset", ""},
+            {"Fitness Score", "0"},
+            {"Kelly Criterion Estimate", "0"},
+            {"Kelly Criterion Probability Value", "0"},
+            {"Sortino Ratio", "79228162514264337593543950335"},
+            {"Return Over Maximum Drawdown", "79228162514264337593543950335"},
+            {"Portfolio Turnover", "0"},
+            {"Total Insights Generated", "0"},
+            {"Total Insights Closed", "0"},
+            {"Total Insights Analysis Completed", "0"},
+            {"Long Insight Count", "0"},
+            {"Short Insight Count", "0"},
+            {"Long/Short Ratio", "100%"},
+            {"Estimated Monthly Alpha Value", "$0"},
+            {"Total Accumulated Estimated Alpha Value", "$0"},
+            {"Mean Population Estimated Insight Value", "$0"},
+            {"Mean Population Direction", "0%"},
+            {"Mean Population Magnitude", "0%"},
+            {"Rolling Averaged Population Direction", "0%"},
+            {"Rolling Averaged Population Magnitude", "0%"},
+            {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"}
+        };
+    }
+}

Algorithm.CSharp/ParameterizedAlgorithm.cs

@@ -28,9 +28,9 @@ namespace QuantConnect.Algorithm.CSharp
     /// <meta name="tag" content="using quantconnect" />
     public class ParameterizedAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
-        // we place attributes on top of our fields or properties that should receive
+        // We place attributes on top of our fields or properties that should receive
         // their values from the job. The values 100 and 200 are just default values that
-        // or only used if the parameters do not exist
+        // are only used if the parameters do not exist.
         [Parameter("ema-fast")]
         public int FastPeriod = 100;
 

Algorithm.CSharp/QuantConnect.Algorithm.CSharp.csproj

@@ -35,7 +35,7 @@
     <DebugType>portable</DebugType>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Accord" Version="3.6.0" />
     <PackageReference Include="Accord.Fuzzy" Version="3.6.0" />
     <PackageReference Include="Accord.MachineLearning" Version="3.6.0" />

Algorithm.Framework/QuantConnect.Algorithm.Framework.csproj

@@ -30,7 +30,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Accord" Version="3.6.0" />
     <PackageReference Include="Accord.Math" Version="3.6.0" />
     <PackageReference Include="Accord.Statistics" Version="3.6.0" />

Algorithm.Python/BasicTemplateIndiaAlgorithm.py

@@ -0,0 +1,49 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+### <summary>
+### Basic template framework algorithm uses framework components to define the algorithm.
+### </summary>
+### <meta name="tag" content="using data" />
+### <meta name="tag" content="using quantconnect" />
+### <meta name="tag" content="trading and orders" />
+class BasicTemplateIndiaAlgorithm(QCAlgorithm):
+    '''Basic template framework algorithm uses framework components to define the algorithm.'''
+
+    def Initialize(self):
+        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
+
+        self.SetStartDate(2019, 1, 23)  #Set Start Date
+        self.SetEndDate(2019, 10, 31)    #Set End Date
+        self.SetCash(100000)           #Set Strategy Cash
+        # Find more symbols here: http://quantconnect.com/data
+        self.AddEquity("YESBANK", Resolution.Minute, Market.India)
+        self.Debug("numpy test >>> print numpy.pi: " + str(np.pi))
+
+        # Set Order Prperties as per the requirements for order placement
+        self.DefaultOrderProperties = IndiaOrderProperties(Exchange.NSE)
+
+    def OnData(self, data):
+        '''OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
+
+        Arguments:
+            data: Slice object keyed by symbol containing the stock data
+        '''
+        if not self.Portfolio.Invested:
+            self.SetHoldings("YESBANK", 1)
+
+    def OnOrderEvent(self, orderEvent):
+        if orderEvent.Status == OrderStatus.Filled:
+            self.Debug("Purchased Stock: {0}".format(orderEvent.Symbol))

Algorithm.Python/IndiaDataRegressionAlgorithm.py

@@ -0,0 +1,81 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+### <summary>
+### Basic template framework algorithm uses framework components to define the algorithm.
+### </summary>
+### <meta name="tag" content="using data" />
+### <meta name="tag" content="using quantconnect" />
+### <meta name="tag" content="trading and orders" />
+class IndiaDataRegressionAlgorithm(QCAlgorithm):
+    '''Basic template framework algorithm uses framework components to define the algorithm.'''
+
+    def Initialize(self):
+        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
+        self.SetStartDate(2004, 5, 20)
+        self.SetEndDate(2016, 7, 26) 
+        self._mappingSymbol = self.AddEquity("3MINDIA", Resolution.Daily, Market.India).Symbol
+        self._splitAndDividendSymbol = self.AddEquity("CCCL", Resolution.Daily, Market.India).Symbol
+        self._receivedWarningEvent = False
+        self._receivedOccurredEvent = False
+        self._initialMapping = False
+        self._executionMapping = False
+        self.Debug("numpy test >>> print numpy.pi: " + str(np.pi))
+
+    def OnData(self, data):
+        '''OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
+
+        Arguments:
+            data: Slice object keyed by symbol containing the stock data
+        '''
+
+        # dividend
+        if data.Dividends.ContainsKey(self._splitAndDividendSymbol):
+            dividend = data.Dividends[self._splitAndDividendSymbol]
+            if ((self.Time.year == 2010 and self.Time.month == 6 and self.Time.day == 15) and
+                    (dividend.Price != 0.5 or dividend.ReferencePrice != 88.8 or dividend.Distribution != 0.5)):
+                raise Exception("Did not receive expected dividend values")
+
+        # split
+        if data.Splits.ContainsKey(self._splitAndDividendSymbol):
+            split = data.Splits[self._splitAndDividendSymbol]
+            if split.Type == SplitType.Warning:
+                self._receivedWarningEvent = True
+            elif split.Type == SplitType.SplitOccurred:
+                self._receivedOccurredEvent = True
+                if split.Price != 421.0 or split.ReferencePrice != 421.0 or split.SplitFactor != 0.2:
+                    raise Exception("Did not receive expected price values")
+
+        # mapping
+        if data.SymbolChangedEvents.ContainsKey(self._mappingSymbol):
+                mappingEvent = [x.Value for x in data.SymbolChangedEvents if x.Key.SecurityType == 1][0]
+                if self.Time.year == 1999 and self.Time.month == 1 and self.Time.day == 1:
+                    self._initialMapping = True
+                elif self.Time.year == 2004 and self.Time.month == 6 and self.Time.day == 15:
+                    if mappingEvent.NewSymbol == "3MINDIA" and mappingEvent.OldSymbol == "BIRLA3M":
+                        self._executionMapping = True
+    
+    def OnEndOfAlgorithm(self):
+        if self._initialMapping:
+            raise Exception("The ticker generated the initial rename event")
+
+        if not self._executionMapping:
+            raise Exception("The ticker did not rename throughout the course of its life even though it should have")
+        
+        if not self._receivedOccurredEvent:
+            raise Exception("Did not receive expected split event")
+
+        if not self._receivedWarningEvent:
+            raise Exception("Did not receive expected split warning event")

Algorithm.Python/QuantConnect.Algorithm.Python.csproj

@@ -37,7 +37,7 @@
     <Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

Algorithm/QCAlgorithm.History.cs

@@ -804,9 +804,30 @@ namespace QuantConnect.Algorithm
             Security security;
             if (Securities.TryGetValue(symbol, out security))
             {
-                return resolution ?? SubscriptionManager.SubscriptionDataConfigService
-                    .GetSubscriptionDataConfigs(symbol)
-                    .GetHighestResolution();
+                if (resolution != null)
+                {
+                    return resolution.Value;
+                }
+
+                Resolution? result = null;
+                var hasNonInternal = false;
+                foreach (var config in SubscriptionManager.SubscriptionDataConfigService
+                    .GetSubscriptionDataConfigs(symbol, includeInternalConfigs: true)
+                    // we process non internal configs first
+                    .OrderBy(config => config.IsInternalFeed ? 1 : 0))
+                {
+                    if (!config.IsInternalFeed || !hasNonInternal)
+                    {
+                        // once we find a non internal config we ignore internals
+                        hasNonInternal |= !config.IsInternalFeed;
+                        if (!result.HasValue || config.Resolution < result)
+                        {
+                            result = config.Resolution;
+                        }
+                    }
+                }
+
+                return result ?? UniverseSettings.Resolution;
             }
             else
             {

Algorithm/QCAlgorithm.Universe.cs

@@ -657,27 +657,9 @@ namespace QuantConnect.Algorithm
                 bar.Symbol = security.Symbol;
                 
                 var maxSupportedResolution = bar.SupportedResolutions().Max();
-
                 var updateFrequency = maxSupportedResolution.ToTimeSpan();
-                int periods;
-                switch (maxSupportedResolution)
-                {
-                    case Resolution.Tick:
-                    case Resolution.Second:
-                        periods = 600;
-                        break;
-                    case Resolution.Minute:
-                        periods = 60 * 24;
-                        break;
-                    case Resolution.Hour:
-                        periods = 24 * 30;
-                        break;
-                    default:
-                        periods = 30;
-                        break;
-                }
                 
-                security.VolatilityModel = new StandardDeviationOfReturnsVolatilityModel(periods, maxSupportedResolution, updateFrequency);
+                security.VolatilityModel = new StandardDeviationOfReturnsVolatilityModel(maxSupportedResolution, updateFrequency);
             }
         }
 

Algorithm/QCAlgorithm.cs

@@ -1680,6 +1680,12 @@ namespace QuantConnect.Algorithm
                             contractDepthOffset: contractOffset
                         );
 
+                        // let's add a MHDB entry for the continuous symbol using the associated security
+                        var continuousContractSymbol = ContinuousContractUniverse.CreateSymbol(security.Symbol);
+                        MarketHoursDatabase.SetEntry(continuousContractSymbol.ID.Market,
+                            continuousContractSymbol.ID.Symbol,
+                            continuousContractSymbol.ID.SecurityType,
+                            security.Exchange.Hours);
                         AddUniverse(new ContinuousContractUniverse(security, new UniverseSettings(settings)
                             {
                                 DataMappingMode = continuousConfigs.First().DataMappingMode,
@@ -1687,7 +1693,7 @@ namespace QuantConnect.Algorithm
                                 ContractDepthOffset = (int)continuousConfigs.First().ContractDepthOffset,
                                 SubscriptionDataTypes = dataTypes
                             }, LiveMode,
-                            new SubscriptionDataConfig(canonicalConfig, symbol: ContinuousContractUniverse.CreateSymbol(security.Symbol))));
+                            new SubscriptionDataConfig(canonicalConfig, symbol: continuousContractSymbol)));
 
                         universe = new FuturesChainUniverse((Future)security, settings);
                     }

Algorithm/QuantConnect.Algorithm.csproj

@@ -30,7 +30,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>

AlgorithmFactory/QuantConnect.AlgorithmFactory.csproj

@@ -29,7 +29,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

Brokerages/InteractiveBrokers/InteractiveBrokersBrokerage.cs

@@ -55,7 +55,7 @@ namespace QuantConnect.Brokerages.InteractiveBrokers
         /// <summary>
         /// The default gateway version to use
         /// </summary>
-        public static string DefaultVersion { get; } = "985";
+        public static string DefaultVersion { get; } = "1012";
 
         private IBAutomater.IBAutomater _ibAutomater;
 

Common/Currencies.cs

@@ -23,24 +23,34 @@ namespace QuantConnect
     public static class Currencies
     {
         /// <summary>
-        /// USD currency string
+        /// USD (United States Dollar) currency string
         /// </summary>
-        public static string USD = "USD";
+        public const string USD = "USD";
 
         /// <summary>
-        /// EUR currency string
+        /// EUR (Euro) currency string
         /// </summary>
-        public static string EUR = "EUR";
+        public const string EUR = "EUR";
 
         /// <summary>
-        /// GBP currency string
+        /// GBP (British pound sterling) currency string
         /// </summary>
-        public static string GBP = "GBP";
+        public const string GBP = "GBP";
 
         /// <summary>
-        /// INR currency string
+        /// INR (Indian rupee) currency string
         /// </summary>
-        public static string INR = "INR";
+        public const string INR = "INR";
+
+        /// <summary>
+        /// CNH (Chinese Yuan Renminbi) currency string
+        /// </summary>
+        public const string CNH = "CNH";
+
+        /// <summary>
+        /// HKD (Hong Kong dollar) currency string
+        /// </summary>
+        public const string HKD = "HKD";
 
         /// <summary>
         /// Null currency used when a real one is not required
@@ -55,19 +65,19 @@ namespace QuantConnect
         /// </remarks>
         public static readonly IReadOnlyDictionary<string, string> CurrencySymbols = new Dictionary<string, string>
         {
-            {"USD", "$"},
-            {"GBP", "₤"},
+            {USD, "$"},
+            {GBP, "₤"},
             {"JPY", "¥"},
-            {"EUR", "€"},
+            {EUR, "€"},
             {"NZD", "$"},
             {"AUD", "$"},
             {"CAD", "$"},
             {"CHF", "Fr"},
-            {"HKD", "$"},
+            {HKD, "$"},
             {"SGD", "$"},
             {"XAG", "Ag"},
             {"XAU", "Au"},
-            {"CNH", "¥"},
+            {CNH, "¥"},
             {"CNY", "¥"},
             {"CZK", "Kč"},
             {"DKK", "kr"},

Common/Data/DiskDataCacheProvider.cs

@@ -16,9 +16,11 @@
 
 using System.IO;
 using Ionic.Zip;
-using QuantConnect.Interfaces;
-using QuantConnect.Logging;
+using System.Linq;
 using QuantConnect.Util;
+using QuantConnect.Logging;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
 
 namespace QuantConnect.Data
 {
@@ -93,6 +95,15 @@ namespace QuantConnect.Data
             Compression.ZipCreateAppendData(filePath, entryName, data, true);
         }
 
+        /// <summary>
+        /// Returns a list of zip entries in a provided zip file
+        /// </summary>
+        public List<string> GetZipEntries(string zipFile)
+        {
+            using var stream = new FileStream(zipFile, FileMode.Open, FileAccess.Read);
+            return Compression.GetZipEntryFileNames(stream).ToList();
+        }
+
         /// <summary>
         /// Dispose for this class
         /// </summary>

Common/Data/Market/OptionContract.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -13,8 +13,10 @@
  * limitations under the License.
 */
 
-using System;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
 using QuantConnect.Securities.Option;
+using System;
 
 namespace QuantConnect.Data.Market
 {
@@ -23,8 +25,7 @@ namespace QuantConnect.Data.Market
     /// </summary>
     public class OptionContract
     {
-        private Lazy<OptionPriceModelResult> _optionPriceModelResult = new Lazy<OptionPriceModelResult>(() => 
-                                                                            new OptionPriceModelResult(0m, new Greeks())); 
+        private Lazy<OptionPriceModelResult> _optionPriceModelResult = new(() => OptionPriceModelResult.None);
 
         /// <summary>
         /// Gets the option contract's symbol
@@ -176,5 +177,40 @@ namespace QuantConnect.Data.Market
         /// A string that represents the current object.
         /// </returns>
         public override string ToString() => Symbol.Value;
+
+        /// <summary>
+        /// Creates a <see cref="OptionContract"/> 
+        /// </summary>
+        /// <param name="baseData"></param>
+        /// <param name="security">provides price properties for a <see cref="Security"/></param>
+        /// <param name="underlyingLastPrice">last price the underlying security traded at</param>
+        /// <returns>Option contract</returns>
+        public static OptionContract Create(BaseData baseData, ISecurityPrice security, decimal underlyingLastPrice)
+            => Create(baseData.Symbol, baseData.Symbol.Underlying, baseData.EndTime, security, underlyingLastPrice);
+
+        /// <summary>
+        /// Creates a <see cref="OptionContract"/>
+        /// </summary>
+        /// <param name="symbol">The option contract symbol</param>
+        /// <param name="underlyingSymbol">The symbol of the underlying security</param>
+        /// <param name="endTime">local date time this contract's data was last updated</param>
+        /// <param name="security">provides price properties for a <see cref="Security"/></param>
+        /// <param name="underlyingLastPrice">last price the underlying security traded at</param>
+        /// <returns>Option contract</returns>
+        public static OptionContract Create(Symbol symbol, Symbol underlyingSymbol, DateTime endTime, ISecurityPrice security, decimal underlyingLastPrice)
+        {
+            return new OptionContract(symbol, underlyingSymbol)
+            {
+                Time = endTime,
+                LastPrice = security.Close,
+                Volume = (long)security.Volume,
+                BidPrice = security.BidPrice,
+                BidSize = (long)security.BidSize,
+                AskPrice = security.AskPrice,
+                AskSize = (long)security.AskSize,
+                OpenInterest = security.OpenInterest,
+                UnderlyingLastPrice = underlyingLastPrice
+            };
+        }
     }
 }

Common/Interfaces/IDataCacheProvider.cs

@@ -16,6 +16,7 @@
 
 using System;
 using System.IO;
+using System.Collections.Generic;
 
 namespace QuantConnect.Interfaces
 {
@@ -42,5 +43,10 @@ namespace QuantConnect.Interfaces
         /// <param name="key">The source of the data, used as a key to retrieve data in the cache</param>
         /// <param name="data">The data to cache as a byte array</param>
         void Store(string key, byte[] data);
+
+        /// <summary>
+        /// Returns a list of zip entries in a provided zip file
+        /// </summary>
+        List<string> GetZipEntries(string zipFile);
     }
 }

Common/Market.cs

@@ -279,5 +279,13 @@ namespace QuantConnect
         {
             return !ReverseMarkets.TryGetValue(code, out var market) ? null : market;
         }
+
+        /// <summary>
+        /// Returns a list of the supported markets
+        /// </summary>
+        public static List<string> SupportedMarkets()
+        {
+            return Markets.Keys.ToList();
+        }
     }
 }

Common/Orders/Fees/InteractiveBrokersFeeModel.cs

@@ -33,9 +33,16 @@ namespace QuantConnect.Orders.Fees
         private readonly Dictionary<string, Func<decimal, decimal, CashAmount>> _optionFee =
             new Dictionary<string, Func<decimal, decimal, CashAmount>>();
 
-        private readonly Dictionary<string, CashAmount> _futureFee =
+        /// <summary>
+        /// Reference at https://www.interactivebrokers.com/en/index.php?f=commission&p=futures1
+        /// </summary>
+        private readonly Dictionary<string, Func<Security, CashAmount>> _futureFee =
             //                                                               IB fee + exchange fee
-            new Dictionary<string, CashAmount> { { Market.USA, new CashAmount(0.85m + 1, "USD") } };
+            new()
+            {
+                { Market.USA, UnitedStatesFutureFees },
+                { Market.HKFE, HongKongFutureFees }
+            };
 
         /// <summary>
         /// Initializes a new instance of the <see cref="ImmediateFillModel"/>
@@ -109,17 +116,18 @@ namespace QuantConnect.Orders.Fees
                     if (market == Market.Globex || market == Market.NYMEX
                         || market == Market.CBOT || market == Market.ICE
                         || market == Market.CFE || market == Market.COMEX
-                        || market == Market.CME || market == Market.HKFE)
+                        || market == Market.CME)
                     {
                         // just in case...
                         market = Market.USA;
                     }
 
-                    CashAmount feeRatePerContract;
-                    if (!_futureFee.TryGetValue(market, out feeRatePerContract))
+                    if (!_futureFee.TryGetValue(market, out var feeRatePerContractFunc))
                     {
                         throw new KeyNotFoundException($"InteractiveBrokersFeeModel(): unexpected future Market {market}");
                     }
+
+                    var feeRatePerContract = feeRatePerContractFunc(security);
                     feeResult = order.AbsoluteQuantity * feeRatePerContract.Amount;
                     feeCurrency = feeRatePerContract.Currency;
                     break;
@@ -129,7 +137,7 @@ namespace QuantConnect.Orders.Fees
                     switch (market)
                     {
                         case Market.USA:
-                            equityFee = new EquityFee("USD", feePerShare: 0.005m, minimumFee: 1, maximumFeeRate: 0.005m);
+                            equityFee = new EquityFee(Currencies.USD, feePerShare: 0.005m, minimumFee: 1, maximumFeeRate: 0.005m);
                             break;
                         default:
                             throw new KeyNotFoundException($"InteractiveBrokersFeeModel(): unexpected equity Market {market}");
@@ -235,6 +243,42 @@ namespace QuantConnect.Orders.Fees
             }
         }
 
+        private static CashAmount UnitedStatesFutureFees(Security security)
+        {
+            return new CashAmount(0.85m + 1, Currencies.USD);
+        }
+
+        /// <summary>
+        /// See https://www.hkex.com.hk/Services/Rules-and-Forms-and-Fees/Fees/Listed-Derivatives/Trading/Transaction?sc_lang=en
+        /// </summary>
+        private static CashAmount HongKongFutureFees(Security security)
+        {
+            if (security.Symbol.ID.Symbol.Equals("HSI", StringComparison.InvariantCultureIgnoreCase))
+            {
+                // IB fee + exchange fee
+                return new CashAmount(30 + 10, Currencies.HKD);
+            }
+
+            decimal ibFeePerContract;
+            switch (security.QuoteCurrency.Symbol)
+            {
+                case Currencies.CNH:
+                    ibFeePerContract = 13;
+                    break;
+                case Currencies.HKD:
+                    ibFeePerContract = 20;
+                    break;
+                case Currencies.USD:
+                    ibFeePerContract = 2.40m;
+                    break;
+                default:
+                    throw new ArgumentException($"Unexpected quote currency {security.QuoteCurrency.Symbol} for Hong Kong futures exchange");
+            }
+
+            // let's add a 50% extra charge for exchange fees
+            return new CashAmount(ibFeePerContract * 1.5m, security.QuoteCurrency.Symbol);
+        }
+
         /// <summary>
         /// Helper class to handle IB Equity fees
         /// </summary>

Common/Packets/BacktestNodePacket.cs

@@ -17,7 +17,6 @@
 using System;
 using System.Collections.Generic;
 using System.Globalization;
-using System.Linq;
 using Newtonsoft.Json;
 using QuantConnect.Securities;
 
@@ -68,22 +67,11 @@ namespace QuantConnect.Packets
         [JsonProperty(PropertyName = "iTradeableDates")]
         public int TradeableDates = 0;
 
-        /// <summary>
-        /// The initial breakpoints for debugging, if any
-        /// </summary>
-        [JsonProperty(PropertyName = "aBreakpoints")]
-        public List<Breakpoint> Breakpoints = new List<Breakpoint>();
-
-        /// <summary>
-        /// The initial Watchlist for debugging, if any
-        /// </summary>
-        [JsonProperty(PropertyName = "aWatchlist")]
-        public List<string> Watchlist = new List<string>();
-
         /// <summary>
         /// True, if this is a debugging backtest
         /// </summary>
-        public bool IsDebugging => Breakpoints.Any();
+        [JsonProperty(PropertyName = "bDebugging")]
+        public bool IsDebugging;
 
         /// <summary>
         /// Optional initial cash amount if set

Common/QuantConnect.csproj

@@ -35,7 +35,7 @@
     <Message Text="SelectedOptimization $(SelectedOptimization)" Importance="high" />
   </Target>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="CloneExtensions" Version="1.3.0" />
     <PackageReference Include="fasterflect" Version="3.0.0" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />

Common/Securities/Option/ConstantQLDividendYieldEstimator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -13,22 +13,15 @@
  * limitations under the License.
 */
 
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
-using QLNet;
-using QuantConnect.Util;
 
 namespace QuantConnect.Securities.Option
 {
     /// <summary>
     /// Class implements default flat dividend yield curve estimator, implementing <see cref="IQLDividendYieldEstimator"/>.  
     /// </summary>
-    class ConstantQLDividendYieldEstimator : IQLDividendYieldEstimator
+    public class ConstantQLDividendYieldEstimator : IQLDividendYieldEstimator
     {
         private readonly double _dividendYield;
         /// <summary>

Common/Securities/Option/ConstantQLRiskFreeRateEstimator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -14,14 +14,8 @@
 */
 
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
-using QuantConnect.Util;
-using QLNet;
 
 namespace QuantConnect.Securities.Option
 {
@@ -30,12 +24,12 @@ namespace QuantConnect.Securities.Option
     /// </summary>
     public class ConstantQLRiskFreeRateEstimator : IQLRiskFreeRateEstimator
     {
-        private readonly double _riskFreeRate;
+        private readonly decimal _riskFreeRate;
         /// <summary>
         /// Constructor initializes class with risk free rate constant
         /// </summary>
         /// <param name="riskFreeRate"></param>
-        public ConstantQLRiskFreeRateEstimator(double riskFreeRate = 0.01)
+        public ConstantQLRiskFreeRateEstimator(decimal riskFreeRate = 0.01m)
         {
             _riskFreeRate = riskFreeRate;
         }
@@ -49,8 +43,6 @@ namespace QuantConnect.Securities.Option
         /// <param name="contract">The option contract to evaluate</param>
         /// <returns>The estimate</returns>
         public double Estimate(Security security, Slice slice, OptionContract contract)
-        {
-            return _riskFreeRate;
-        }
+            => Convert.ToDouble(_riskFreeRate);
     }
 }

Common/Securities/Option/ConstantQLUnderlyingVolatilityEstimator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -13,15 +13,8 @@
  * limitations under the License.
 */
 
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
-using QuantConnect.Util;
-using QLNet;
 
 namespace QuantConnect.Securities.Option
 {
@@ -29,8 +22,13 @@ namespace QuantConnect.Securities.Option
     /// Class implements default underlying constant volatility estimator (<see cref="IQLUnderlyingVolatilityEstimator"/>.), that projects the underlying own volatility 
     /// model into corresponding option pricing model.
     /// </summary>
-    class ConstantQLUnderlyingVolatilityEstimator : IQLUnderlyingVolatilityEstimator
+    public class ConstantQLUnderlyingVolatilityEstimator : IQLUnderlyingVolatilityEstimator
     {
+        /// <summary>
+        /// Indicates whether volatility model has been warmed ot not
+        /// </summary>
+        public bool IsReady { get; private set; }
+
         /// <summary>
         /// Returns current estimate of the underlying volatility
         /// </summary>
@@ -48,6 +46,7 @@ namespace QuantConnect.Securities.Option
                 option.Underlying.VolatilityModel != null &&
                 option.Underlying.VolatilityModel.Volatility > 0m)
             {
+                IsReady = true;
                 return (double)option.Underlying.VolatilityModel.Volatility;
             }
 

Common/Securities/Option/IQLDividendYieldEstimator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -15,12 +15,6 @@
 
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
-using QLNet;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace QuantConnect.Securities.Option
 {
@@ -28,7 +22,7 @@ namespace QuantConnect.Securities.Option
     /// Defines QuantLib dividend yield estimator for option pricing model. User may define his own estimators, 
     /// including those forward and backward looking ones.
     /// </summary>
-    interface IQLDividendYieldEstimator
+    public interface IQLDividendYieldEstimator
     {
         /// <summary>
         /// Returns current estimate of the stock dividend yield

Common/Securities/Option/IQLRiskFreeRateEstimator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -15,19 +15,13 @@
 
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
-using QLNet;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace QuantConnect.Securities.Option
 {
     /// <summary>
     /// Defines QuantLib risk free rate estimator for option pricing model. 
     /// </summary>
-    interface IQLRiskFreeRateEstimator
+    public interface IQLRiskFreeRateEstimator
     {
         /// <summary>
         /// Returns current estimate of the risk free rate

Common/Securities/Option/IQLUnderlyingVolatilityEstimator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -15,12 +15,6 @@
 
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
-using QLNet;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace QuantConnect.Securities.Option
 {
@@ -28,7 +22,7 @@ namespace QuantConnect.Securities.Option
     /// Defines QuantLib underlying volatility estimator for option pricing model. User may define his own estimators, 
     /// including those forward and backward looking ones.
     /// </summary>
-    interface IQLUnderlyingVolatilityEstimator
+    public interface IQLUnderlyingVolatilityEstimator
     {
         /// <summary>
         /// Returns current estimate of the underlying volatility
@@ -39,5 +33,10 @@ namespace QuantConnect.Securities.Option
         /// <param name="contract">The option contract to evaluate</param>
         /// <returns>Volatility</returns>
         double Estimate(Security security, Slice slice, OptionContract contract);
+
+        /// <summary>
+        /// Indicates whether volatility model is warmed up or no
+        /// </summary>
+        bool IsReady { get; }
     }
 }

Common/Securities/Option/OptionPriceModelResult.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -23,6 +23,11 @@ namespace QuantConnect.Securities.Option
     /// </summary>
     public class OptionPriceModelResult
     {
+        /// <summary>
+        /// Represents the zero option price and greeks.
+        /// </summary>
+        public static OptionPriceModelResult None { get; } = new(0, new Greeks());
+
         private readonly Lazy<Greeks> _greeks;
         private readonly Lazy<decimal> _impliedVolatility;
 
@@ -81,4 +86,4 @@ namespace QuantConnect.Securities.Option
             _greeks = new Lazy<Greeks>(greeks);
         }
     }
-}
+}

Common/Securities/Option/OptionPriceModels.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -13,12 +13,11 @@
  * limitations under the License.
 */
 
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using QLNet;
+using System;
+using System.Globalization;
+using System.Linq;
+using Fasterflect;
 
 namespace QuantConnect.Securities.Option
 {
@@ -43,7 +42,27 @@ namespace QuantConnect.Securities.Option
         private const int _timeStepsFD = 100;
 
         /// <summary>
-        /// Pricing engine for European vanilla options using analytical formulae. 
+        /// Creates pricing engine by engine type name. 
+        /// </summary>
+        /// <param name="priceEngineName">QL price engine name</param>
+        /// <param name="riskFree">The risk free rate</param>
+        /// <returns>New option price model instance of specific engine</returns>
+        public static IOptionPriceModel Create(string priceEngineName, decimal riskFree)
+        {
+            var type = AppDomain.CurrentDomain.GetAssemblies()
+                .Where(a => !a.IsDynamic)
+                .SelectMany(a => a.GetTypes())
+                .Where(s => s.Implements(typeof(IPricingEngine)))
+                .FirstOrDefault(t => t.FullName?.EndsWith(priceEngineName, StringComparison.InvariantCulture) == true);
+
+            return new QLOptionPriceModel(process => (IPricingEngine)Activator.CreateInstance(type, process),
+                _underlyingVolEstimator,
+                new ConstantQLRiskFreeRateEstimator(riskFree),
+                _dividendYieldEstimator);
+        }
+
+        /// <summary>
+        /// Pricing engine for European vanilla options using analytical formula. 
         /// QuantLib reference: http://quantlib.org/reference/class_quant_lib_1_1_analytic_european_engine.html
         /// </summary>
         /// <returns>New option price model instance</returns>
@@ -103,7 +122,7 @@ namespace QuantConnect.Securities.Option
         {
             PricingEngineFuncEx pricingEngineFunc = (symbol, process) =>
                             symbol.ID.OptionStyle == OptionStyle.American ?
-                            new FDAmericanEngine(process, _timeStepsFD, _timeStepsFD - 1) as IPricingEngine:
+                            new FDAmericanEngine(process, _timeStepsFD, _timeStepsFD - 1) as IPricingEngine :
                             new FDEuropeanEngine(process, _timeStepsFD, _timeStepsFD - 1) as IPricingEngine;
 
             return new QLOptionPriceModel(pricingEngineFunc,

Common/Securities/Option/QLOptionPriceModel.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -28,7 +28,7 @@ namespace QuantConnect.Securities.Option
     /// <summary>
     /// Provides QuantLib(QL) implementation of <see cref="IOptionPriceModel"/> to support major option pricing models, available in QL.
     /// </summary>
-    class QLOptionPriceModel : IOptionPriceModel
+    public class QLOptionPriceModel : IOptionPriceModel
     {
         private readonly IQLUnderlyingVolatilityEstimator _underlyingVolEstimator;
         private readonly IQLRiskFreeRateEstimator _riskFreeRateEstimator;
@@ -41,6 +41,11 @@ namespace QuantConnect.Securities.Option
         /// </summary>
         public bool EnableGreekApproximation { get; set; } = true;
 
+        /// <summary>
+        /// True if volatility model is warmed up, i.e. has generated volatility value different from zero, otherwise false.
+        /// </summary>
+        public bool VolatilityEstimatorWarmedUp => _underlyingVolEstimator.IsReady;
+
         /// <summary>
         /// Method constructs QuantLib option price model with necessary estimators of underlying volatility, risk free rate, and underlying dividend yield
         /// </summary>
@@ -83,6 +88,12 @@ namespace QuantConnect.Securities.Option
         {
             try
             {
+                // expired options has no price
+                if (contract.Time > contract.Expiry)
+                {
+                    return OptionPriceModelResult.None;
+                }
+
                 // setting up option pricing parameters
                 var calendar = new UnitedStates();
                 var dayCounter = new Actual365Fixed();
@@ -101,6 +112,11 @@ namespace QuantConnect.Securities.Option
                 var underlyingVolValue = new SimpleQuote(_underlyingVolEstimator.Estimate(security, slice, contract));
                 var underlyingVol = new Handle<BlackVolTermStructure>(new BlackConstantVol(0, calendar, new Handle<Quote>(underlyingVolValue), dayCounter));
 
+                if (!_underlyingVolEstimator.IsReady)
+                {
+                    return OptionPriceModelResult.None;
+                }
+
                 // preparing stochastic process and payoff functions
                 var stochasticProcess = new BlackScholesMertonProcess(new Handle<Quote>(underlyingQuoteValue), dividendYield, riskFreeRate, underlyingVol);
                 var payoff = new PlainVanillaPayoff(contract.Right == OptionRight.Call ? QLNet.Option.Type.Call : QLNet.Option.Type.Put, (double)contract.Strike);
@@ -116,7 +132,8 @@ namespace QuantConnect.Securities.Option
                 option.setPricingEngine(_pricingEngineFunc(contract.Symbol, stochasticProcess));
 
                 // running calculations
-                var npv = EvaluateOption(option);
+                // can return negative value in neighbourhood of 0
+                var npv = Math.Max(0, EvaluateOption(option));
 
                 // function extracts QL greeks catching exception if greek is not generated by the pricing engine and reevaluates option to get numerical estimate of the seisitivity
                 Func<Func<double>, Func<double>, decimal> tryGetGreekOrReevaluate = (greek, reevalFunc) =>
@@ -217,10 +234,10 @@ namespace QuantConnect.Securities.Option
                                             () => tryGetGreekOrReevaluate(() => option.rho(), reevalRho),
                                             () => tryGetGreek(() => option.elasticity())));
             }
-            catch(Exception err)
+            catch (Exception err)
             {
                 Log.Debug($"QLOptionPriceModel.Evaluate() error: {err.Message}");
-                return new OptionPriceModelResult(0m, new Greeks());
+                return OptionPriceModelResult.None;
             }
         }
 

Common/Securities/Volatility/StandardDeviationOfReturnsVolatilityModel.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -35,10 +35,10 @@ namespace QuantConnect.Securities
         private decimal _volatility;
         private DateTime _lastUpdate = DateTime.MinValue;
         private decimal _lastPrice;
-        private readonly Resolution? _resolution;
-        private readonly TimeSpan _periodSpan;
+        private Resolution? _resolution;
+        private TimeSpan _periodSpan;
         private readonly object _sync = new object();
-        private readonly RollingWindow<double> _window;
+        private RollingWindow<double> _window;
 
         /// <summary>
         /// Gets the volatility of the security as a percentage
@@ -92,7 +92,7 @@ namespace QuantConnect.Securities
         {
             if (periods < 2)
             {
-                throw new ArgumentOutOfRangeException("periods", "'periods' must be greater than or equal to 2.");
+                throw new ArgumentOutOfRangeException(nameof(periods), "'periods' must be greater than or equal to 2.");
             }
 
             _window = new RollingWindow<double>(periods);
@@ -100,6 +100,30 @@ namespace QuantConnect.Securities
             _periodSpan = updateFrequency ?? resolution?.ToTimeSpan() ?? TimeSpan.FromDays(1);
         }
 
+        /// <summary>
+        /// Initializes a new instance of the <see cref="StandardDeviationOfReturnsVolatilityModel"/> class
+        /// </summary>
+        /// <param name="resolution">
+        /// Resolution of the price data inserted into the rolling window series to calculate standard deviation.
+        /// Will be used as the default value for update frequency if a value is not provided for <paramref name="updateFrequency"/>.
+        /// This only has a material effect in live mode. For backtesting, this value does not cause any behavioral changes.
+        /// </param>
+        /// <param name="updateFrequency">Frequency at which we insert new values into the rolling window for the standard deviation calculation</param>
+        /// <remarks>
+        /// The volatility model will be updated with the most granular/highest resolution data that was added to your algorithm.
+        /// That means that if I added <see cref="Resolution.Tick"/> data for my Futures strategy, that this model will be
+        /// updated using <see cref="Resolution.Tick"/> data as the algorithm progresses in time.
+        ///
+        /// Keep this in mind when setting the period and update frequency. The Resolution parameter is only used for live mode, or for
+        /// the default value of the <paramref name="updateFrequency"/> if no value is provided.
+        /// </remarks>
+        public StandardDeviationOfReturnsVolatilityModel(
+            Resolution resolution,
+            TimeSpan? updateFrequency = null
+            ) : this(PeriodsInResolution(resolution), resolution, updateFrequency)
+        {
+        }
+
         /// <summary>
         /// Updates this model using the new price information in
         /// the specified security instance
@@ -139,5 +163,28 @@ namespace QuantConnect.Securities
                 _resolution,
                 _window.Size + 1);
         }
+
+        private static int PeriodsInResolution(Resolution resolution)
+        {
+            int periods;
+            switch (resolution)
+            {
+                case Resolution.Tick:
+                case Resolution.Second:
+                    periods = 600;
+                    break;
+                case Resolution.Minute:
+                    periods = 60 * 24;
+                    break;
+                case Resolution.Hour:
+                    periods = 24 * 30;
+                    break;
+                default:
+                    periods = 30;
+                    break;
+            }
+
+            return periods;
+        }
     }
-}
+}

Configuration/Config.cs

@@ -298,8 +298,8 @@ namespace QuantConnect.Configuration
         /// </summary>
         /// <typeparam name="T">The desired output type</typeparam>
         /// <param name="key">The configuration key</param>
-        /// <param name="value">The output value</param>
-        /// <returns>True on successful parse, false when output value is default(T)</returns>
+        /// <param name="value">The output value. If the key is found and parsed successfully, it will be the parsed value, else default(T).</param>
+        /// <returns>True on successful parse or if they key is not found. False only when key is found but fails to parse.</returns>
         public static bool TryGetValue<T>(string key, out T value)
         {
             return TryGetValue(key, default(T), out value);
@@ -312,8 +312,8 @@ namespace QuantConnect.Configuration
         /// <typeparam name="T">The desired output type</typeparam>
         /// <param name="key">The configuration key</param>
         /// <param name="defaultValue">The default value to use on key not found or unsuccessful parse</param>
-        /// <param name="value">The output value</param>
-        /// <returns>True on successful parse, false when output value is defaultValue</returns>
+        /// <param name="value">The output value. If the key is found and parsed successfully, it will be the parsed value, else defaultValue.</param>
+        /// <returns>True on successful parse or if they key is not found and using defaultValue. False only when key is found but fails to parse.</returns>
         public static bool TryGetValue<T>(string key, T defaultValue, out T value)
         {
             try

Configuration/ToolboxArgumentParser.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -77,6 +77,9 @@ namespace QuantConnect.Configuration
                 new CommandLineOption("splits-percentage", CommandOptionType.SingleValue, "[OPTIONAL for RandomDataGenerator. Sets the probability each equity generated will have a stock split event. Note that this is not the total probability for all symbols generated. Only used for Equity. Defaults to 15.0: Example: --splits-percentage=10.0 ]"),
                 new CommandLineOption("dividends-percentage", CommandOptionType.SingleValue, "[OPTIONAL for RandomDataGenerator. Sets the probability each equity generated will have dividends. Note that this is not the probability for all symbols genearted. Only used for Equity. Defaults to 60.0: Example: --dividends-percentage=25.5 ]"),
                 new CommandLineOption("dividend-every-quarter-percentage", CommandOptionType.SingleValue, "[OPTIONAL for RandomDataGenerator. Sets the probability each equity generated will have a dividend event every quarter. Note that this is not the total probability for all symbols generated. Only used for Equity. Defaults to 30.0: Example: --dividend-every-quarter-percentage=15.0 ]"),
+                new CommandLineOption("option-price-engine", CommandOptionType.SingleValue, "[OPTIONAL for RandomDataGenerator. Sets the stochastic process, and returns new pricing engine to run calculations for that option. Defaults to BaroneAdesiWhaleyApproximationEngine: Example: --option-price-engine=BaroneAdesiWhaleyApproximationEngine ]"),
+                new CommandLineOption("volatility-model-resolution", CommandOptionType.SingleValue, "[OPTIONAL for RandomDataGenerator. Sets the volatility model period span. Defaults to Daily: Example: --volatility-model-resolution=Daily ]"),
+                new CommandLineOption("chain-symbol-count", CommandOptionType.SingleValue, "[OPTIONAL for RandomDataGenerator. Sets the size of the option chain. Defaults to 1 put and 1 call: Example: --chain-symbol-count=2 ]")
             };
 
         /// <summary>

Data/equity/india/factor_files/cccl.csv

@@ -0,0 +1,5 @@
+20100104,0.9800619,0.2,413
+20100209,0.9800619,0.2,421
+20100615,0.9800619,1,88.8
+20110615,0.9856115,1,34.75
+20501231,1,1,0

Data/equity/india/map_files/3mindia.csv

@@ -0,0 +1,3 @@
+19990101,birla3m
+20040615,birla3m
+20501231,3mindia

Dockerfile

@@ -17,6 +17,10 @@ RUN pip install pydevd-pycharm~=201.8538.36
 # Install vsdbg for remote C# debugging in Visual Studio and Visual Studio Code
 RUN wget https://aka.ms/getvsdbgsh -O - 2>/dev/null | /bin/sh /dev/stdin -v 16.9.20122.2 -l /root/vsdbg
 
+# Install NetCoreDbg
+RUN wget https://github.com/Samsung/netcoredbg/releases/download/2.0.0-880/netcoredbg-linux-amd64.tar.gz \
+&& tar xvzf netcoredbg-linux-amd64.tar.gz && rm netcoredbg-linux-amd64.tar.gz
+
 COPY ./DataLibraries /Lean/Launcher/bin/Debug/
 COPY ./AlphaStreams/QuantConnect.AlphaStream/bin/Debug/ /Lean/Launcher/bin/Debug/
 COPY ./Lean/Launcher/bin/Debug/ /Lean/Launcher/bin/Debug/

DockerfileLeanFoundation

@@ -29,10 +29,10 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
 
 # Install IB Gateway: Installs to /root/ibgateway
 RUN mkdir -p /root/ibgateway && \
-    wget https://cdn.quantconnect.com/interactive/ibgateway-latest-standalone-linux-x64.v985.1j.sh && \
-    chmod 777 ibgateway-latest-standalone-linux-x64.v985.1j.sh && \
-    ./ibgateway-latest-standalone-linux-x64.v985.1j.sh -q -dir /root/ibgateway && \
-    rm ibgateway-latest-standalone-linux-x64.v985.1j.sh
+    wget https://cdn.quantconnect.com/interactive/ibgateway-latest-standalone-linux-x64.v10.12.2d.sh && \
+    chmod 777 ibgateway-latest-standalone-linux-x64.v10.12.2d.sh && \
+    ./ibgateway-latest-standalone-linux-x64.v10.12.2d.sh -q -dir /root/ibgateway && \
+    rm ibgateway-latest-standalone-linux-x64.v10.12.2d.sh
 
 # Install dotnet 5 sdk & runtime
 RUN wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb && \
@@ -53,9 +53,9 @@ ENV PATH="/opt/miniconda3/bin:${PATH}"
 RUN wget https://cdn.quantconnect.com/miniconda/${CONDA} && \
     bash ${CONDA} -b -p /opt/miniconda3 && rm -rf ${CONDA} && \
     ln -s /opt/miniconda3/lib/libpython3.6m.so /usr/lib/libpython3.6m.so && \
-    conda update -y conda pip && \
-    pip install --upgrade --no-cache-dir pip && \
-    conda install -y python=3.6.8 && conda clean -y --all
+    conda install -y conda=4.10.3 && \
+    pip install --upgrade --no-cache-dir pip==21.2.2 && \
+    conda install -y python=3.6.8 && conda install -y pip=21.2.2 && conda clean -y --all
 
 # Avoid pip install read timeouts
 ENV PIP_DEFAULT_TIMEOUT=120
@@ -167,7 +167,7 @@ RUN pip install --no-cache-dir      \
     ruptures==1.1.3                 \
     simpy==4.0.1                    \
     scikit-learn-extra==0.2.0       \
-    ray==1.5.1
+    ray==1.9.1
 
 # feature_selector has overly strict dependency version ranges
 # We already installed close-enough versions of all of its dependencies above
@@ -259,10 +259,10 @@ RUN wget https://cdn.quantconnect.com/tigramite/tigramite-4.1.zip && \
     python setup.py install && cd .. && rm -rf tigramite-4.1 && rm tigramite-4.1.zip
 
 # Install H2O: https://www.h2o.ai/download/
-RUN wget https://cdn.quantconnect.com/h2o/h2o-3.30.0.3.zip && \
-    unzip -q h2o-3.30.0.3.zip && \
-    pip install h2o-3.30.0.3/python/h2o-3.30.0.3-py2.py3-none-any.whl && \
-    rm -rf h2o-3.30.0.3 && rm h2o-3.30.0.3.zip
+RUN wget https://cdn.quantconnect.com/h2o/h2o-3.34.0.7.zip && \
+    unzip -q h2o-3.34.0.7.zip && \
+    pip install h2o-3.34.0.7-py2.py3-none-any.whl && \
+    rm h2o-3.34.0.7.zip h2o-3.34.0.7-py2.py3-none-any.whl
 
 # Remove black-listed packages
 RUN pip uninstall -y s3transfer

DockerfileLeanFoundationARM

@@ -27,12 +27,12 @@ RUN apt-get update && apt-fast install -y git libgtk2.0.0 cmake bzip2 curl unzip
 # We update the install script so it doesn't use the bundled JVM
 # The bundled JVM doesn't work on ARM64, so we update it to use the JVM installed in the previous command
 RUN mkdir -p /root/ibgateway && \
-    wget https://cdn.quantconnect.com/interactive/ibgateway-latest-standalone-linux-x64.v985.1j.sh && \
+    wget https://cdn.quantconnect.com/interactive/ibgateway-latest-standalone-linux-x64.v10.12.2d.sh && \
     java_patch_version=$(java -version 2>&1 | head -n 1 | cut -d'_' -f2 | cut -d'"' -f1) && \
-    bbe -e 's|# INSTALL4J_JAVA_HOME_OVERRIDE=|INSTALL4J_JAVA_HOME_OVERRIDE="/usr/lib/jvm/java-1.8.0-openjdk-arm64"|' -e "s|-lt \"152\"|-lt \"$java_patch_version\"|" -e "s|-gt \"152\"|-gt \"$java_patch_version\"|" ibgateway-latest-standalone-linux-x64.v985.1j.sh > ibgateway-stable-standalone-linux-custom-jvm.sh && \
+    bbe -e 's|# INSTALL4J_JAVA_HOME_OVERRIDE=|INSTALL4J_JAVA_HOME_OVERRIDE="/usr/lib/jvm/java-1.8.0-openjdk-arm64"|' -e "s|-lt \"152\"|-lt \"$java_patch_version\"|" -e "s|-gt \"152\"|-gt \"$java_patch_version\"|" ibgateway-latest-standalone-linux-x64.v10.12.2d.sh > ibgateway-stable-standalone-linux-custom-jvm.sh && \
     chmod 777 ibgateway-stable-standalone-linux-custom-jvm.sh && \
     ./ibgateway-stable-standalone-linux-custom-jvm.sh -q -dir /root/ibgateway && \
-    rm ibgateway-latest-standalone-linux-x64.v985.1j.sh ibgateway-stable-standalone-linux-custom-jvm.sh
+    rm ibgateway-latest-standalone-linux-x64.v10.12.2d.sh ibgateway-stable-standalone-linux-custom-jvm.sh
 
 # Install dotnet 5 sdk & runtime
 # The .deb packages don't support ARM, the install script does
@@ -54,9 +54,9 @@ RUN wget -O Miniforge3.sh https://github.com/conda-forge/miniforge/releases/late
     rm Miniforge3.sh && \
     ln -s /opt/miniforge3/lib/libpython3.6m.so /usr/lib/libpython3.6m.so && \
     conda config --append channels https://repo.anaconda.com/pkgs/main && \
-    conda update -y conda pip && \
-    pip install --upgrade --no-cache-dir pip && \
-    conda install -y python=3.6.7 && conda clean -y --all
+    conda install -y conda=4.10.3 && \
+    pip install --upgrade --no-cache-dir pip==21.2.2 && \
+    conda install -y python=3.6.7 && conda install -y pip=21.2.2 && conda clean -y --all
 
 # Avoid pip install read timeouts
 ENV PIP_DEFAULT_TIMEOUT=120
@@ -179,10 +179,10 @@ RUN wget https://cdn.quantconnect.com/tigramite/tigramite-4.1.zip && \
     python setup.py install && cd .. && rm -rf tigramite-4.1 && rm tigramite-4.1.zip
 
 # Install H2O: https://www.h2o.ai/download/
-RUN wget https://cdn.quantconnect.com/h2o/h2o-3.30.0.3.zip && \
-    unzip -q h2o-3.30.0.3.zip && \
-    pip install h2o-3.30.0.3/python/h2o-3.30.0.3-py2.py3-none-any.whl && \
-    rm -rf h2o-3.30.0.3 && rm h2o-3.30.0.3.zip
+RUN wget https://cdn.quantconnect.com/h2o/h2o-3.34.0.7.zip && \
+    unzip -q h2o-3.34.0.7.zip && \
+    pip install h2o-3.34.0.7-py2.py3-none-any.whl && \
+    rm h2o-3.34.0.7.zip h2o-3.34.0.7-py2.py3-none-any.whl
 
 # Remove black-listed packages
 RUN pip uninstall -y s3transfer

Engine/DataFeeds/ApiDataProvider.cs

@@ -42,6 +42,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         private readonly DataPricesList _dataPrices;
         private readonly Api.Api _api;
         private readonly bool _subscribedToEquityMapAndFactorFiles;
+        private readonly bool _subscribedToFutureMapAndFactorFiles;
         private volatile bool _invalidSecurityTypeLog;
 
         /// <summary>
@@ -65,10 +66,18 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             _dataPrices = _api.ReadDataPrices(_organizationId);
             var organization = _api.ReadOrganization(_organizationId);
 
-            // Determine if the user is subscribed to map and factor files (Data product Id 37)
-            if (organization.Products.Where(x => x.Type == ProductType.Data).Any(x => x.Items.Any(x => x.Id == 37)))
+            foreach (var productItem in organization.Products.Where(x => x.Type == ProductType.Data).SelectMany(product => product.Items))
             {
-                _subscribedToEquityMapAndFactorFiles = true;
+                if (productItem.Id == 37)
+                {
+                    // Determine if the user is subscribed to Equity map and factor files (Data product Id 37)
+                    _subscribedToEquityMapAndFactorFiles = true;
+                }
+                else if (productItem.Id == 137)
+                {
+                    // Determine if the user is subscribed to Future map and factor files (Data product Id 137)
+                    _subscribedToFutureMapAndFactorFiles = true;
+                }
             }
 
             // Verify user has agreed to data provider agreements
@@ -154,23 +163,41 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             // Some security types can't be downloaded, lets attempt to extract that information
             if (LeanData.TryParseSecurityType(filePath, out SecurityType securityType) && _unsupportedSecurityType.Contains(securityType))
             {
-                if (!_invalidSecurityTypeLog)
+                // we do support future auxiliary data (map and factor files)
+                if (securityType != SecurityType.Future || !IsAuxiliaryData(filePath))
                 {
-                    // let's log this once. Will still use any existing data on disk
-                    _invalidSecurityTypeLog = true;
-                    Log.Error($"ApiDataProvider(): does not support security types: {string.Join(", ", _unsupportedSecurityType)}");
+                    if (!_invalidSecurityTypeLog)
+                    {
+                        // let's log this once. Will still use any existing data on disk
+                        _invalidSecurityTypeLog = true;
+                        Log.Error($"ApiDataProvider(): does not support security types: {string.Join(", ", _unsupportedSecurityType)}");
+                    }
+                    return false;
                 }
-                return false;
             }
 
             // Only download if it doesn't exist or is out of date.
             // Files are only "out of date" for non date based files (hour, daily, margins, etc.) because this data is stored all in one file
             var shouldDownload = !File.Exists(filePath) || filePath.IsOutOfDate();
 
-            // Final check; If we want to download and the request requires equity data we need to be sure they are subscribed to map and factor files
-            if (shouldDownload && (securityType == SecurityType.Equity || securityType == SecurityType.Option || IsEquitiesAux(filePath)))
+            if (shouldDownload)
             {
-                CheckMapFactorFileSubscription();
+                if (securityType == SecurityType.Future)
+                {
+                    if (!_subscribedToFutureMapAndFactorFiles)
+                    {
+                        throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
+                            "to download Future auxiliary data from QuantConnect. " +
+                            "Please visit https://www.quantconnect.com/datasets/quantconnect-us-futures-security-master for details.");
+                    }
+                }
+                // Final check; If we want to download and the request requires equity data we need to be sure they are subscribed to map and factor files
+                else if (!_subscribedToEquityMapAndFactorFiles && (securityType == SecurityType.Equity || securityType == SecurityType.Option || IsAuxiliaryData(filePath)))
+                {
+                    throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
+                        "to download Equity data from QuantConnect. " +
+                        "Please visit https://www.quantconnect.com/datasets/quantconnect-security-master for details.");
+                }
             }
 
             return shouldDownload;
@@ -198,29 +225,16 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         }
 
         /// <summary>
-        /// Helper method to determine if this filepath is Equity Aux data
+        /// Helper method to determine if this filepath is auxiliary data
         /// </summary>
-        /// <param name="filepath"></param>
-        /// <returns>True if this file is EquitiesAux</returns>
-        private static bool IsEquitiesAux(string filepath)
+        /// <param name="filepath">The target file path</param>
+        /// <returns>True if this file is of auxiliary data</returns>
+        private static bool IsAuxiliaryData(string filepath)
         {
             return filepath.Contains("map_files", StringComparison.InvariantCulture)
                 || filepath.Contains("factor_files", StringComparison.InvariantCulture)
                 || filepath.Contains("fundamental", StringComparison.InvariantCulture)
                 || filepath.Contains("shortable", StringComparison.InvariantCulture);
         }
-
-        /// <summary>
-        /// Helper to check map and factor file subscription, throws if not subscribed.
-        /// </summary>
-        private void CheckMapFactorFileSubscription()
-        {
-            if(!_subscribedToEquityMapAndFactorFiles)
-            {
-                throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
-                    "to download Equity data from QuantConnect. " + 
-                    "Please visit https://www.quantconnect.com/datasets/quantconnect-security-master for details.");
-            }
-        }
     }
 }

Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactory.cs

@@ -31,6 +31,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
     {
         private readonly Func<SubscriptionRequest, IEnumerable<DateTime>> _tradableDaysProvider;
         private readonly IMapFileProvider _mapFileProvider;
+        private readonly IDataCacheProvider _dataCacheProvider;
         private readonly bool _isLiveMode;
 
         /// <summary>
@@ -38,13 +39,10 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
         /// </summary>
         /// <param name="isLiveMode">True for live mode, false otherwise</param>
         /// <param name="mapFileProvider">Used for resolving the correct map files</param>
-        /// <param name="factorFileProvider">Used for getting factor files</param>
-        /// <param name="tradableDaysProvider">Function used to provide the tradable dates to be enumerator.
-        /// Specify null to default to <see cref="SubscriptionRequest.TradableDays"/></param>
-        public BaseDataSubscriptionEnumeratorFactory(bool isLiveMode, IMapFileProvider mapFileProvider, IFactorFileProvider factorFileProvider, Func<SubscriptionRequest, IEnumerable<DateTime>> tradableDaysProvider = null)
+        /// <param name="dataCacheProvider">The cache provider instance to use</param>
+        public BaseDataSubscriptionEnumeratorFactory(bool isLiveMode, IMapFileProvider mapFileProvider, IDataCacheProvider dataCacheProvider)
+            : this(isLiveMode, dataCacheProvider)
         {
-            _isLiveMode = isLiveMode;
-            _tradableDaysProvider = tradableDaysProvider ?? (request => request.TradableDays);
             _mapFileProvider = mapFileProvider;
         }
 
@@ -52,12 +50,12 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
         /// Initializes a new instance of the <see cref="BaseDataSubscriptionEnumeratorFactory"/> class
         /// </summary>
         /// <param name="isLiveMode">True for live mode, false otherwise</param>
-        /// <param name="tradableDaysProvider">Function used to provide the tradable dates to be enumerator.
-        /// Specify null to default to <see cref="SubscriptionRequest.TradableDays"/></param>
-        public BaseDataSubscriptionEnumeratorFactory(bool isLiveMode, Func<SubscriptionRequest, IEnumerable<DateTime>> tradableDaysProvider = null)
+        /// <param name="dataCacheProvider">The cache provider instance to use</param>
+        public BaseDataSubscriptionEnumeratorFactory(bool isLiveMode, IDataCacheProvider dataCacheProvider)
         {
             _isLiveMode = isLiveMode;
-            _tradableDaysProvider = tradableDaysProvider ?? (request => request.TradableDays);
+            _dataCacheProvider = dataCacheProvider;
+            _tradableDaysProvider = (request => request.TradableDays);
         }
 
         /// <summary>
@@ -73,28 +71,19 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
             // It has the added benefit of caching any zip files that we request from the filesystem, and reading
             // files contained within the zip file, which the SingleEntryDataCacheProvider does not support.
             var sourceFactory = request.Configuration.GetBaseDataInstance();
-            using (var dataCacheProvider = new ZipDataCacheProvider(dataProvider))
+            foreach (var date in _tradableDaysProvider(request))
             {
-                foreach (var date in _tradableDaysProvider(request))
+                if (sourceFactory.RequiresMapping() && _mapFileProvider != null)
                 {
-                    if (sourceFactory.RequiresMapping() && _mapFileProvider != null)
-                    {
-                        request.Configuration.MappedSymbol = GetMappedSymbol(request.Configuration, date);
-                    }
+                    request.Configuration.MappedSymbol = GetMappedSymbol(request.Configuration, date);
+                }
 
-                    var source = sourceFactory.GetSource(request.Configuration, date, _isLiveMode);
-                    var factory = SubscriptionDataSourceReader.ForSource(source, dataCacheProvider, request.Configuration, date, _isLiveMode, sourceFactory, dataProvider);
-                    var entriesForDate = factory.Read(source);
-                    foreach (var entry in entriesForDate)
-                    {
-                        // Fix for Daily/Hour options cases when reading in all equity data from daily/hourly file
-                        if (entry.Time.Date != date)
-                        {
-                            continue;
-                        }
-
-                        yield return entry;
-                    }
+                var source = sourceFactory.GetSource(request.Configuration, date, _isLiveMode);
+                var factory = SubscriptionDataSourceReader.ForSource(source, _dataCacheProvider, request.Configuration, date, _isLiveMode, sourceFactory, dataProvider);
+                var entriesForDate = factory.Read(source);
+                foreach (var entry in entriesForDate)
+                {
+                    yield return entry;
                 }
             }
         }

Engine/DataFeeds/Enumerators/Factories/FuturesChainUniverseSubscriptionEnumeratorFactory.cs

@@ -16,7 +16,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using QuantConnect.Data;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Interfaces;
@@ -32,15 +31,18 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
         private readonly bool _isLiveMode;
 
         private readonly IDataQueueUniverseProvider _symbolUniverse;
+        private readonly IDataCacheProvider _dataCacheProvider;
         private readonly ITimeProvider _timeProvider;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="FuturesChainUniverseSubscriptionEnumeratorFactory"/> class
         /// </summary>
         /// <param name="enumeratorConfigurator">Function used to configure the sub-enumerators before sync (fill-forward/filter/ect...)</param>
-        public FuturesChainUniverseSubscriptionEnumeratorFactory(Func<SubscriptionRequest, IEnumerator<BaseData>, IEnumerator<BaseData>> enumeratorConfigurator)
+        /// <param name="dataCacheProvider">The cache provider instance to use</param>
+        public FuturesChainUniverseSubscriptionEnumeratorFactory(Func<SubscriptionRequest, IEnumerator<BaseData>, IEnumerator<BaseData>> enumeratorConfigurator, IDataCacheProvider dataCacheProvider)
         {
             _isLiveMode = false;
+            _dataCacheProvider = dataCacheProvider;
             _enumeratorConfigurator = enumeratorConfigurator;
         }
 
@@ -67,36 +69,28 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
         {
             if (_isLiveMode)
             {
-                var subscriptionConfiguration = GetSubscriptionConfigurations(request).First();
-                var subscriptionRequest = new SubscriptionRequest(request, configuration: subscriptionConfiguration);
+                var subscriptionRequest = new SubscriptionRequest(request, configuration: GetSubscriptionConfiguration(request));
 
                 return new DataQueueFuturesChainUniverseDataCollectionEnumerator(subscriptionRequest, _symbolUniverse, _timeProvider);
             }
             else
             {
-                var factory = new BaseDataSubscriptionEnumeratorFactory(_isLiveMode);
+                var factory = new BaseDataSubscriptionEnumeratorFactory(_isLiveMode, _dataCacheProvider);
 
-                var enumerators = GetSubscriptionConfigurations(request)
-                    .Select(c => new SubscriptionRequest(request, configuration: c))
-                    .Select(sr => _enumeratorConfigurator(request, factory.CreateEnumerator(sr, dataProvider)));
+                var newRequest = new SubscriptionRequest(request, configuration: GetSubscriptionConfiguration(request));
+                var enumerator = _enumeratorConfigurator(request, factory.CreateEnumerator(newRequest, dataProvider));
 
-                var sync = new SynchronizingEnumerator(enumerators);
-                return new FuturesChainUniverseDataCollectionAggregatorEnumerator(sync, request.Security.Symbol);
+                return new FuturesChainUniverseDataCollectionAggregatorEnumerator(enumerator, request.Security.Symbol);
             }
         }
 
-        private static IEnumerable<SubscriptionDataConfig> GetSubscriptionConfigurations(SubscriptionRequest request)
+        private static SubscriptionDataConfig GetSubscriptionConfiguration(SubscriptionRequest request)
         {
             var config = request.Configuration;
             var resolution = config.Resolution;
 
-            var configurations = new List<SubscriptionDataConfig>
-            {
-                // rewrite the primary to be fill forward
-                new SubscriptionDataConfig(config, resolution: resolution, fillForward: true)
-            };
-
-            return configurations;
+            // rewrite the primary to be fill forward
+            return new SubscriptionDataConfig(config, resolution: resolution, fillForward: true);
         }
     }
-}
+}

Engine/DataFeeds/Enumerators/Factories/SubscriptionDataReaderSubscriptionEnumeratorFactory.cs

@@ -35,7 +35,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
         private readonly bool _isLiveMode;
         private readonly IResultHandler _resultHandler;
         private readonly IFactorFileProvider _factorFileProvider;
-        private readonly ZipDataCacheProvider _zipDataCacheProvider;
+        private readonly IDataCacheProvider _dataCacheProvider;
         private readonly ConcurrentDictionary<Symbol, string> _numericalPrecisionLimitedWarnings;
         private readonly int _numericalPrecisionLimitedWarningsMaxCount = 10;
         private readonly ConcurrentDictionary<Symbol, string> _startDateLimitedWarnings;
@@ -50,14 +50,14 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
         /// <param name="resultHandler">The result handler for the algorithm</param>
         /// <param name="mapFileProvider">The map file provider</param>
         /// <param name="factorFileProvider">The factor file provider</param>
-        /// <param name="dataProvider">Provider used to get data when it is not present on disk</param>
+        /// <param name="cacheProvider">Provider used to get data when it is not present on disk</param>
         /// <param name="tradableDaysProvider">Function used to provide the tradable dates to be enumerator.
         /// Specify null to default to <see cref="SubscriptionRequest.TradableDays"/></param>
         /// <param name="enablePriceScaling">Applies price factor</param>
         public SubscriptionDataReaderSubscriptionEnumeratorFactory(IResultHandler resultHandler,
             IMapFileProvider mapFileProvider,
             IFactorFileProvider factorFileProvider,
-            IDataProvider dataProvider,
+            IDataCacheProvider cacheProvider,
             Func<SubscriptionRequest, IEnumerable<DateTime>> tradableDaysProvider = null,
             bool enablePriceScaling = true
             )
@@ -65,7 +65,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
             _resultHandler = resultHandler;
             _mapFileProvider = mapFileProvider;
             _factorFileProvider = factorFileProvider;
-            _zipDataCacheProvider = new ZipDataCacheProvider(dataProvider, isDataEphemeral: false);
+            _dataCacheProvider = cacheProvider;
             _numericalPrecisionLimitedWarnings = new ConcurrentDictionary<Symbol, string>();
             _startDateLimitedWarnings = new ConcurrentDictionary<Symbol, string>();
             _isLiveMode = false;
@@ -88,7 +88,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
                 _factorFileProvider,
                 _tradableDaysProvider(request),
                 _isLiveMode,
-                 _zipDataCacheProvider,
+                _dataCacheProvider,
                 dataProvider
                 );
 
@@ -159,8 +159,6 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
 
                 _resultHandler.DebugMessage(message);
             }
-
-            _zipDataCacheProvider?.DisposeSafely();
         }
     }
 }

Engine/DataFeeds/Enumerators/FilterEnumerator.cs

@@ -0,0 +1,84 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
+{
+    /// <summary>
+    /// Enumerator that allow applying a filtering function
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    public class FilterEnumerator<T> : IEnumerator<T>
+    {
+        private readonly IEnumerator<T> _enumerator;
+        private readonly Func<T, bool> _filter;
+
+        /// <summary>
+        /// Creates a new instance
+        /// </summary>
+        /// <param name="enumerator">The underlying enumerator to filter on</param>
+        /// <param name="filter">The filter to apply</param>
+        public FilterEnumerator(IEnumerator<T> enumerator, Func<T, bool> filter)
+        {
+            _enumerator = enumerator;
+            _filter = filter;
+        }
+
+        #region Implementation of IDisposable
+
+        public void Dispose()
+        {
+            _enumerator.Dispose();
+        }
+
+        #endregion
+
+        #region Implementation of IEnumerator
+
+        public bool MoveNext()
+        {
+            // run the enumerator until it passes the specified filter
+            while (_enumerator.MoveNext())
+            {
+                if (_filter(_enumerator.Current))
+                {
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        public void Reset()
+        {
+            _enumerator.Reset();
+        }
+
+        public T Current
+        {
+            get { return _enumerator.Current; }
+        }
+
+        object IEnumerator.Current
+        {
+            get { return _enumerator.Current; }
+        }
+
+        #endregion
+    }
+}

Engine/DataFeeds/Enumerators/LiveSubscriptionEnumerator.cs

@@ -29,13 +29,20 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
     /// </summary>
     public class LiveSubscriptionEnumerator : IEnumerator<BaseData>
     {
+        private BaseData _current;
         private readonly Symbol _requestedSymbol;
         private SubscriptionDataConfig _currentConfig;
         private IEnumerator<BaseData> _previousEnumerator;
         private IEnumerator<BaseData> _underlyingEnumerator;
 
-        public BaseData Current => _underlyingEnumerator.Current;
+        /// <summary>
+        /// The current data object instance
+        /// </summary>
+        public BaseData Current => _current;
 
+        /// <summary>
+        /// The current data object instance
+        /// </summary>
         object IEnumerator.Current => Current;
 
         /// <summary>
@@ -73,9 +80,14 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
             }
 
             var result = _underlyingEnumerator.MoveNext();
-            if (Current != null)
+            _current = _underlyingEnumerator.Current;
+
+            if (_current != null && _current.Symbol != _requestedSymbol)
             {
-                Current.Symbol = _requestedSymbol;
+                // if we've done some mapping at this layer let's clone the underlying and set the requested symbol,
+                // don't trust the IDQH implementations for data uniqueness, since the configuration could be shared
+                _current = _current.Clone();
+                _current.Symbol = _requestedSymbol;
             }
 
             return result;

Engine/DataFeeds/FileSystemDataFeed.cs

@@ -18,7 +18,6 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using QuantConnect.Data;
-using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Market;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Interfaces;
@@ -44,6 +43,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         private IMapFileProvider _mapFileProvider;
         private IFactorFileProvider _factorFileProvider;
         private IDataProvider _dataProvider;
+        private IDataCacheProvider _cacheProvider;
         private SubscriptionCollection _subscriptions;
         private CancellationTokenSource _cancellationTokenSource = new CancellationTokenSource();
         private SubscriptionDataReaderSubscriptionEnumeratorFactory _subscriptionFactory;
@@ -74,11 +74,12 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             _timeProvider = dataFeedTimeProvider.FrontierTimeProvider;
             _subscriptions = subscriptionManager.DataFeedSubscriptions;
             _cancellationTokenSource = new CancellationTokenSource();
+            _cacheProvider = new ZipDataCacheProvider(dataProvider, isDataEphemeral: false);
             _subscriptionFactory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(
                 _resultHandler,
                 _mapFileProvider,
                 _factorFileProvider,
-                _dataProvider,
+                _cacheProvider,
                 enablePriceScaling: false);
 
             IsActive = true;
@@ -150,13 +151,19 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             {
                 factory = new OptionChainUniverseSubscriptionEnumeratorFactory((req) =>
                 {
-                    var underlyingFactory = new BaseDataSubscriptionEnumeratorFactory(false, _mapFileProvider, _factorFileProvider);
+                    if (!req.Configuration.SecurityType.IsOption())
+                    {
+                        var enumerator = _subscriptionFactory.CreateEnumerator(req, _dataProvider);
+                        enumerator = new FilterEnumerator<BaseData>(enumerator, data => data.DataType != MarketDataType.Auxiliary);
+                        return ConfigureEnumerator(req, true, enumerator);
+                    }
+                    var underlyingFactory = new BaseDataSubscriptionEnumeratorFactory(false, _mapFileProvider, _cacheProvider);
                     return ConfigureEnumerator(req, true, underlyingFactory.CreateEnumerator(req, _dataProvider));
                 });
             }
             if (request.Universe is FuturesChainUniverse)
             {
-                factory = new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e));
+                factory = new FuturesChainUniverseSubscriptionEnumeratorFactory((req, e) => ConfigureEnumerator(req, true, e), _cacheProvider);
             }
 
             // define our data enumerator

Engine/DataFeeds/SingleEntryDataCacheProvider.cs

@@ -14,11 +14,14 @@
  *
 */
 
+using System;
 using System.IO;
 using Ionic.Zip;
-using QuantConnect.Interfaces;
-using QuantConnect.Logging;
+using System.Linq;
 using QuantConnect.Util;
+using QuantConnect.Logging;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
 
 namespace QuantConnect.Lean.Engine.DataFeeds
 {
@@ -89,6 +92,22 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             //
         }
 
+        /// <summary>
+        /// Returns a list of zip entries in a provided zip file
+        /// </summary>
+        public List<string> GetZipEntries(string zipFile)
+        {
+            var stream = _dataProvider.Fetch(zipFile);
+            if (stream == null)
+            {
+                throw new ArgumentException($"Failed to create source stream {zipFile}");
+            }
+            var entryNames = Compression.GetZipEntryFileNames(stream).ToList();
+            stream.DisposeSafely();
+
+            return entryNames;
+        }
+
         /// <summary>
         /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
         /// </summary>

Engine/DataFeeds/SubscriptionDataSourceReader.cs

@@ -55,7 +55,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                     break;
 
                 case FileFormat.ZipEntryName:
-                    reader = new ZipEntryNameSubscriptionDataSourceReader(dataProvider, config, date, isLiveMode);
+                    reader = new ZipEntryNameSubscriptionDataSourceReader(dataCacheProvider, config, date, isLiveMode);
                     break;
 
                 case FileFormat.Index:

Engine/DataFeeds/TimeSliceFactory.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -450,19 +450,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             OptionContract contract;
             if (!chain.Contracts.TryGetValue(baseData.Symbol, out contract))
             {
-                var underlyingSymbol = baseData.Symbol.Underlying;
-                contract = new OptionContract(baseData.Symbol, underlyingSymbol)
-                {
-                    Time = baseData.EndTime,
-                    LastPrice = security.Close,
-                    Volume = (long)security.Volume,
-                    BidPrice = security.BidPrice,
-                    BidSize = (long)security.BidSize,
-                    AskPrice = security.AskPrice,
-                    AskSize = (long)security.AskSize,
-                    OpenInterest = security.OpenInterest,
-                    UnderlyingLastPrice = chain.Underlying.Price
-                };
+                contract = OptionContract.Create(baseData, security, chain.Underlying.Price);
 
                 chain.Contracts[baseData.Symbol] = contract;
 

Engine/DataFeeds/ZipDataCacheProvider.cs

@@ -15,15 +15,15 @@
 
 using System;
 using System.IO;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
-using QuantConnect.Logging;
-using System.Linq;
-using System.Threading;
 using Ionic.Zip;
 using Ionic.Zlib;
-using QuantConnect.Interfaces;
+using System.Linq;
+using System.Threading;
 using QuantConnect.Util;
+using QuantConnect.Logging;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
 
 namespace QuantConnect.Lean.Engine.DataFeeds
 {
@@ -89,6 +89,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                                 }
                                 else
                                 {
+                                    existingZip.Refresh();
                                     stream = CreateEntryStream(existingZip, entryName, filename);
                                 }
                             }
@@ -173,6 +174,26 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             }
         }
 
+        /// <summary>
+        /// Returns a list of zip entries in a provided zip file
+        /// </summary>
+        public List<string> GetZipEntries(string zipFile)
+        {
+            if (!_zipFileCache.TryGetValue(zipFile, out var cachedZip))
+            {
+                if (!Cache(zipFile, out cachedZip))
+                {
+                    throw new ArgumentException($"Failed to get zip entries from {zipFile}");
+                }
+            }
+
+            lock (cachedZip)
+            {
+                cachedZip.Refresh();
+                return cachedZip.EntryCache.Keys.ToList();
+            }
+        }
+
         /// <summary>
         /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
         /// </summary>
@@ -461,8 +482,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             public void WriteEntry(string entryName, byte[] content)
             {
                 Interlocked.Increment(ref _modified);
-                // we refresh our cache time when modified
-                _dateCached = new ReferenceWrapper<DateTime>(DateTime.UtcNow);
+                Refresh();
 
                 // If the entry already exists remove it 
                 if (_zipFile.ContainsEntry(entryName))
@@ -476,6 +496,14 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                 EntryCache.Add(entryName, new ZipEntryCache { Entry = newEntry, Modified = true });
             }
 
+            /// <summary>
+            /// We refresh our cache time when used to avoid it being clean up
+            /// </summary>
+            public void Refresh()
+            {
+                _dateCached = new ReferenceWrapper<DateTime>(DateTime.UtcNow);
+            }
+
             /// <summary>
             /// Dispose of the ZipFile
             /// </summary>
@@ -497,7 +525,6 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                     _zipFile.Save(tempFileName);
                 }
 
-                EntryCache.Clear();
                 _zipFile?.DisposeSafely();
                 _dataStream?.DisposeSafely();
 

Engine/DataFeeds/ZipEntryNameSubscriptionDataSourceReader.cs

@@ -14,10 +14,7 @@
 */
 
 using System;
-using Ionic.Zip;
-using System.Linq;
 using QuantConnect.Data;
-using QuantConnect.Util;
 using QuantConnect.Interfaces;
 using System.Collections.Generic;
 
@@ -28,7 +25,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
     /// </summary>
     public class ZipEntryNameSubscriptionDataSourceReader : ISubscriptionDataSourceReader
     {
-        private readonly IDataProvider _dataProvider;
+        private readonly IDataCacheProvider _dataProvider;
         private readonly SubscriptionDataConfig _config;
         private readonly DateTime _date;
         private readonly bool _isLiveMode;
@@ -47,13 +44,13 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         /// <param name="config">The subscription's configuration</param>
         /// <param name="date">The date this factory was produced to read data for</param>
         /// <param name="isLiveMode">True if we're in live mode, false for backtesting</param>
-        public ZipEntryNameSubscriptionDataSourceReader(IDataProvider dataProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
+        public ZipEntryNameSubscriptionDataSourceReader(IDataCacheProvider dataProvider, SubscriptionDataConfig config, DateTime date, bool isLiveMode)
         {
             _date = date;
             _config = config;
             _isLiveMode = isLiveMode;
             _dataProvider = dataProvider;
-            _factory = _factory = config.GetBaseDataInstance();
+            _factory = config.GetBaseDataInstance();
         }
 
         /// <summary>
@@ -66,16 +63,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             List<string> entryNames;
             try
             {
-                var stream = _dataProvider.Fetch(source.Source);
-                if (stream == null)
-                {
-                    OnInvalidSource(source, new ArgumentException($"Failed to create source stream {source.Source}"));
-                    yield break;
-                }
-                entryNames = Compression.GetZipEntryFileNames(stream).ToList();
-                stream.DisposeSafely();
+                entryNames = _dataProvider.GetZipEntries(source.Source);
             }
-            catch (ZipException err)
+            catch (Exception err)
             {
                 OnInvalidSource(source, err);
                 yield break;

Engine/HistoricalData/SubscriptionDataReaderHistoryProvider.cs

@@ -15,11 +15,9 @@
 */
 
 using System;
-using System.Collections;
 using System.Collections.Generic;
 using NodaTime;
 using QuantConnect.Data;
-using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Market;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Interfaces;
@@ -192,58 +190,5 @@ namespace QuantConnect.Lean.Engine.HistoricalData
         {
             return null;
         }
-
-        private class FilterEnumerator<T> : IEnumerator<T>
-        {
-            private readonly IEnumerator<T> _enumerator;
-            private readonly Func<T, bool> _filter;
-
-            public FilterEnumerator(IEnumerator<T> enumerator, Func<T, bool> filter)
-            {
-                _enumerator = enumerator;
-                _filter = filter;
-            }
-
-            #region Implementation of IDisposable
-
-            public void Dispose()
-            {
-                _enumerator.Dispose();
-            }
-
-            #endregion
-
-            #region Implementation of IEnumerator
-
-            public bool MoveNext()
-            {
-                // run the enumerator until it passes the specified filter
-                while (_enumerator.MoveNext())
-                {
-                    if (_filter(_enumerator.Current))
-                    {
-                        return true;
-                    }
-                }
-                return false;
-            }
-
-            public void Reset()
-            {
-                _enumerator.Reset();
-            }
-
-            public T Current
-            {
-                get { return _enumerator.Current; }
-            }
-
-            object IEnumerator.Current
-            {
-                get { return _enumerator.Current; }
-            }
-
-            #endregion
-        }
     }
 }

Engine/QuantConnect.Lean.Engine.csproj

@@ -43,7 +43,7 @@
     <Message Text="SelectedOptimization $(SelectedOptimization)" Importance="high" />
   </Target>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="fasterflect" Version="3.0.0" />
     <PackageReference Include="FSharp.Core" Version="4.5.2" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />

Indicators/QuantConnect.Indicators.csproj

@@ -32,7 +32,7 @@
     <Message Text="SelectedOptimization $(SelectedOptimization)" Importance="high" />
   </Target>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>

Report/QuantConnect.Report.csproj

@@ -42,7 +42,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Deedle" Version="2.1.0" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">

Research/QuantConnect.Research.csproj

@@ -32,7 +32,7 @@
     <Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

Tests/Algorithm/AlgorithmAddSecurityTests.cs

@@ -17,15 +17,18 @@
 using System;
 using NUnit.Framework;
 using QuantConnect.Algorithm;
-using QuantConnect.Data.Custom.AlphaStreams;
-using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Securities;
+using System.Collections.Generic;
 using QuantConnect.Securities.Cfd;
 using QuantConnect.Securities.Crypto;
 using QuantConnect.Securities.Equity;
 using QuantConnect.Securities.Forex;
 using QuantConnect.Securities.Future;
 using QuantConnect.Securities.Option;
+using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Tests.Engine.DataFeeds;
+using QuantConnect.Securities.IndexOption;
+using QuantConnect.Data.Custom.AlphaStreams;
 using Index = QuantConnect.Securities.Index.Index;
 
 namespace QuantConnect.Tests.Algorithm
@@ -80,6 +83,9 @@ namespace QuantConnect.Tests.Algorithm
                     case SecurityType.Index:
                         var index = (Index)security;
                         break;
+                    case SecurityType.IndexOption:
+                        var indexOption = (IndexOption)security;
+                        break;
                     case SecurityType.Crypto:
                         var crypto = (Crypto)security;
                         break;
@@ -102,7 +108,7 @@ namespace QuantConnect.Tests.Algorithm
         {
             get
             {
-                return new[]
+                var result = new List<TestCaseData>()
                 {
                     new TestCaseData(Symbols.SPY, null),
                     new TestCaseData(Symbols.EURUSD, null),
@@ -117,6 +123,21 @@ namespace QuantConnect.Tests.Algorithm
                     new TestCaseData(Symbol.Create("CustomData", SecurityType.Base, Market.Binance), null),
                     new TestCaseData(Symbol.Create("CustomData2", SecurityType.Base, Market.COMEX), null)
                 };
+
+                foreach (var market in Market.SupportedMarkets())
+                {
+                    foreach (var kvp in SymbolPropertiesDatabase.FromDataFolder().GetSymbolPropertiesList(market))
+                    {
+                        var securityDatabaseKey = kvp.Key;
+                        if (securityDatabaseKey.SecurityType != SecurityType.FutureOption)
+                        {
+                            result.Add(new TestCaseData(Symbol.Create(securityDatabaseKey.Symbol, securityDatabaseKey.SecurityType,
+                                securityDatabaseKey.Market), null));
+                        }
+                    }
+                }
+
+                return result.ToArray();
             }
         }
     }

Tests/Algorithm/AlgorithmWarmupTests.cs

@@ -26,6 +26,7 @@ using QuantConnect.Packets;
 using QuantConnect.Indicators;
 using QuantConnect.Tests.Engine.DataFeeds;
 using Python.Runtime;
+using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Lean.Engine.HistoricalData;
 using QuantConnect.Util;
@@ -109,6 +110,30 @@ namespace QuantConnect.Tests.Algorithm
             Assert.GreaterOrEqual(_algorithm.WarmUpDataCount, estimateExpectedDataCount);
         }
 
+        [Test]
+        public void WarmUpInternalSubscriptionsHistoryRequest()
+        {
+            var algo = new AlgorithmStub(new MockDataFeed())
+            {
+                HistoryProvider = new SubscriptionDataReaderHistoryProvider()
+            };
+
+            algo.SetStartDate(2013, 10, 08);
+            algo.AddCfd("DE30EUR", Resolution.Second, Market.Oanda);
+            algo.SetWarmup(10);
+            algo.PostInitialize();
+            algo.OnEndOfTimeStep();
+            algo.DataManager.UniverseSelection.EnsureCurrencyDataFeeds(SecurityChanges.None);
+
+            var result = algo.GetWarmupHistoryRequests();
+
+            foreach (var historyRequest in result)
+            {
+                Assert.AreEqual(Resolution.Second, historyRequest.Resolution);
+                Assert.AreEqual(TimeSpan.FromSeconds(10), historyRequest.EndTimeUtc - historyRequest.StartTimeUtc);
+            }
+        }
+
         [Test]
         public void WarmUpPythonIndicatorProperly()
         {

Tests/Common/Orders/Fees/InteractiveBrokersFeeModelTests.cs

@@ -92,6 +92,39 @@ namespace QuantConnect.Tests.Common.Orders.Fees
             Assert.AreEqual(1000 * 1.85m, fee.Value.Amount);
         }
 
+        [TestCase(false)]
+        [TestCase(true)]
+        public void HongKongFutureFee(bool canonical)
+        {
+            var symbol = Symbols.CreateFutureSymbol(Futures.Indices.HangSeng, SecurityIdentifier.DefaultDate);
+            if (!canonical)
+            {
+                symbol = Symbols.CreateFutureSymbol(Futures.Indices.HangSeng,
+                    FuturesExpiryFunctions.FuturesExpiryFunction(symbol)(new DateTime(2021, 12, 1)));
+            }
+            var entry = MarketHoursDatabase.FromDataFolder().GetEntry(symbol.ID.Market, symbol, symbol.SecurityType);
+            var properties = SymbolPropertiesDatabase.FromDataFolder()
+                .GetSymbolProperties(symbol.ID.Market, symbol, symbol.SecurityType, null);
+            var security = new Future(symbol, entry.ExchangeHours,
+                new Cash(properties.QuoteCurrency, 0, 0),
+                properties,
+                ErrorCurrencyConverter.Instance,
+                RegisteredSecurityDataTypesProvider.Null,
+                new SecurityCache()
+            );
+            security.SetMarketPrice(new Tick(DateTime.UtcNow, security.Symbol, 100, 100));
+
+            var fee = _feeModel.GetOrderFee(
+                new OrderFeeParameters(
+                    security,
+                    new MarketOrder(security.Symbol, 1000, DateTime.UtcNow)
+                )
+            );
+
+            Assert.AreEqual(Currencies.HKD, fee.Value.Currency);
+            Assert.AreEqual(1000 * 40m, fee.Value.Amount);
+        }
+
         [Test]
         public void USAOptionFee()
         {

Tests/Common/Packets/BreakpointTests.cs

@@ -1,46 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-
-using System.Collections.Generic;
-using Newtonsoft.Json;
-using NUnit.Framework;
-using QuantConnect.Packets;
-
-namespace QuantConnect.Tests.Common.Packets
-{
-    [TestFixture, Parallelizable(ParallelScope.All)]
-    public class BreakpointTests
-    {
-        [Test]
-        public void SurvivesSerializationRoundTrip()
-        {
-            var breakpoints = new List<Breakpoint>
-            {
-                new Breakpoint{ FileName = "MichelAngelo", LineNumber = 1475},
-                new Breakpoint{ FileName = "LeonardoDaVinci", LineNumber = 1452}
-            };
-
-            var serialized = JsonConvert.SerializeObject(breakpoints);
-            var deserialized = JsonConvert.DeserializeObject<List<Breakpoint>>(serialized);
-
-            Assert.AreEqual(deserialized.Count, 2);
-            Assert.AreEqual(deserialized[0].FileName, "MichelAngelo");
-            Assert.AreEqual(deserialized[0].LineNumber, 1475);
-            Assert.AreEqual(deserialized[1].FileName, "LeonardoDaVinci");
-            Assert.AreEqual(deserialized[1].LineNumber, 1452);
-        }
-    }
-}

Tests/Common/Securities/OptionPriceModelTests.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -13,15 +13,19 @@
  * limitations under the License.
 */
 
-using System;
-using System.Linq;
 using NUnit.Framework;
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
 using QuantConnect.Securities;
 using QuantConnect.Securities.Equity;
 using QuantConnect.Securities.Option;
+using System;
 using System.Collections.Generic;
+using System.Linq;
+using Moq;
+using QLNet;
+using Cash = QuantConnect.Securities.Cash;
+using Option = QuantConnect.Securities.Option.Option;
 
 namespace QuantConnect.Tests.Common
 {
@@ -195,7 +199,7 @@ namespace QuantConnect.Tests.Common
             var rightPart = greeks.Theta + riskFreeRate * underlyingPrice * greeks.Delta + 0.5m * underlyingVol * underlyingVol * underlyingPrice * underlyingPrice * greeks.Gamma;
             var leftPart = riskFreeRate * callPrice;
 
-            Assert.GreaterOrEqual(Math.Round(leftPart, 4), Math.Round(rightPart,4));
+            Assert.GreaterOrEqual(Math.Round(leftPart, 4), Math.Round(rightPart, 4));
         }
 
         [Test]
@@ -252,6 +256,20 @@ namespace QuantConnect.Tests.Common
             Assert.Greater(callPrice1, callPrice2);
         }
 
+        [Test]
+        [TestCase("BaroneAdesiWhaleyApproximationEngine")]
+        [TestCase("QLNet.BaroneAdesiWhaleyApproximationEngine")]
+        public void CreatesOptionPriceModelByName(string priceEngineName)
+        {
+            IOptionPriceModel priceModel = null;
+            Assert.DoesNotThrow(() =>
+            {
+                priceModel = OptionPriceModels.Create(priceEngineName, 0.01m);
+            });
+
+            Assert.NotNull(priceModel);
+            Assert.IsInstanceOf<QLOptionPriceModel>(priceModel);
+        }
 
         [Test]
         public void GreekApproximationTest()
@@ -315,6 +333,83 @@ namespace QuantConnect.Tests.Common
             Assert.Greater(greeks.Vega, 0);
         }
 
+        [Test]
+        [TestCase(true)]
+        [TestCase(false)]
+        public void HasBeenWarmedUp(bool warmUp)
+        {
+            var volatilityModel = new Mock<IQLUnderlyingVolatilityEstimator>();
+            volatilityModel.SetupGet(s => s.IsReady).Returns(warmUp);
+            var priceModel = new QLOptionPriceModel(
+                process => new IntegralEngine(process),
+                volatilityModel.Object,
+                null,
+                null);
+
+            Assert.AreEqual(warmUp, priceModel.VolatilityEstimatorWarmedUp);
+        }
+
+        [Test]
+        public void ReturnsNoneIfNotWarmedUp()
+        {
+            const decimal underlyingPrice = 200m;
+            const decimal underlyingVol = 0.15m;
+            const decimal riskFreeRate = 0.01m;
+            var tz = TimeZones.NewYork;
+            var evaluationDate = new DateTime(2015, 2, 19);
+            var SPY_C_192_Feb19_2016E = Symbol.CreateOption("SPY", Market.USA, OptionStyle.European, OptionRight.Call, 192m, new DateTime(2016, 02, 19));
+            var SPY_P_192_Feb19_2016E = Symbol.CreateOption("SPY", Market.USA, OptionStyle.European, OptionRight.Put, 192m, new DateTime(2016, 02, 19));
+
+            // setting up underlying
+            var equity = new Equity(
+                SecurityExchangeHours.AlwaysOpen(tz),
+                new SubscriptionDataConfig(typeof(TradeBar), Symbols.SPY, Resolution.Minute, tz, tz, true, false, false),
+                new Cash(Currencies.USD, 0, 1m),
+                SymbolProperties.GetDefault(Currencies.USD),
+                ErrorCurrencyConverter.Instance,
+                RegisteredSecurityDataTypesProvider.Null
+            );
+            equity.SetMarketPrice(new Tick { Value = underlyingPrice });
+            equity.VolatilityModel = new DummyVolatilityModel(underlyingVol);
+
+            // setting up European style call option
+            var contractCall = new OptionContract(SPY_C_192_Feb19_2016E, Symbols.SPY) { Time = evaluationDate };
+            var optionCall = new Option(
+                SecurityExchangeHours.AlwaysOpen(tz),
+                new SubscriptionDataConfig(typeof(TradeBar), SPY_C_192_Feb19_2016E, Resolution.Minute, tz, tz, true, false, false),
+                new Cash(Currencies.USD, 0, 1m),
+                new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
+                ErrorCurrencyConverter.Instance,
+                RegisteredSecurityDataTypesProvider.Null
+            );
+            optionCall.Underlying = equity;
+
+            // setting up European style put option
+            var contractPut = new OptionContract(SPY_P_192_Feb19_2016E, Symbols.SPY) { Time = evaluationDate };
+            var optionPut = new Option(
+                SecurityExchangeHours.AlwaysOpen(tz),
+                new SubscriptionDataConfig(typeof(TradeBar), SPY_P_192_Feb19_2016E, Resolution.Minute, tz, tz, true, false, false),
+                new Cash(Currencies.USD, 0, 1m),
+                new OptionSymbolProperties(SymbolProperties.GetDefault(Currencies.USD)),
+                ErrorCurrencyConverter.Instance,
+                RegisteredSecurityDataTypesProvider.Null
+            );
+            optionPut.Underlying = equity;
+
+            // running evaluation
+            var volatilityModel = new Mock<IQLUnderlyingVolatilityEstimator>();
+            volatilityModel.SetupGet(s => s.IsReady).Returns(false);
+            var priceModel = new QLOptionPriceModel(process => new AnalyticEuropeanEngine(process),
+                volatilityModel.Object,
+                null,
+                null);
+            var resultsCall = priceModel.Evaluate(optionCall, null, contractCall);
+            var resultsPut = priceModel.Evaluate(optionPut, null, contractPut);
+
+            Assert.AreEqual(OptionPriceModelResult.None, resultsCall);
+            Assert.AreEqual(OptionPriceModelResult.None, resultsCall);
+        }
+
         /// <summary>
         /// Dummy implementation of volatility model (for tests only)
         /// </summary>

Tests/Engine/DataFeeds/DataManagerStub.cs

@@ -93,7 +93,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds
 
         public DataManagerStub(IDataFeed dataFeed, IAlgorithm algorithm, ITimeKeeper timeKeeper, MarketHoursDatabase marketHoursDatabase, SecurityService securityService, DataPermissionManager dataPermissionManager, bool liveMode = false)
             : base(dataFeed,
-                new UniverseSelection(algorithm, securityService, dataPermissionManager, new DefaultDataProvider()),
+                new UniverseSelection(algorithm, securityService, dataPermissionManager, TestGlobals.DataProvider),
                 algorithm,
                 timeKeeper,
                 marketHoursDatabase,

Tests/Engine/DataFeeds/Enumerators/Factories/BaseDataSubscriptionEnumeratorFactoryTests.cs

@@ -47,7 +47,8 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
             );
 
             var fileProvider = TestGlobals.DataProvider;
-            var factory = new BaseDataSubscriptionEnumeratorFactory(false, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider);
+            using var cache = new ZipDataCacheProvider(fileProvider);
+            var factory = new BaseDataSubscriptionEnumeratorFactory(false, TestGlobals.MapFileProvider, cache);
 
             GC.Collect();
             var ramUsageBeforeLoop = OS.TotalPhysicalMemoryUsed;

Tests/Engine/DataFeeds/Enumerators/Factories/OptionChainUniverseSubscriptionEnumeratorFactoryTests.cs

@@ -71,7 +71,8 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators.Factories
             );
 
             var dataProvider = TestGlobals.DataProvider;
-            var enumeratorFactory = new BaseDataSubscriptionEnumeratorFactory(false, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider);
+            using var cache = new ZipDataCacheProvider(dataProvider);
+            var enumeratorFactory = new BaseDataSubscriptionEnumeratorFactory(false, TestGlobals.MapFileProvider, cache);
             var fillForwardResolution = Ref.CreateReadOnly(() => Resolution.Minute.ToTimeSpan());
             Func<SubscriptionRequest, IEnumerator<BaseData>> underlyingEnumeratorFunc = (req) =>
                 {

Tests/Engine/DataFeeds/Enumerators/LiveSubscriptionEnumeratorTests.cs

@@ -15,6 +15,7 @@
 
 using System;
 using NodaTime;
+using System.Linq;
 using NUnit.Framework;
 using QuantConnect.Data;
 using QuantConnect.Packets;
@@ -35,12 +36,12 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
             var canonical = Symbols.Fut_SPY_Feb19_2016.Canonical;
             var dataQueue = new TestDataQueueHandler
             {
-                DataPerSymbol = new Dictionary<Symbol, IEnumerator<BaseData>>
+                DataPerSymbol = new Dictionary<Symbol, List<BaseData>>
                 {
                     { Symbols.Fut_SPY_Feb19_2016,
-                        new List<BaseData>{ new Tick(Time.BeginningOfTime, Symbols.Fut_SPY_Feb19_2016, 1, 1)}.GetEnumerator() },
+                        new List<BaseData>{ new Tick(Time.BeginningOfTime, Symbols.Fut_SPY_Feb19_2016, 1, 1)} },
                     { Symbols.Fut_SPY_Mar19_2016,
-                        new List<BaseData>{ new Tick(Time.BeginningOfTime, Symbols.Fut_SPY_Mar19_2016, 2, 2)}.GetEnumerator() },
+                        new List<BaseData>{ new Tick(Time.BeginningOfTime, Symbols.Fut_SPY_Mar19_2016, 2, 2)} },
                 }
             };
             var config = new SubscriptionDataConfig(typeof(Tick), canonical, Resolution.Tick,
@@ -67,6 +68,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
             Assert.IsTrue(data.MoveNext());
             Assert.AreEqual(2, (data.Current as Tick).AskPrice);
             Assert.AreEqual(canonical, (data.Current as Tick).Symbol);
+            Assert.AreNotEqual(canonical, dataQueue.DataPerSymbol[Symbols.Fut_SPY_Mar19_2016].Single().Symbol);
 
             Assert.IsFalse(data.MoveNext());
             Assert.IsNull(data.Current);
@@ -81,13 +83,13 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
         {
             public bool IsConnected => true;
 
-            public Dictionary<Symbol, IEnumerator<BaseData>> DataPerSymbol;
+            public Dictionary<Symbol, List<BaseData>> DataPerSymbol;
 
             public IEnumerator<BaseData> Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler)
             {
-                if (DataPerSymbol.TryGetValue(dataConfig.Symbol, out var enumerator))
+                if (DataPerSymbol.TryGetValue(dataConfig.Symbol, out var baseDatas))
                 {
-                    return enumerator;
+                    return baseDatas.GetEnumerator();
                 }
                 throw new Exception($"Failed to find a data enumerator for symbol {dataConfig.Symbol}!");
             }

Tests/Engine/DataFeeds/FileSystemDataFeedTests.cs

@@ -95,7 +95,8 @@ namespace QuantConnect.Tests.Engine.DataFeeds
             algorithm.PostInitialize();
 
             var resultHandler = new BacktestingResultHandler();
-            var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, enablePriceScaling: false);
+            using var cache = new ZipDataCacheProvider(TestGlobals.DataProvider);
+            var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, cache, enablePriceScaling: false);
 
             var universe = algorithm.UniverseManager.Single().Value;
             var security = algorithm.Securities.Single().Value;
@@ -139,9 +140,9 @@ namespace QuantConnect.Tests.Engine.DataFeeds
             algorithm.Initialize();
             algorithm.PostInitialize();
 
-            var dataProvider = new DefaultDataProvider();
             var resultHandler = new TestResultHandler();
-            var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, TestGlobals.DataProvider, enablePriceScaling: false);
+            using var cache = new ZipDataCacheProvider(TestGlobals.DataProvider);
+            var factory = new SubscriptionDataReaderSubscriptionEnumeratorFactory(resultHandler, TestGlobals.MapFileProvider, TestGlobals.FactorFileProvider, cache, enablePriceScaling: false);
 
             var universe = algorithm.UniverseManager.Single().Value;
             var security = algorithm.AddEquity("AAA", Resolution.Daily);
@@ -149,7 +150,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds
             // start date is before the first date in the map file
             var subscriptionRequest = new SubscriptionRequest(false, universe, security, securityConfig, new DateTime(2001, 12, 1),
                 new DateTime(2016, 11, 1));
-            var enumerator = factory.CreateEnumerator(subscriptionRequest, dataProvider);
+            var enumerator = factory.CreateEnumerator(subscriptionRequest, TestGlobals.DataProvider);
             // should initialize the data source reader
             enumerator.MoveNext();
 

Tests/Engine/DataFeeds/IndexSubscriptionDataSourceReaderTests.cs

@@ -14,6 +14,7 @@
 */
 
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using NUnit.Framework;
@@ -131,6 +132,10 @@ namespace QuantConnect.Tests.Engine.DataFeeds
             public void Store(string key, byte[] data)
             {
             }
+            public List<string> GetZipEntries(string zipFile)
+            {
+                throw new NotImplementedException();
+            }
             public void Dispose()
             {
             }

Tests/Engine/DataFeeds/SubscriptionDataReaderTests.cs

@@ -15,6 +15,7 @@
 */
 
 using System;
+using System.Collections.Generic;
 using System.IO;
 using NUnit.Framework;
 using QuantConnect.Data;
@@ -79,7 +80,10 @@ namespace QuantConnect.Tests.Engine.DataFeeds
             public void Dispose()
             {
             }
-
+            public List<string> GetZipEntries(string zipFile)
+            {
+                throw new NotImplementedException();
+            }
             public bool IsDataEphemeral => true;
             public Stream Fetch(string key)
             {

Tests/Engine/DataFeeds/TextSubscriptionDataSourceReaderTests.cs

@@ -376,6 +376,10 @@ namespace QuantConnect.Tests.Engine.DataFeeds
             public string Data { set; get; }
             public bool IsDataEphemeral { set; get; }
 
+            public List<string> GetZipEntries(string zipFile)
+            {
+                throw new NotImplementedException();
+            }
             public Stream Fetch(string key)
             {
                 var stream = new MemoryStream();

Tests/Engine/DataFeeds/ZipEntryNameSubscriptionFactoryTests.cs

@@ -33,7 +33,8 @@ namespace QuantConnect.Tests.Engine.DataFeeds
             var source = Path.Combine("TestData", "20151224_quote_american.zip");
             var config = new SubscriptionDataConfig(typeof (ZipEntryName), Symbol.Create("XLRE", SecurityType.Option, Market.USA), Resolution.Tick,
                 TimeZones.NewYork, TimeZones.NewYork, false, false, false);
-            var factory = new ZipEntryNameSubscriptionDataSourceReader(TestGlobals.DataProvider, config, time, false);
+            using var cacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider);
+            var factory = new ZipEntryNameSubscriptionDataSourceReader(cacheProvider, config, time, false);
             var expected = new[]
             {
                 Symbol.CreateOption("XLRE", Market.USA, OptionStyle.American, OptionRight.Call, 21m, new DateTime(2016, 08, 19)),

Tests/Engine/DataProviders/ApiDataProviderTests.cs

@@ -80,7 +80,7 @@ namespace QuantConnect.Tests.Engine.DataProviders
         [TestCase("forex/oanda/minute/eurusd/20140501_quote.zip", 10, false)]       // Date based Forex minute data
         [TestCase("forex/oanda/second/eurusd/20140501_quote.zip", 10, false)]       // Date based Forex second data
         [TestCase("forex/oanda/tick/eurusd/20140501_quote.zip", 10, false)]         // Date based Forex tick data
-        // Future Cases ** All False because Unsupported
+        // Price Future Cases False because Unsupported
         [TestCase("future/cboe/margins/VX.csv", 0, false)]                           // Fresh Margins data
         [TestCase("future/cboe/margins/VX.csv", 10, false)]                          // Stale Margins data
         [TestCase("future/comex/minute/gc/20131007_openinterest.zip", 10, false)]   // Date based minute data
@@ -127,9 +127,12 @@ namespace QuantConnect.Tests.Engine.DataProviders
         [TestCase("option/usa/minute/aapl/20100603_quote_american.zip")]
         // Forex
         [TestCase("forex/oanda/minute/eurusd/20020516_quote.zip")]
-        // Futures          * False because unsupported
+        // Price Futures          * False because unsupported
         [TestCase("future/cbot/minute/zs/20090501_trade.zip", false)]
         [TestCase("future/sgx/margins/IN.csv", false)]
+        // Auxiliary Data Future Cases true
+        [TestCase("future/comex/map_files/map_files_20211225.zip", true)]
+        [TestCase("future/cme/factor_files/factor_files_20211225.zip", true)]
         // Future Options   * False because unsupported
         [TestCase("futureoption/comex/minute/og/20120227/20120105_quote_american.zip", false)]
         // Index            * False because unsupported

Tests/QuantConnect.Tests.csproj

@@ -33,7 +33,7 @@
   </PropertyGroup>
   <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Accord" Version="3.6.0" />
     <PackageReference Include="Accord.Math" Version="3.6.0" />
     <PackageReference Include="Common.Logging" Version="3.4.1" />

Tests/ToolBox/LeanDataReaderTests.cs

@@ -179,7 +179,8 @@ namespace QuantConnect.Tests.ToolBox
             //load future chain first
             var config = new SubscriptionDataConfig(typeof(ZipEntryName), baseFuture, res,
                                                     TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, tickType);
-            var factory = new ZipEntryNameSubscriptionDataSourceReader(TestGlobals.DataProvider, config, date, false);
+            using var cacheProvider = new ZipDataCacheProvider(TestGlobals.DataProvider);
+            var factory = new ZipEntryNameSubscriptionDataSourceReader(cacheProvider, config, date, false);
 
             var result = factory.Read(new SubscriptionDataSource(filePath, SubscriptionTransportMedium.LocalFile, FileFormat.ZipEntryName))
                           .Select(s => s.Symbol).ToList();

Tests/ToolBox/RandomDataGenerator/BaseSymbolGeneratorTests.cs

@@ -0,0 +1,85 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using Moq;
+using NUnit.Framework;
+using QuantConnect.Securities;
+using QuantConnect.ToolBox.RandomDataGenerator;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+
+namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
+{
+    [TestFixture]
+    public class BaseSymbolGeneratorTests
+    {
+        [Test]
+        [TestCase(2, 5)]
+        [TestCase(3, 3)]
+        [TestCase(1, 4)]
+        public void NextUpperCaseString_CreatesString_WithinSpecifiedMinMaxLength(int min, int max)
+        {
+            var symbolGenerator = new Mock<BaseSymbolGenerator>(Mock.Of<RandomDataGeneratorSettings>(), new RandomValueGenerator()).Object;
+            var str = symbolGenerator.NextUpperCaseString(min, max);
+            Assert.LessOrEqual(min, str.Length);
+            Assert.GreaterOrEqual(max, str.Length);
+        }
+
+        [Test]
+        public void NextUpperCaseString_CreatesUpperCaseString()
+        {
+            var symbolGenerator = new Mock<BaseSymbolGenerator>(Mock.Of<RandomDataGeneratorSettings>(), new RandomValueGenerator()).Object;
+            var str = symbolGenerator.NextUpperCaseString(10, 10);
+            Assert.IsTrue(str.All(char.IsUpper));
+        }
+
+        [Test]
+        [TestCase(SecurityType.Option)]
+        [TestCase(SecurityType.Future)]
+        public void ThrowsArgumentException_ForDerivativeSymbols(SecurityType securityType)
+        {
+            var symbolGenerator = new Mock<BaseSymbolGenerator>(Mock.Of<RandomDataGeneratorSettings>(), new RandomValueGenerator()).Object;
+            Assert.Throws<ArgumentException>(() =>
+                symbolGenerator.NextSymbol(securityType, Market.USA)
+            );
+        }
+
+        [Test]
+        public void ThrowIsSettingsAreNull()
+        {
+            Assert.Throws<ArgumentNullException>(() =>
+            {
+                BaseSymbolGenerator.Create(null, Mock.Of<IRandomValueGenerator>());
+            });
+        }
+
+        [Test]
+        public void ThrowIsRundomValueGeneratorIsNull()
+        {
+            Assert.Throws<ArgumentNullException>(() =>
+            {
+                BaseSymbolGenerator.Create(new RandomDataGeneratorSettings(), null);
+            });
+        }
+
+        internal static IEnumerable<Symbol> GenerateAsset(BaseSymbolGenerator instance)
+        {
+            var generateAsset = typeof(BaseSymbolGenerator).GetMethod("GenerateAsset", BindingFlags.NonPublic | BindingFlags.Instance);
+            return (IEnumerable<Symbol>)generateAsset.Invoke(instance, new [] { (object)null });
+        }
+    }
+}

Tests/ToolBox/RandomDataGenerator/DefaultSymbolGeneratorTests.cs

@@ -0,0 +1,172 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using Moq;
+using NUnit.Framework;
+using QuantConnect.Securities;
+using QuantConnect.ToolBox.RandomDataGenerator;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+
+namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
+{
+    [TestFixture]
+    public class DefaultSymbolGeneratorTests
+    {
+        private const int Seed = 123456789;
+        private static readonly IRandomValueGenerator _randomValueGenerator = new RandomValueGenerator(Seed);
+
+        private BaseSymbolGenerator _symbolGenerator;
+        private DateTime _minExpiry = new(2000, 01, 01);
+        private DateTime _maxExpiry = new(2001, 01, 01);
+
+        [SetUp]
+        public void Setup()
+        {
+            // initialize using a seed for deterministic tests
+            _symbolGenerator = new DefaultSymbolGenerator(
+                new RandomDataGeneratorSettings()
+                {
+                    Market = Market.CME,
+                    Start = _minExpiry,
+                    End = _maxExpiry
+                },
+                _randomValueGenerator);
+        }
+
+        [Test]
+        [TestCase(SecurityType.Equity)]
+        [TestCase(SecurityType.Index)]
+        public void ReturnsDefaultSymbolGeneratorInstance(SecurityType securityType)
+        {
+            Assert.IsInstanceOf<DefaultSymbolGenerator>(BaseSymbolGenerator.Create(
+                    new RandomDataGeneratorSettings() { SecurityType = securityType },
+                    Mock.Of<IRandomValueGenerator>()
+                ));
+        }
+
+        [Test]
+        [TestCase(SecurityType.Equity, Market.USA, true)]
+        [TestCase(SecurityType.Cfd, Market.FXCM, false)]
+        [TestCase(SecurityType.Cfd, Market.Oanda, false)]
+        [TestCase(SecurityType.Forex, Market.FXCM, false)]
+        [TestCase(SecurityType.Forex, Market.Oanda, false)]
+        [TestCase(SecurityType.Crypto, Market.GDAX, false)]
+        [TestCase(SecurityType.Crypto, Market.Bitfinex, false)]
+        public void GetAvailableSymbolCount(SecurityType securityType, string market, bool expectInfinity)
+        {
+            var expected = expectInfinity
+                ? int.MaxValue
+                : SymbolPropertiesDatabase.FromDataFolder().GetSymbolPropertiesList(market, securityType).Count();
+
+            var symbolGenerator = new DefaultSymbolGenerator(new RandomDataGeneratorSettings
+            {
+                SecurityType = securityType,
+                Market = market
+            }, Mock.Of<RandomValueGenerator>());
+
+            Assert.AreEqual(expected, symbolGenerator.GetAvailableSymbolCount());
+        }
+
+        [Test]
+        [TestCase(SecurityType.Equity, Market.USA)]
+        [TestCase(SecurityType.Cfd, Market.FXCM)]
+        [TestCase(SecurityType.Cfd, Market.Oanda)]
+        [TestCase(SecurityType.Forex, Market.FXCM)]
+        [TestCase(SecurityType.Forex, Market.Oanda)]
+        [TestCase(SecurityType.Crypto, Market.GDAX)]
+        [TestCase(SecurityType.Crypto, Market.Bitfinex)]
+        public void NextSymbol_CreatesSymbol_WithRequestedSecurityTypeAndMarket(SecurityType securityType, string market)
+        {
+            var symbolGenerator = new DefaultSymbolGenerator(new RandomDataGeneratorSettings
+            {
+                SecurityType = securityType,
+                Market = market
+            }, _randomValueGenerator);
+
+            var symbols = BaseSymbolGeneratorTests.GenerateAsset(symbolGenerator).ToList().ToList();
+            Assert.AreEqual(1, symbols.Count);
+
+            var symbol = symbols.First();
+            Assert.AreEqual(securityType, symbol.SecurityType);
+            Assert.AreEqual(market, symbol.ID.Market);
+        }
+
+        [Test]
+        [TestCase(SecurityType.Equity, Market.USA)]
+        [TestCase(SecurityType.Cfd, Market.FXCM)]
+        [TestCase(SecurityType.Cfd, Market.Oanda)]
+        [TestCase(SecurityType.Forex, Market.FXCM)]
+        [TestCase(SecurityType.Forex, Market.Oanda)]
+        [TestCase(SecurityType.Crypto, Market.GDAX)]
+        [TestCase(SecurityType.Crypto, Market.Bitfinex)]
+        public void NextSymbol_CreatesSymbol_WithEntryInSymbolPropertiesDatabase(SecurityType securityType, string market)
+        {
+            var symbolGenerator = new DefaultSymbolGenerator(new RandomDataGeneratorSettings
+            {
+                SecurityType = securityType,
+                Market = market
+            }, _randomValueGenerator);
+
+            var symbols = BaseSymbolGeneratorTests.GenerateAsset(symbolGenerator).ToList().ToList();
+            Assert.AreEqual(1, symbols.Count);
+
+            var symbol = symbols.First();
+
+            var db = SymbolPropertiesDatabase.FromDataFolder();
+            if (db.ContainsKey(market, SecurityDatabaseKey.Wildcard, securityType))
+            {
+                // there is a wildcard entry, so no need to check whether there is a specific entry for the symbol
+                Assert.Pass();
+            }
+            else
+            {
+                // there is no wildcard entry, so there should be a specific entry for the symbol instead
+                Assert.IsTrue(db.ContainsKey(market, symbol, securityType));
+            }
+        }
+
+        [Test]
+        [TestCase(SecurityType.Cfd, Market.FXCM)]
+        [TestCase(SecurityType.Cfd, Market.Oanda)]
+        [TestCase(SecurityType.Forex, Market.FXCM)]
+        [TestCase(SecurityType.Forex, Market.Oanda)]
+        [TestCase(SecurityType.Crypto, Market.GDAX)]
+        [TestCase(SecurityType.Crypto, Market.Bitfinex)]
+        public void NextSymbol_ThrowsNoTickersAvailableException_WhenAllSymbolsGenerated(SecurityType securityType, string market)
+        {
+            var db = SymbolPropertiesDatabase.FromDataFolder();
+            var symbolCount = db.GetSymbolPropertiesList(market, securityType).Count();
+
+            var symbolGenerator = new DefaultSymbolGenerator(new RandomDataGeneratorSettings
+            {
+                SecurityType = securityType,
+                Market = market
+            }, _randomValueGenerator);
+
+            for (var i = 0; i < symbolCount; i++)
+            {
+                var symbols = BaseSymbolGeneratorTests.GenerateAsset(symbolGenerator).ToList();
+                Assert.AreEqual(1, symbols.Count);
+            }
+
+            Assert.Throws<NoTickersAvailableException>(() =>
+                BaseSymbolGeneratorTests.GenerateAsset(symbolGenerator).ToList()
+            );
+        }
+    }
+}

Tests/ToolBox/RandomDataGenerator/FutureSymbolGeneratorTests.cs

@@ -0,0 +1,104 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using Moq;
+using NUnit.Framework;
+using QuantConnect.Securities;
+using QuantConnect.ToolBox.RandomDataGenerator;
+using System;
+using System.Linq;
+
+namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
+{
+    [TestFixture]
+    public class FutureSymbolGeneratorTests
+    {
+        private const int Seed = 123456789;
+        private static readonly IRandomValueGenerator _randomValueGenerator = new RandomValueGenerator(Seed);
+
+        private BaseSymbolGenerator _symbolGenerator;
+        private DateTime _minExpiry = new(2000, 01, 01);
+        private DateTime _maxExpiry = new(2001, 01, 01);
+
+        [SetUp]
+        public void Setup()
+        {
+            // initialize using a seed for deterministic tests
+            _symbolGenerator = new FutureSymbolGenerator(
+                new RandomDataGeneratorSettings()
+                {
+                    Market = Market.CME,
+                    Start = _minExpiry,
+                    End = _maxExpiry
+                },
+                _randomValueGenerator);
+        }
+
+        [Test]
+        [TestCase(SecurityType.Future)]
+        public void ReturnsFutureSymbolGeneratorInstance(SecurityType securityType)
+        {
+            Assert.IsInstanceOf<FutureSymbolGenerator>(BaseSymbolGenerator.Create(
+                new RandomDataGeneratorSettings { SecurityType = securityType },
+                Mock.Of<IRandomValueGenerator>()
+            ));
+        }
+
+        [Test]
+        public void GetAvailableSymbolCount()
+        {
+            Assert.AreEqual(int.MaxValue, _symbolGenerator.GetAvailableSymbolCount());
+        }
+
+        [Test]
+        public void NextFuture_CreatesSymbol_WithFutureSecurityTypeAndRequestedMarket()
+        {
+            var symbols = BaseSymbolGeneratorTests.GenerateAsset(_symbolGenerator).ToList();
+            Assert.AreEqual(1, symbols.Count);
+
+            var symbol = symbols.First();
+
+            Assert.AreEqual(Market.CME, symbol.ID.Market);
+            Assert.AreEqual(SecurityType.Future, symbol.SecurityType);
+        }
+
+        [Test]
+        public void NextFuture_CreatesSymbol_WithFutureWithValidFridayExpiry()
+        {
+            var symbols = BaseSymbolGeneratorTests.GenerateAsset(_symbolGenerator).ToList();
+            Assert.AreEqual(1, symbols.Count);
+
+            var symbol = symbols.First();
+
+            var expiry = symbol.ID.Date;
+            Assert.Greater(expiry, _minExpiry);
+            Assert.LessOrEqual(expiry, _maxExpiry);
+            Assert.AreEqual(DayOfWeek.Friday, expiry.DayOfWeek);
+        }
+
+        [Test]
+        public void NextFuture_CreatesSymbol_WithEntryInSymbolPropertiesDatabase()
+        {
+            var symbols = BaseSymbolGeneratorTests.GenerateAsset(_symbolGenerator).ToList();
+            Assert.AreEqual(1, symbols.Count);
+
+            var symbol = symbols.First();
+
+            var db = SymbolPropertiesDatabase.FromDataFolder();
+            Assert.IsTrue(db.ContainsKey(Market.CME, symbol, SecurityType.Future));
+        }
+    }
+
+}

Tests/ToolBox/RandomDataGenerator/OptionPriceModelPriceGeneratorTests.cs

@@ -0,0 +1,133 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using Moq;
+using NUnit.Framework;
+using QuantConnect.ToolBox.RandomDataGenerator;
+using System;
+using QLNet;
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using QuantConnect.Securities;
+using QuantConnect.Securities.Option;
+using Cash = QuantConnect.Securities.Cash;
+using Option = QuantConnect.Securities.Option.Option;
+
+namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
+{
+    [TestFixture]
+    public class OptionPriceModelPriceGeneratorTests
+    {
+        private Security _underlying;
+        private Option _option;
+
+        public OptionPriceModelPriceGeneratorTests()
+        {
+            _underlying = new Security(
+                SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
+                new SubscriptionDataConfig(
+                    typeof(TradeBar),
+                    Symbols.SPY,
+                    Resolution.Minute,
+                    TimeZones.NewYork,
+                    TimeZones.NewYork,
+                    false,
+                    false,
+                    false
+                ),
+                new Cash("USD", 0, 1m),
+                SymbolProperties.GetDefault("USD"),
+                ErrorCurrencyConverter.Instance,
+                RegisteredSecurityDataTypesProvider.Null,
+                new SecurityCache()
+            );
+
+            var optionSymbol = Symbol.CreateOption(
+                _underlying.Symbol,
+                _underlying.Symbol.ID.Market,
+                _underlying.Symbol.SecurityType.DefaultOptionStyle(),
+                OptionRight.Call,
+                20,
+                new DateTime(2022, 1, 1));
+
+            _option = new Option(
+                optionSymbol,
+                SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
+                new Cash("USD", 0, 1m),
+                new OptionSymbolProperties(_underlying.SymbolProperties),
+                new CashBook(),
+                new RegisteredSecurityDataTypesProvider(),
+                new OptionCache(),
+                _underlying);
+        }
+
+        [Test]
+        public void ThrowsIfSecurityIsNull()
+        {
+            Assert.Throws<ArgumentNullException>(() =>
+            {
+                _ = new OptionPriceModelPriceGenerator(null);
+            });
+        }
+
+        [Test]
+        public void ThrowsIfSecurityIsNotOption()
+        {
+            Assert.Throws<ArgumentException>(() =>
+            {
+                _ = new OptionPriceModelPriceGenerator(_underlying);
+            });
+        }
+
+        [Test]
+        public void ReturnsNewPrice()
+        {
+            var priceModelMock = new Mock<IOptionPriceModel>();
+            priceModelMock
+                .Setup(s => s.Evaluate(It.IsAny<Security>(), It.IsAny<Slice>(), It.IsAny<OptionContract>()))
+                .Returns(new OptionPriceModelResult(1000, new Greeks()));
+            _option.PriceModel = priceModelMock.Object;
+            var randomPriceGenerator = new OptionPriceModelPriceGenerator(_option);
+
+            Assert.AreEqual(1000, randomPriceGenerator.NextValue(50, new DateTime(2020, 1, 1)));
+        }
+
+        [Test]
+        public void WarmedUpIfNotQLOptionPriceModel()
+        {
+            _option.PriceModel = Mock.Of<IOptionPriceModel>();
+            var blackScholesModel = new OptionPriceModelPriceGenerator(_option);
+
+            Assert.True(blackScholesModel.WarmedUp);
+        }
+
+        [Test]
+        [TestCase(true)]
+        [TestCase(false)]
+        public void WarmedUpSameQLOptionPriceModel(bool warmUp)
+        {
+            var volatilityModel = new Mock<IQLUnderlyingVolatilityEstimator>();
+            volatilityModel.SetupGet(s => s.IsReady).Returns(warmUp);
+            _option.PriceModel = new QLOptionPriceModel(process => new AnalyticEuropeanEngine(process),
+                volatilityModel.Object,
+                null,
+                null);
+
+            var blackScholesModel = new OptionPriceModelPriceGenerator(_option);
+
+            Assert.AreEqual(warmUp, blackScholesModel.WarmedUp);
+        }
+    }
+}

Tests/ToolBox/RandomDataGenerator/OptionSymbolGeneratorTests.cs

@@ -0,0 +1,144 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using Moq;
+using NUnit.Framework;
+using QuantConnect.ToolBox.RandomDataGenerator;
+using System;
+using System.Linq;
+
+namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
+{
+    [TestFixture]
+    public class OptionSymbolGeneratorTests
+    {
+        private const int Seed = 123456789;
+        private static readonly IRandomValueGenerator _randomValueGenerator = new RandomValueGenerator(Seed);
+
+        private BaseSymbolGenerator _symbolGenerator;
+        private DateTime _minExpiry = new(2000, 01, 01);
+        private DateTime _maxExpiry = new(2001, 01, 01);
+        private decimal _underlyingPrice = 100m;
+        private decimal _maximumStrikePriceDeviation = 50m;
+
+        [SetUp]
+        public void Setup()
+        {
+            // initialize using a seed for deterministic tests
+            _symbolGenerator = new OptionSymbolGenerator(
+                new RandomDataGeneratorSettings()
+                {
+                    Market = Market.USA,
+                    Start = _minExpiry,
+                    End = _maxExpiry
+                },
+                _randomValueGenerator,
+                _underlyingPrice,
+                _maximumStrikePriceDeviation);
+        }
+
+        [Test]
+        [TestCase(SecurityType.Option)]
+        public void ReturnsFutureSymbolGeneratorInstance(SecurityType securityType)
+        {
+            Assert.IsInstanceOf<OptionSymbolGenerator>(BaseSymbolGenerator.Create(
+                new RandomDataGeneratorSettings { SecurityType = securityType },
+                Mock.Of<IRandomValueGenerator>()
+            ));
+        }
+
+        [Test]
+        public void GetAvailableSymbolCount()
+        {
+            Assert.AreEqual(int.MaxValue,
+                new OptionSymbolGenerator(Mock.Of<RandomDataGeneratorSettings>(), Mock.Of<RandomValueGenerator>(), 100m,
+                    75m).GetAvailableSymbolCount());
+        }
+
+        [Test]
+        public void NextOptionSymbol_CreatesOptionSymbol_WithCorrectSecurityTypeAndEquityUnderlying()
+        {
+            var symbols = BaseSymbolGeneratorTests.GenerateAsset(_symbolGenerator).ToList();
+            Assert.AreEqual(3, symbols.Count);
+
+            var underlying = symbols[0];
+            var option = symbols[1];
+
+            Assert.AreEqual(SecurityType.Option, option.SecurityType);
+            Assert.AreEqual(OptionRight.Put, symbols[1].ID.OptionRight);
+            Assert.AreEqual(SecurityType.Option, symbols[2].SecurityType);
+            Assert.AreEqual(OptionRight.Call, symbols[2].ID.OptionRight);
+
+            var underlyingOrigin = option.Underlying;
+            Assert.AreEqual(underlying.Value, underlyingOrigin.Value);
+            Assert.AreEqual(Market.USA, underlying.ID.Market);
+            Assert.AreEqual(SecurityType.Equity, underlying.SecurityType);
+        }
+
+        [Test]
+        public void NextOptionSymbol_CreatesOptionSymbol_WithinSpecifiedExpiration_OnFriday()
+        {
+            var symbols = BaseSymbolGeneratorTests.GenerateAsset(_symbolGenerator).ToList();
+            Assert.AreEqual(3, symbols.Count);
+
+            foreach (var option in new[] { symbols[1], symbols[2] })
+            {
+                var expiration = option.ID.Date;
+                Assert.LessOrEqual(_minExpiry, expiration);
+                Assert.GreaterOrEqual(_maxExpiry, expiration);
+            }
+        }
+
+        [Test]
+        public void NextOptionSymbol_CreatesOptionSymbol_WithRequestedMarket()
+        {
+            for (int i = 0; i < 50; i++)
+            {
+                var price = _randomValueGenerator.NextPrice(SecurityType.Equity, Market.USA, 100m, 100m);
+                var symbolGenerator = new OptionSymbolGenerator(
+                    new RandomDataGeneratorSettings()
+                    {
+                        Market = Market.USA,
+                        Start = _minExpiry,
+                        End = _maxExpiry
+                    },
+                    _randomValueGenerator,
+                    price,
+                    50m);
+                var symbols = BaseSymbolGeneratorTests.GenerateAsset(_symbolGenerator).ToList();
+                Assert.AreEqual(3, symbols.Count);
+
+                var option = symbols[1];
+
+                Assert.AreEqual(Market.USA, option.ID.Market);
+            }
+        }
+
+        [Test]
+        public void NextOptionSymbol_CreatesOptionSymbol_WithinSpecifiedStrikePriceDeviation()
+        {
+            var symbols = BaseSymbolGeneratorTests.GenerateAsset(_symbolGenerator).ToList();
+            Assert.AreEqual(3, symbols.Count);
+
+            foreach (var option in new []{ symbols[1], symbols[2] })
+            {
+                var strikePrice = option.ID.StrikePrice;
+                var maximumDeviation = _underlyingPrice * (_maximumStrikePriceDeviation / 100m);
+                Assert.LessOrEqual(_underlyingPrice - maximumDeviation, strikePrice);
+                Assert.GreaterOrEqual(_underlyingPrice + maximumDeviation, strikePrice);
+            }
+        }
+    }
+}

Tests/ToolBox/RandomDataGenerator/RandomDataGeneratorTests.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -20,7 +20,7 @@ using QuantConnect.ToolBox.RandomDataGenerator;
 namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
 {
     [TestFixture]
-    public class RandomDataGeneratorProgramTests
+    public class RandomDataGeneratorTests
     {
         [Test]
         [TestCase("2020, 1, 1 00:00:00", "2020, 1, 1 00:00:00", "2020, 1, 1 00:00:00")]
@@ -31,8 +31,8 @@ namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
         public void NextRandomGeneratedData(DateTime start, DateTime end, DateTime expectedMidPoint)
         {
             var randomValueGenerator = new RandomValueGenerator();
-            var midPoint = RandomDataGeneratorProgram.GetDateMidpoint(start, end);
-            var delistDate = RandomDataGeneratorProgram.GetDelistingDate(start, end, randomValueGenerator);
+            var midPoint = QuantConnect.ToolBox.RandomDataGenerator.RandomDataGenerator.GetDateMidpoint(start, end);
+            var delistDate = QuantConnect.ToolBox.RandomDataGenerator.RandomDataGenerator.GetDelistingDate(start, end, randomValueGenerator);
 
             // midPoint and expectedMidPoint must be the same
             Assert.AreEqual(expectedMidPoint, midPoint);
@@ -45,4 +45,4 @@ namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
             Assert.GreaterOrEqual(delistDate, midPoint);
         }
     }
-}
+}

Tests/ToolBox/RandomDataGenerator/RandomPriceGeneratorTests.cs

@@ -0,0 +1,83 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using Moq;
+using NUnit.Framework;
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using QuantConnect.Securities;
+using QuantConnect.ToolBox.RandomDataGenerator;
+using System;
+using System.Linq;
+using QuantConnect.Util;
+
+namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
+{
+    [TestFixture]
+    public class RandomPriceGeneratorTests
+    {
+        private Security _security;
+
+        public RandomPriceGeneratorTests()
+        {
+            _security = new Security(
+                SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
+                new SubscriptionDataConfig(
+                    typeof(TradeBar),
+                    Symbols.SPY,
+                    Resolution.Minute,
+                    TimeZones.NewYork,
+                    TimeZones.NewYork,
+                    false,
+                    false,
+                    false
+                ),
+                new Cash("USD", 0, 1m),
+                SymbolProperties.GetDefault("USD"),
+                ErrorCurrencyConverter.Instance,
+                RegisteredSecurityDataTypesProvider.Null,
+                new SecurityCache()
+            );
+        }
+
+        [Test]
+        public void ReturnsSameAsReference()
+        {
+            var randomMock = new Mock<IRandomValueGenerator>();
+            randomMock.Setup(s => s.NextPrice(It.IsAny<SecurityType>(), It.IsNotNull<string>(), It.IsAny<decimal>(),
+                It.IsAny<decimal>()))
+                .Returns(50);
+            var randomPriceGenerator = new RandomPriceGenerator(_security, randomMock.Object);
+            _security.SetMarketPrice(new Tick(DateTime.UtcNow, Symbols.SPY, 10, 100));
+
+            var actual = randomPriceGenerator.NextValue(1, DateTime.MinValue);
+            randomMock.Verify(s => s.NextPrice(It.IsAny<SecurityType>(), It.IsNotNull<string>(), 55, 1), Times.Once);
+            Assert.AreEqual(50, actual);
+        }
+
+        [Test]
+        public void AlwaysReady()
+        {
+            var priceGenerator = new RandomPriceGenerator(_security, Mock.Of<IRandomValueGenerator>());
+            Assert.True(priceGenerator.WarmedUp);
+        }
+
+        [Test]
+        public void ComposeRandomDataGenerator()
+        {
+            Assert.NotNull(Composer.Instance.GetExportedValueByTypeName<QuantConnect.ToolBox.RandomDataGenerator.RandomDataGenerator>("QuantConnect.ToolBox.RandomDataGenerator.RandomDataGenerator"));
+        }
+    }
+}

Tests/ToolBox/RandomDataGenerator/RandomValueGeneratorTests.cs

@@ -1,9 +1,21 @@
-using System;
-using System.Linq;
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
 using NUnit.Framework;
-using QuantConnect.Brokerages;
-using QuantConnect.Securities;
 using QuantConnect.ToolBox.RandomDataGenerator;
+using System;
 
 namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
 {
@@ -20,24 +32,6 @@ namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
             randomValueGenerator = new RandomValueGenerator(Seed);
         }
 
-        [Test]
-        [TestCase(2,5)]
-        [TestCase(3,3)]
-        [TestCase(1,4)]
-        public void NextUpperCaseString_CreatesString_WithinSpecifiedMinMaxLength(int min, int max)
-        {
-            var str = randomValueGenerator.NextUpperCaseString(min, max);
-            Assert.LessOrEqual(min, str.Length);
-            Assert.GreaterOrEqual(max, str.Length);
-        }
-
-        [Test]
-        public void NextUpperCaseString_CreatesUpperCaseString()
-        {
-            var str = randomValueGenerator.NextUpperCaseString(10, 10);
-            Assert.IsTrue(str.All(char.IsUpper));
-        }
-
         [Test]
         public void NextDateTime_CreatesDateTime_WithinSpecifiedMinMax()
         {
@@ -86,326 +80,5 @@ namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
                 randomValueGenerator.NextDate(min, max, DayOfWeek.Monday)
             );
         }
-
-        [Test]
-        [TestCase(SecurityType.Option)]
-        [TestCase(SecurityType.Future)]
-        public void NextSymbol_ThrowsArgumentException_ForDerivativeSymbols(SecurityType securityType)
-        {
-            Assert.Throws<ArgumentException>(() =>
-                randomValueGenerator.NextSymbol(securityType, Market.USA)
-            );
-        }
-
-        [Test]
-        [TestCase(SecurityType.Equity, Market.USA)]
-        [TestCase(SecurityType.Cfd, Market.FXCM)]
-        [TestCase(SecurityType.Cfd, Market.Oanda)]
-        [TestCase(SecurityType.Forex, Market.FXCM)]
-        [TestCase(SecurityType.Forex, Market.Oanda)]
-        [TestCase(SecurityType.Crypto, Market.GDAX)]
-        [TestCase(SecurityType.Crypto, Market.Bitfinex)]
-        public void NextSymbol_CreatesSymbol_WithRequestedSecurityTypeAndMarket(SecurityType securityType, string market)
-        {
-            var symbol = randomValueGenerator.NextSymbol(securityType, market);
-
-            Assert.AreEqual(securityType, symbol.SecurityType);
-            Assert.AreEqual(market, symbol.ID.Market);
-        }
-
-        [Test]
-        [TestCase(SecurityType.Equity, Market.USA)]
-        [TestCase(SecurityType.Cfd, Market.FXCM)]
-        [TestCase(SecurityType.Cfd, Market.Oanda)]
-        [TestCase(SecurityType.Forex, Market.FXCM)]
-        [TestCase(SecurityType.Forex, Market.Oanda)]
-        [TestCase(SecurityType.Crypto, Market.GDAX)]
-        [TestCase(SecurityType.Crypto, Market.Bitfinex)]
-        public void NextSymbol_CreatesSymbol_WithEntryInSymbolPropertiesDatabase(SecurityType securityType, string market)
-        {
-            var symbol = randomValueGenerator.NextSymbol(securityType, market);
-
-            var db = SymbolPropertiesDatabase.FromDataFolder();
-            if (db.ContainsKey(market, SecurityDatabaseKey.Wildcard, securityType))
-            {
-                // there is a wildcard entry, so no need to check whether there is a specific entry for the symbol
-                Assert.Pass();
-            }
-            else
-            {
-                // there is no wildcard entry, so there should be a specific entry for the symbol instead
-                Assert.IsTrue(db.ContainsKey(market, symbol, securityType));
-            }
-        }
-
-        [Test]
-        [TestCase(SecurityType.Cfd, Market.FXCM)]
-        [TestCase(SecurityType.Cfd, Market.Oanda)]
-        [TestCase(SecurityType.Forex, Market.FXCM)]
-        [TestCase(SecurityType.Forex, Market.Oanda)]
-        [TestCase(SecurityType.Crypto, Market.GDAX)]
-        [TestCase(SecurityType.Crypto, Market.Bitfinex)]
-        public void NextSymbol_ThrowsNoTickersAvailableException_WhenAllSymbolsGenerated(SecurityType securityType, string market)
-        {
-            var db = SymbolPropertiesDatabase.FromDataFolder();
-            var symbolCount = db.GetSymbolPropertiesList(market, securityType).Count();
-
-            for (var i = 0; i < symbolCount; i++)
-            {
-                randomValueGenerator.NextSymbol(securityType, market);
-            }
-
-            Assert.Throws<NoTickersAvailableException>(() =>
-                randomValueGenerator.NextSymbol(securityType, market)
-            );
-        }
-
-        [Test]
-        public void NextOptionSymbol_CreatesOptionSymbol_WithCorrectSecurityTypeAndEquityUnderlying()
-        {
-            var minExpiry = new DateTime(2000, 01, 01);
-            var maxExpiry = new DateTime(2001, 01, 01);
-            var symbol = randomValueGenerator.NextOption(Market.USA, minExpiry, maxExpiry, 100m, 50);
-
-            Assert.AreEqual(SecurityType.Option, symbol.SecurityType);
-
-            var underlying = symbol.Underlying;
-            Assert.AreEqual(Market.USA, underlying.ID.Market);
-            Assert.AreEqual(SecurityType.Equity, underlying.SecurityType);
-        }
-
-        [Test]
-        public void NextOptionSymbol_CreatesOptionSymbol_WithinSpecifiedExpiration_OnFriday()
-        {
-            var minExpiry = new DateTime(2000, 01, 01);
-            var maxExpiry = new DateTime(2001, 01, 01);
-            var symbol = randomValueGenerator.NextOption(Market.USA, minExpiry, maxExpiry, 100m, 50);
-
-            var expiration = symbol.ID.Date;
-            Assert.LessOrEqual(minExpiry, expiration);
-            Assert.GreaterOrEqual(maxExpiry, expiration);
-        }
-
-        [Test]
-        public void NextOptionSymbol_CreatesOptionSymbol_WithRequestedMarket()
-        {
-            for (int i = 0; i < 50; i++)
-            {
-                var minExpiry = new DateTime(2000, 01, 01);
-                var maxExpiry = new DateTime(2001, 01, 01);
-                var price = randomValueGenerator.NextPrice(SecurityType.Equity, Market.USA, 100m, 100m);
-                var symbol = randomValueGenerator.NextOption(Market.USA, minExpiry, maxExpiry, price, 50);
-
-                Assert.AreEqual(Market.USA, symbol.ID.Market);
-            }
-        }
-
-        [Test]
-        public void NextOptionSymbol_CreatesOptionSymbol_WithinSpecifiedStrikePriceDeviation()
-        {
-            var underlyingPrice = 100m;
-            var maximumStrikePriceDeviation = 50m;
-            var minExpiry = new DateTime(2000, 01, 01);
-            var maxExpiry = new DateTime(2001, 01, 01);
-            var symbol = randomValueGenerator.NextOption(Market.USA, minExpiry, maxExpiry, underlyingPrice, maximumStrikePriceDeviation);
-
-            var strikePrice = symbol.ID.StrikePrice;
-            var maximumDeviation = underlyingPrice * (maximumStrikePriceDeviation / 100m);
-            Assert.LessOrEqual(underlyingPrice - maximumDeviation, strikePrice);
-            Assert.GreaterOrEqual(underlyingPrice + maximumDeviation, strikePrice);
-        }
-
-        [Test]
-        public void NextTick_CreatesTradeTick_WithPriceAndQuantity()
-        {
-            var dateTime = new DateTime(2000, 01, 01);
-            var symbol = randomValueGenerator.NextSymbol(SecurityType.Equity, Market.USA);
-            var tick = randomValueGenerator.NextTick(symbol, dateTime, TickType.Trade, 100m, 1m);
-
-            Assert.AreEqual(symbol, tick.Symbol);
-            Assert.AreEqual(dateTime, tick.Time);
-            Assert.AreEqual(TickType.Trade, tick.TickType);
-            Assert.LessOrEqual(99m, tick.Value);
-            Assert.GreaterOrEqual(101m, tick.Value);
-
-            Assert.Greater(tick.Quantity, 0);
-            Assert.LessOrEqual(tick.Quantity, 1500);
-        }
-
-        [Test]
-        public void NextTick_CreatesQuoteTick_WithCommonValues()
-        {
-            var dateTime = new DateTime(2000, 01, 01);
-            var symbol = randomValueGenerator.NextSymbol(SecurityType.Equity, Market.USA);
-            var tick = randomValueGenerator.NextTick(symbol, dateTime, TickType.Quote, 100m, 1m);
-
-            Assert.AreEqual(symbol, tick.Symbol);
-            Assert.AreEqual(dateTime, tick.Time);
-            Assert.AreEqual(TickType.Quote, tick.TickType);
-            Assert.GreaterOrEqual(tick.Value, 99m);
-            Assert.LessOrEqual(tick.Value, 101m);
-        }
-
-        [Test]
-        public void NextTick_CreatesQuoteTick_WithBidData()
-        {
-            var dateTime = new DateTime(2000, 01, 01);
-            var symbol = randomValueGenerator.NextSymbol(SecurityType.Equity, Market.USA);
-            var tick = randomValueGenerator.NextTick(symbol, dateTime, TickType.Quote, 100m, 1m);
-
-            Assert.Greater(tick.BidSize, 0);
-            Assert.LessOrEqual(tick.BidSize, 1500);
-            Assert.GreaterOrEqual(tick.BidPrice, 98.9m);
-            Assert.LessOrEqual(tick.BidPrice, 100.9m);
-            Assert.GreaterOrEqual(tick.Value, tick.BidPrice);
-        }
-
-        [Test]
-        public void NextTick_CreatesQuoteTick_WithAskData()
-        {
-            var dateTime = new DateTime(2000, 01, 01);
-            var symbol = randomValueGenerator.NextSymbol(SecurityType.Equity, Market.USA);
-            var tick = randomValueGenerator.NextTick(symbol, dateTime, TickType.Quote, 100m, 1m);
-
-            Assert.GreaterOrEqual(tick.AskSize, 0);
-            Assert.LessOrEqual(tick.AskSize, 1500);
-            Assert.GreaterOrEqual(tick.AskPrice, 99.1m);
-            Assert.LessOrEqual(tick.AskPrice, 101.1m);
-            Assert.LessOrEqual(tick.Value, tick.AskPrice);
-        }
-
-        [Test]
-        public void NextTick_CreatesOpenInterestTick()
-        {
-            var dateTime = new DateTime(2000, 01, 01);
-            var symbol = randomValueGenerator.NextSymbol(SecurityType.Equity, Market.USA);
-            var tick = randomValueGenerator.NextTick(symbol, dateTime, TickType.OpenInterest, 10000m, 10m);
-
-            Assert.AreEqual(dateTime, tick.Time);
-            Assert.AreEqual(TickType.OpenInterest, tick.TickType);
-            Assert.AreEqual(symbol, tick.Symbol);
-            Assert.GreaterOrEqual(tick.Quantity, 9000);
-            Assert.LessOrEqual(tick.Quantity, 11000);
-            Assert.AreEqual(tick.Value, tick.Quantity);
-        }
-
-        [Test]
-        [TestCase(Resolution.Tick, DataDensity.Dense)]
-        [TestCase(Resolution.Second, DataDensity.Dense)]
-        [TestCase(Resolution.Minute, DataDensity.Dense)]
-        [TestCase(Resolution.Hour, DataDensity.Dense)]
-        [TestCase(Resolution.Daily, DataDensity.Dense)]
-        [TestCase(Resolution.Tick, DataDensity.Sparse)]
-        [TestCase(Resolution.Second, DataDensity.Sparse)]
-        [TestCase(Resolution.Minute, DataDensity.Sparse)]
-        [TestCase(Resolution.Hour, DataDensity.Sparse)]
-        [TestCase(Resolution.Daily, DataDensity.Sparse)]
-        [TestCase(Resolution.Tick, DataDensity.VerySparse)]
-        [TestCase(Resolution.Second, DataDensity.VerySparse)]
-        [TestCase(Resolution.Minute, DataDensity.VerySparse)]
-        [TestCase(Resolution.Hour, DataDensity.VerySparse)]
-        [TestCase(Resolution.Daily, DataDensity.VerySparse)]
-        public void NextTickTime_CreatesTimes(Resolution resolution, DataDensity density)
-        {
-            var count = 100;
-            var deltaSum = TimeSpan.Zero;
-            var previous = new DateTime(2019, 01, 14, 9, 30, 0);
-            var increment = resolution.ToTimeSpan();
-            if (increment == TimeSpan.Zero)
-            {
-                increment = TimeSpan.FromMilliseconds(500);
-            }
-            var symbol = randomValueGenerator.NextSymbol(SecurityType.Equity, Market.USA);
-            var marketHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
-            for (int i = 0; i < count; i++)
-            {
-                var next = randomValueGenerator.NextTickTime(symbol, previous, resolution, density);
-                var barStart = next.Subtract(increment);
-                Assert.Less(previous, next);
-                Assert.IsTrue(marketHours.IsOpen(barStart, next, false));
-
-                var delta = next - previous;
-                deltaSum += delta;
-
-                previous = next;
-            }
-
-            var avgDelta = TimeSpan.FromTicks(deltaSum.Ticks / count);
-            switch (density)
-            {
-                case DataDensity.Dense:
-                    // more frequent than once an increment
-                    Assert.Less(avgDelta, increment);
-                    break;
-
-                case DataDensity.Sparse:
-                    // less frequent that once an increment
-                    Assert.Greater(avgDelta, increment);
-                    break;
-
-                case DataDensity.VerySparse:
-                    // less frequent than one every 10 increments
-                    Assert.Greater(avgDelta, TimeSpan.FromTicks(increment.Ticks * 10));
-                    break;
-
-                default:
-                    throw new ArgumentOutOfRangeException(nameof(density), density, null);
-            }
-        }
-
-        [Test]
-        public void NextFuture_CreatesSymbol_WithFutureSecurityTypeAndRequestedMarket()
-        {
-            var minExpiry = new DateTime(2000, 01, 01);
-            var maxExpiry = new DateTime(2001, 01, 01);
-            var symbol = randomValueGenerator.NextFuture(Market.CME, minExpiry, maxExpiry);
-
-            Assert.AreEqual(Market.CME, symbol.ID.Market);
-            Assert.AreEqual(SecurityType.Future, symbol.SecurityType);
-        }
-
-        [Test]
-        public void NextFuture_CreatesSymbol_WithFutureWithValidFridayExpiry()
-        {
-            var minExpiry = new DateTime(2000, 01, 01);
-            var maxExpiry = new DateTime(2001, 01, 01);
-            var symbol = randomValueGenerator.NextFuture(Market.CME, minExpiry, maxExpiry);
-
-            var expiry = symbol.ID.Date;
-            Assert.Greater(expiry, minExpiry);
-            Assert.LessOrEqual(expiry, maxExpiry);
-            Assert.AreEqual(DayOfWeek.Friday, expiry.DayOfWeek);
-        }
-
-        [Test]
-        public void NextFuture_CreatesSymbol_WithEntryInSymbolPropertiesDatabase()
-        {
-            var minExpiry = new DateTime(2000, 01, 01);
-            var maxExpiry = new DateTime(2001, 01, 01);
-            var symbol = randomValueGenerator.NextFuture(Market.CME, minExpiry, maxExpiry);
-
-            var db = SymbolPropertiesDatabase.FromDataFolder();
-            Assert.IsTrue(db.ContainsKey(Market.CME, symbol, SecurityType.Future));
-        }
-
-        [Test]
-        [TestCase(SecurityType.Equity, Market.USA, true)]
-        [TestCase(SecurityType.Cfd, Market.FXCM, false)]
-        [TestCase(SecurityType.Cfd, Market.Oanda, false)]
-        [TestCase(SecurityType.Forex, Market.FXCM, false)]
-        [TestCase(SecurityType.Forex, Market.Oanda, false)]
-        [TestCase(SecurityType.Crypto, Market.GDAX, false)]
-        [TestCase(SecurityType.Crypto, Market.Bitfinex, false)]
-        [TestCase(SecurityType.Option, Market.USA, true)]
-        [TestCase(SecurityType.Future, Market.CME, true)]
-        [TestCase(SecurityType.Future, Market.CBOE, true)]
-        public void GetAvailableSymbolCount(SecurityType securityType, string market, bool expectInfinity)
-        {
-            var expected = expectInfinity
-                ? int.MaxValue
-                : SymbolPropertiesDatabase.FromDataFolder().GetSymbolPropertiesList(market, securityType).Count();
-
-            Assert.AreEqual(expected, randomValueGenerator.GetAvailableSymbolCount(securityType, market));
-        }
     }
 }

Tests/ToolBox/RandomDataGenerator/SecurityInitializerProviderTests.cs

@@ -0,0 +1,33 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using Moq;
+using NUnit.Framework;
+using QuantConnect.Securities;
+using QuantConnect.ToolBox.RandomDataGenerator;
+
+namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
+{
+    [TestFixture]
+    public class SecurityInitializerProviderTests
+    {
+        [Test]
+        public void NotNull()
+        {
+            var securityInitializerProvider = new SecurityInitializerProvider(Mock.Of<ISecurityInitializer>());
+            Assert.NotNull(securityInitializerProvider.SecurityInitializer);
+        }
+    }
+}

Tests/ToolBox/RandomDataGenerator/TickGeneratorTests.cs

@@ -0,0 +1,217 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using NUnit.Framework;
+using QuantConnect.Data;
+using QuantConnect.Data.Market;
+using QuantConnect.Securities;
+using QuantConnect.ToolBox.RandomDataGenerator;
+
+namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
+{
+    [TestFixture]
+    public class TickGeneratorTests
+    {
+        private Dictionary<SecurityType, List<TickType>> _tickTypesPerSecurityType =
+            SubscriptionManager.DefaultDataTypes();
+
+        private Symbol _symbol = Symbol.Create("AAPL", SecurityType.Equity, Market.USA);
+        private Security _security;
+        private ITickGenerator _tickGenerator;
+
+        [SetUp]
+        public void Setup()
+        {
+            var start = new DateTime(2020, 1, 1);
+            var end = new DateTime(2020, 1, 2);
+
+            // initialize using a seed for deterministic tests
+            _symbol = Symbol.Create("AAPL", SecurityType.Equity, Market.USA);
+
+            _security = new Security(
+                SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork),
+                new SubscriptionDataConfig(typeof(TradeBar),
+                    _symbol,
+                    Resolution.Minute,
+                    TimeZones.NewYork,
+                    TimeZones.NewYork,
+                    true, true, false),
+                new Cash(Currencies.USD, 0, 0),
+                SymbolProperties.GetDefault(Currencies.USD),
+                ErrorCurrencyConverter.Instance,
+                RegisteredSecurityDataTypesProvider.Null,
+                new SecurityCache()
+            );
+            _security.SetMarketPrice(new Tick(start, _security.Symbol, 100, 100));
+            _security.SetMarketPrice(new OpenInterest(start, _security.Symbol, 10000));
+
+            _tickGenerator = new TickGenerator(
+                new RandomDataGeneratorSettings()
+                {
+                    Start = start,
+                    End = end
+                },
+                _tickTypesPerSecurityType[_symbol.SecurityType].ToArray(),
+                _security,
+                new RandomValueGenerator());
+
+        }
+
+        [Test]
+        public void NextTick_CreatesTradeTick_WithPriceAndQuantity()
+        {
+            var dateTime = new DateTime(2000, 01, 01);
+            var tick = _tickGenerator.NextTick(dateTime, TickType.Trade, 1m);
+
+            Assert.AreEqual(_symbol, tick.Symbol);
+            Assert.AreEqual(dateTime, tick.Time);
+            Assert.AreEqual(TickType.Trade, tick.TickType);
+            Assert.LessOrEqual(99m, tick.Value);
+            Assert.GreaterOrEqual(101m, tick.Value);
+
+            Assert.Greater(tick.Quantity, 0);
+            Assert.LessOrEqual(tick.Quantity, 1500);
+        }
+
+        [Test]
+        public void NextTick_CreatesQuoteTick_WithCommonValues()
+        {
+            var dateTime = new DateTime(2000, 01, 01);
+            var tick = _tickGenerator.NextTick(dateTime, TickType.Quote, 1m);
+
+            Assert.AreEqual(_symbol, tick.Symbol);
+            Assert.AreEqual(dateTime, tick.Time);
+            Assert.AreEqual(TickType.Quote, tick.TickType);
+            Assert.GreaterOrEqual(tick.Value, 99m);
+            Assert.LessOrEqual(tick.Value, 101m);
+        }
+
+        [Test]
+        public void NextTick_CreatesQuoteTick_WithBidData()
+        {
+            var dateTime = new DateTime(2000, 01, 01);
+            var tick = _tickGenerator.NextTick(dateTime, TickType.Quote, 1m);
+
+            Assert.Greater(tick.BidSize, 0);
+            Assert.LessOrEqual(tick.BidSize, 1500);
+            Assert.GreaterOrEqual(tick.BidPrice, 98.9m);
+            Assert.LessOrEqual(tick.BidPrice, 100.9m);
+            Assert.GreaterOrEqual(tick.Value, tick.BidPrice);
+        }
+
+        [Test]
+        public void NextTick_CreatesQuoteTick_WithAskData()
+        {
+            var dateTime = new DateTime(2000, 01, 01);
+            var tick = _tickGenerator.NextTick(dateTime, TickType.Quote, 1m);
+
+            Assert.GreaterOrEqual(tick.AskSize, 0);
+            Assert.LessOrEqual(tick.AskSize, 1500);
+            Assert.GreaterOrEqual(tick.AskPrice, 99.1m);
+            Assert.LessOrEqual(tick.AskPrice, 101.1m);
+            Assert.LessOrEqual(tick.Value, tick.AskPrice);
+        }
+
+        [Test]
+        public void NextTick_CreatesOpenInterestTick()
+        {
+            var dateTime = new DateTime(2000, 01, 01);
+            var tick = _tickGenerator.NextTick(dateTime, TickType.OpenInterest, 10m);
+
+            Assert.AreEqual(dateTime, tick.Time);
+            Assert.AreEqual(TickType.OpenInterest, tick.TickType);
+            Assert.AreEqual(_symbol, tick.Symbol);
+            Assert.GreaterOrEqual(tick.Quantity, 9000);
+            Assert.LessOrEqual(tick.Quantity, 11000);
+            Assert.AreEqual(tick.Value, tick.Quantity);
+        }
+
+        [Test]
+        [TestCase(Resolution.Tick, DataDensity.Dense)]
+        [TestCase(Resolution.Second, DataDensity.Dense)]
+        [TestCase(Resolution.Minute, DataDensity.Dense)]
+        [TestCase(Resolution.Hour, DataDensity.Dense)]
+        [TestCase(Resolution.Daily, DataDensity.Dense)]
+        [TestCase(Resolution.Tick, DataDensity.Sparse)]
+        [TestCase(Resolution.Second, DataDensity.Sparse)]
+        [TestCase(Resolution.Minute, DataDensity.Sparse)]
+        [TestCase(Resolution.Hour, DataDensity.Sparse)]
+        [TestCase(Resolution.Daily, DataDensity.Sparse)]
+        [TestCase(Resolution.Tick, DataDensity.VerySparse)]
+        [TestCase(Resolution.Second, DataDensity.VerySparse)]
+        [TestCase(Resolution.Minute, DataDensity.VerySparse)]
+        [TestCase(Resolution.Hour, DataDensity.VerySparse)]
+        [TestCase(Resolution.Daily, DataDensity.VerySparse)]
+        public void NextTickTime_CreatesTimes(Resolution resolution, DataDensity density)
+        {
+            var count = 100;
+            var deltaSum = TimeSpan.Zero;
+            var previous = new DateTime(2019, 01, 14, 9, 30, 0);
+            var increment = resolution.ToTimeSpan();
+            if (increment == TimeSpan.Zero)
+            {
+                increment = TimeSpan.FromMilliseconds(500);
+            }
+
+            var marketHours = MarketHoursDatabase.FromDataFolder()
+                .GetExchangeHours(_symbol.ID.Market, _symbol, _symbol.SecurityType);
+            for (int i = 0; i < count; i++)
+            {
+                var next = _tickGenerator.NextTickTime(previous, resolution, density);
+                var barStart = next.Subtract(increment);
+                Assert.Less(previous, next);
+                Assert.IsTrue(marketHours.IsOpen(barStart, next, false));
+
+                var delta = next - previous;
+                deltaSum += delta;
+
+                previous = next;
+            }
+
+            var avgDelta = TimeSpan.FromTicks(deltaSum.Ticks / count);
+            switch (density)
+            {
+                case DataDensity.Dense:
+                    // more frequent than once an increment
+                    Assert.Less(avgDelta, increment);
+                    break;
+
+                case DataDensity.Sparse:
+                    // less frequent that once an increment
+                    Assert.Greater(avgDelta, increment);
+                    break;
+
+                case DataDensity.VerySparse:
+                    // less frequent than one every 10 increments
+                    Assert.Greater(avgDelta, TimeSpan.FromTicks(increment.Ticks * 10));
+                    break;
+
+                default:
+                    throw new ArgumentOutOfRangeException(nameof(density), density, null);
+            }
+        }
+
+        [Test]
+        public void HistoryIsNotEmpty()
+        {
+            var history = _tickGenerator.GenerateTicks().ToList();
+            Assert.IsNotEmpty(history);
+            Assert.That(history.Select(s => s.Symbol), Is.All.EqualTo(_symbol));
+        }
+    }
+}

ToolBox/FactorFileGenerator.cs

@@ -298,22 +298,11 @@ namespace QuantConnect.ToolBox
         /// <returns>A list of <see cref="TradeBar"/> read from file</returns>
         private static List<TradeBar> ReadDailyEquityData(string pathForDailyEquityData)
         {
-            using (var zipToOpen = new FileStream(pathForDailyEquityData, FileMode.Open, FileAccess.Read, FileShare.Read))
-            {
-                using (var archive = new ZipArchive(zipToOpen, ZipArchiveMode.Read))
-                {
-                    foreach (var entry in archive.Entries)
-                    {
-                        var parser = new LeanParser();
-                        var stream = entry.Open();
-                        return parser.Parse(pathForDailyEquityData, stream)
-                                     .OrderByDescending(x => x.Time)
-                                     .Select(x => (TradeBar)x)
-                                     .ToList();
-                    }
-                }
-            }
-            return new List<TradeBar>();
+            var dataReader = new LeanDataReader(pathForDailyEquityData);
+            var bars = dataReader.Parse();
+            return bars.OrderByDescending(x => x.Time)
+                         .Select(x => (TradeBar)x)
+                         .ToList();
         }
 
         /// <summary>

ToolBox/Program.cs

@@ -41,6 +41,7 @@ using QuantConnect.ToolBox.ZerodhaDownloader;
 using QuantConnect.Util;
 using System;
 using System.IO;
+using QuantConnect.Interfaces;
 using static QuantConnect.Configuration.ApplicationParser;
 
 namespace QuantConnect.ToolBox
@@ -64,6 +65,16 @@ namespace QuantConnect.ToolBox
                 PrintMessageAndExit();
             }
 
+            var dataProvider
+                = Composer.Instance.GetExportedValueByTypeName<IDataProvider>(Config.Get("data-provider", "DefaultDataProvider"));
+            var mapFileProvider
+                = Composer.Instance.GetExportedValueByTypeName<IMapFileProvider>(Config.Get("map-file-provider", "LocalDiskMapFileProvider"));
+            var factorFileProvider
+                = Composer.Instance.GetExportedValueByTypeName<IFactorFileProvider>(Config.Get("factor-file-provider", "LocalDiskFactorFileProvider"));
+            
+            mapFileProvider.Initialize(dataProvider);
+            factorFileProvider.Initialize(mapFileProvider, dataProvider);
+
             var targetApp = GetParameterOrExit(optionsObject, "app").ToLowerInvariant();
             if (targetApp.Contains("download") || targetApp.EndsWith("dl"))
             {
@@ -220,6 +231,7 @@ namespace QuantConnect.ToolBox
                         break;
                     case "rdg":
                     case "randomdatagenerator":
+                        var tickers = ToolboxArgumentParser.GetTickers(optionsObject);
                         RandomDataGeneratorProgram.RandomDataGenerator(
                             GetParameterOrExit(optionsObject, "start"),
                             GetParameterOrExit(optionsObject, "end"),
@@ -235,7 +247,11 @@ namespace QuantConnect.ToolBox
                             GetParameterOrDefault(optionsObject, "rename-percentage", "30.0"),
                             GetParameterOrDefault(optionsObject, "splits-percentage", "15.0"),
                             GetParameterOrDefault(optionsObject, "dividends-percentage", "60.0"),
-                            GetParameterOrDefault(optionsObject, "dividend-every-quarter-percentage", "30.0")
+                            GetParameterOrDefault(optionsObject, "dividend-every-quarter-percentage", "30.0"),
+                            GetParameterOrDefault(optionsObject, "option-price-engine", "BaroneAdesiWhaleyApproximationEngine"),
+                            GetParameterOrDefault(optionsObject, "volatility-model-resolution", "Daily"),
+                            GetParameterOrDefault(optionsObject, "chain-symbol-count", "1"),
+                            tickers
                         );
                         break;
 

ToolBox/QuantConnect.ToolBox.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
     <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
@@ -40,7 +40,7 @@
     <Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="CoinAPI.WebSocket.V1" Version="1.6.7" />
     <PackageReference Include="Common.Logging" Version="3.4.1" />
     <PackageReference Include="Common.Logging.Core" Version="3.4.1" />

ToolBox/RandomDataGenerator/BaseSymbolGenerator.cs

@@ -0,0 +1,253 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using QuantConnect.Securities;
+using QuantConnect.Util;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace QuantConnect.ToolBox.RandomDataGenerator
+{
+    /// <summary>
+    /// Provide the base symbol generator implementation
+    /// </summary>
+    public abstract class BaseSymbolGenerator
+    {
+
+        /// <summary>
+        /// <see cref="IRandomValueGenerator"/> instance producing random values for use in random data generation
+        /// </summary>
+        protected IRandomValueGenerator Random { get; }
+
+        /// <summary>
+        /// Settings of current random data generation run
+        /// </summary>
+        protected RandomDataGeneratorSettings Settings { get; }
+
+        /// <summary>
+        /// Exchange hours and raw data times zones in various markets
+        /// </summary>
+        protected MarketHoursDatabase MarketHoursDatabase { get; }
+
+        /// <summary>
+        /// Access to specific properties for various symbols
+        /// </summary>
+        protected SymbolPropertiesDatabase SymbolPropertiesDatabase { get; }
+
+        // used to prevent generating duplicates, but also caps
+        // the memory allocated to checking for duplicates
+        private readonly FixedSizeHashQueue<Symbol> _symbols;
+
+        /// <summary>
+        /// Base constructor implementation for Symbol generator
+        /// </summary>
+        /// <param name="settings">random data generation run settings</param>
+        /// <param name="random">produces random values for use in random data generation</param>
+        protected BaseSymbolGenerator(RandomDataGeneratorSettings settings, IRandomValueGenerator random)
+        {
+            Settings = settings;
+            Random = random;
+            _symbols = new FixedSizeHashQueue<Symbol>(1000);
+            SymbolPropertiesDatabase = SymbolPropertiesDatabase.FromDataFolder();
+            MarketHoursDatabase = MarketHoursDatabase.FromDataFolder();
+        }
+
+        /// <summary>
+        /// Creates a ad-hoc symbol generator depending on settings
+        /// </summary>
+        /// <param name="settings">random data generator settings</param>
+        /// <param name="random">produces random values for use in random data generation</param>
+        /// <returns>New symbol generator</returns>
+        public static BaseSymbolGenerator Create(RandomDataGeneratorSettings settings, IRandomValueGenerator random)
+        {
+            if (settings is null)
+            {
+                throw new ArgumentNullException(nameof(settings), "Settings cannot be null or empty");
+            }
+
+            if (random is null)
+            {
+                throw new ArgumentNullException(nameof(random), "Randomizer cannot be null");
+            }
+
+            switch (settings.SecurityType)
+            {
+                case SecurityType.Option:
+                    return new OptionSymbolGenerator(settings, random, 100m, 75m);
+
+                case SecurityType.Future:
+                    return new FutureSymbolGenerator(settings, random);
+
+                default:
+                    return new DefaultSymbolGenerator(settings, random);
+            }
+        }
+
+        /// <summary>
+        /// Generates specified number of symbols
+        /// </summary>
+        /// <returns>Set of random symbols</returns>
+        public IEnumerable<Symbol> GenerateRandomSymbols()
+        {
+            if (!Settings.Tickers.IsNullOrEmpty())
+            {
+                foreach (var symbol in Settings.Tickers.SelectMany(GenerateAsset))
+                {
+                    yield return symbol;
+                }
+            }
+            else
+            {
+                for (var i = 0; i < Settings.SymbolCount; i++)
+                {
+                    foreach (var symbol in GenerateAsset())
+                    {
+                        yield return symbol;
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Generates a random asset
+        /// </summary>
+        /// <param name="ticker">Optionally can provide a ticker that should be used</param>
+        /// <returns>Random asset</returns>
+        protected abstract IEnumerable<Symbol> GenerateAsset(string ticker = null);
+
+        /// <summary>
+        /// Generates random symbol, used further down for asset
+        /// </summary>
+        /// <param name="securityType">security type</param>
+        /// <param name="market">market</param>
+        /// <param name="ticker">Optionally can provide a ticker to use</param>
+        /// <returns>Random symbol</returns>
+        public Symbol NextSymbol(SecurityType securityType, string market, string ticker = null)
+        {
+            if (securityType == SecurityType.Option || securityType == SecurityType.Future)
+            {
+                throw new ArgumentException("Please use OptionSymbolGenerator or FutureSymbolGenerator for SecurityType.Option and SecurityType.Future respectively.");
+            }
+
+            if (ticker == null)
+            {
+                // we must return a Symbol matching an entry in the Symbol properties database
+                // if there is a wildcard entry, we can generate a truly random Symbol
+                // if there is no wildcard entry, the symbols we can generate are limited by the entries in the database
+                if (SymbolPropertiesDatabase.ContainsKey(market, SecurityDatabaseKey.Wildcard, securityType))
+                {
+                    // let's make symbols all have 3 chars as it's acceptable for all security types with wildcard entries
+                    ticker = NextUpperCaseString(3, 3);
+                }
+                else
+                {
+                    ticker = NextTickerFromSymbolPropertiesDatabase(securityType, market);
+                }
+            }
+
+            // by chance we may generate a ticker that actually exists, and if map files exist that match this
+            // ticker then we'll end up resolving the first trading date for use in the SID, otherwise, all
+            // generated Symbol will have a date equal to SecurityIdentifier.DefaultDate
+            var symbol = Symbol.Create(ticker, securityType, market);
+            if (_symbols.Add(symbol))
+            {
+                return symbol;
+            }
+
+            // lo' and behold, we created a duplicate --recurse to find a unique value
+            // this is purposefully done as the last statement to enable the compiler to
+            // unroll this method into a tail-recursion loop :)
+            return NextSymbol(securityType, market);
+        }
+
+        /// <summary>
+        /// Return a Ticker matching an entry in the Symbol properties database
+        /// </summary>
+        /// <param name="securityType">security type</param>
+        /// <param name="market"></param>
+        /// <returns>Random Ticker matching an entry in the Symbol properties database</returns>
+        protected string NextTickerFromSymbolPropertiesDatabase(SecurityType securityType, string market)
+        {
+            // prevent returning a ticker matching any previously generated Symbol
+            var existingTickers = _symbols
+                .Where(sym => sym.ID.Market == market && sym.ID.SecurityType == securityType)
+                .Select(sym => sym.Value);
+
+            // get the available tickers from the Symbol properties database and remove previously generated tickers
+            var availableTickers = Enumerable.Except(SymbolPropertiesDatabase.GetSymbolPropertiesList(market, securityType)
+                    .Select(kvp => kvp.Key.Symbol), existingTickers)
+                .ToList();
+
+            // there is a limited number of entries in the Symbol properties database so we may run out of tickers
+            if (availableTickers.Count == 0)
+            {
+                throw new NoTickersAvailableException(securityType, market);
+            }
+
+            return availableTickers[Random.NextInt(availableTickers.Count)];
+        }
+
+        /// <summary>
+        /// Generates random expiration date on a friday within specified time range
+        /// </summary>
+        /// <param name="marketHours">market hours</param>
+        /// <param name="minExpiry">minimum expiration date</param>
+        /// <param name="maxExpiry">maximum expiration date</param>
+        /// <returns>Random date on a friday within specified time range</returns>
+        protected DateTime GetRandomExpiration(SecurityExchangeHours marketHours, DateTime minExpiry, DateTime maxExpiry)
+        {
+            // generate a random expiration date on a friday
+            var expiry = Random.NextDate(minExpiry, maxExpiry, DayOfWeek.Friday);
+
+            // check to see if we're open on this date and if not, back track until we are
+            // we're using the equity market hours as a proxy since we haven't generated the option Symbol yet
+            while (!marketHours.IsDateOpen(expiry))
+            {
+                expiry = expiry.AddDays(-1);
+            }
+
+            return expiry;
+        }
+
+        /// <summary>
+        /// Generates a random <see cref="string"/> within the specified lengths.
+        /// </summary>
+        /// <param name="minLength">The minimum length, inclusive</param>
+        /// <param name="maxLength">The maximum length, inclusive</param>
+        /// <returns>A new upper case string within the specified lengths</returns>
+        public string NextUpperCaseString(int minLength, int maxLength)
+        {
+            var str = string.Empty;
+            var length = Random.NextInt(minLength, maxLength);
+            for (int i = 0; i < length; i++)
+            {
+                // A=65 - inclusive lower bound
+                // Z=90 - inclusive upper bound
+                var c = (char)Random.NextInt(65, 91);
+                str += c;
+            }
+
+            return str;
+        }
+
+        /// <summary>
+        /// Returns the number of symbols with the specified parameters can be generated.
+        /// Returns int.MaxValue if there is no limit for the given parameters.
+        /// </summary>
+        /// <returns>The number of available symbols for the given parameters, or int.MaxValue if no limit</returns>
+        public abstract int GetAvailableSymbolCount();
+    }
+}

ToolBox/RandomDataGenerator/ConsoleLeveledOutput.cs

@@ -1,39 +0,0 @@
-using System;
-using System.IO;
-using QuantConnect.Util;
-
-namespace QuantConnect.ToolBox.RandomDataGenerator
-{
-    public class ConsoleLeveledOutput
-    {
-        public TextWriter Info { get; }
-        public TextWriter Warn { get; }
-        public TextWriter Error { get; }
-        public bool ErrorMessageWritten { get; private set; }
-
-        public ConsoleLeveledOutput()
-        {
-            Info = Console.Out;
-            Warn = new FuncTextWriter(line =>
-            {
-                Console.ForegroundColor = ConsoleColor.Yellow;
-                Console.WriteLine(line);
-                Console.ResetColor();
-            });
-            Error = new FuncTextWriter(line =>
-            {
-                Console.ForegroundColor = ConsoleColor.Red;
-                Console.WriteLine(line);
-                Console.ResetColor();
-                ErrorMessageWritten = true;
-            });
-        }
-
-        public ConsoleLeveledOutput(TextWriter info, TextWriter warn, TextWriter error)
-        {
-            Info = info;
-            Warn = warn;
-            Error = error;
-        }
-    }
-}

ToolBox/RandomDataGenerator/DefaultSymbolGenerator.cs

@@ -0,0 +1,72 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using QuantConnect.Securities;
+
+namespace QuantConnect.ToolBox.RandomDataGenerator
+{
+    /// <summary>
+    /// Generates a new random <see cref="Symbol"/> object of the specified security type.
+    /// All returned symbols have a matching entry in the Symbol properties database.
+    /// </summary>
+    /// <remarks>
+    /// A valid implementation will keep track of generated Symbol objects to ensure duplicates
+    /// are not generated.
+    /// </remarks>
+    public class DefaultSymbolGenerator : BaseSymbolGenerator
+    {
+        private readonly string _market;
+        private readonly SecurityType _securityType;
+
+        /// <summary>
+        /// Creates <see cref="DefaultSymbolGenerator"/> instance
+        /// </summary>
+        /// <param name="settings">random data generation run settings</param>
+        /// <param name="random">produces random values for use in random data generation</param>
+        public DefaultSymbolGenerator(RandomDataGeneratorSettings settings, IRandomValueGenerator random)
+            : base(settings, random)
+        {
+            _market = settings.Market;
+            _securityType = settings.SecurityType;
+        }
+
+        /// <summary>
+        /// Generates a single-item list at a time using base random implementation 
+        /// </summary>
+        /// <returns></returns>
+        protected override IEnumerable<Symbol> GenerateAsset(string ticker = null)
+        {
+            yield return NextSymbol(Settings.SecurityType, Settings.Market, ticker);
+        }
+
+        /// <summary>
+        /// Returns the number of symbols with the specified parameters can be generated.
+        /// Returns int.MaxValue if there is no limit for the given parameters.
+        /// </summary>
+        /// <returns>The number of available symbols for the given parameters, or int.MaxValue if no limit</returns>
+        public override int GetAvailableSymbolCount()
+        {
+            // check the Symbol properties database to determine how many symbols we can generate
+            // if there is a wildcard entry, we can generate as many symbols as we want
+            // if there is no wildcard entry, we can only generate as many symbols as there are entries
+            return SymbolPropertiesDatabase.ContainsKey(_market, SecurityDatabaseKey.Wildcard, _securityType)
+                ? int.MaxValue
+                : SymbolPropertiesDatabase.GetSymbolPropertiesList(_market, _securityType).Count();
+        }
+    }
+}

ToolBox/RandomDataGenerator/DividendSplitMapGenerator.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
@@ -24,15 +24,15 @@ namespace QuantConnect.ToolBox.RandomDataGenerator
         /// <summary>
         /// Stores <see cref="MapFileRow"/> instances
         /// </summary>
-        public List<MapFileRow> MapRows = new List<MapFileRow>();
+        public List<MapFileRow> MapRows = new();
 
         /// <summary>
         /// Stores <see cref="CorporateFactorRow"/> instances
         /// </summary>
         public List<CorporateFactorRow> DividendsSplits = new List<CorporateFactorRow>();
-        
+
         /// <summary>
-        /// Current symbol value. Can be renamed
+        /// Current Symbol value. Can be renamed
         /// </summary>
         public Symbol CurrentSymbol { get; private set; }
 
@@ -41,11 +41,13 @@ namespace QuantConnect.ToolBox.RandomDataGenerator
         private readonly RandomDataGeneratorSettings _settings;
         private readonly DateTime _delistDate;
         private readonly bool _willBeDelisted;
+        private readonly BaseSymbolGenerator _symbolGenerator;
 
         public DividendSplitMapGenerator(
-            Symbol symbol, 
-            RandomDataGeneratorSettings settings, 
-            RandomValueGenerator randomValueGenerator, 
+            Symbol symbol,
+            RandomDataGeneratorSettings settings,
+            RandomValueGenerator randomValueGenerator,
+            BaseSymbolGenerator symbolGenerator,
             Random random,
             DateTime delistDate,
             bool willBeDelisted)
@@ -56,6 +58,7 @@ namespace QuantConnect.ToolBox.RandomDataGenerator
             _random = random;
             _delistDate = delistDate;
             _willBeDelisted = willBeDelisted;
+            _symbolGenerator = symbolGenerator;
         }
 
         /// <summary>
@@ -184,7 +187,7 @@ namespace QuantConnect.ToolBox.RandomDataGenerator
                             var randomDate = _randomValueGenerator.NextDate(tick.Time, tick.Time.AddMonths(1), (DayOfWeek)_random.Next(1, 5));
                             MapRows.Add(new MapFileRow(randomDate, CurrentSymbol.Value));
 
-                            CurrentSymbol = _randomValueGenerator.NextSymbol(_settings.SecurityType, _settings.Market);
+                            CurrentSymbol = _symbolGenerator.NextSymbol(_settings.SecurityType, _settings.Market);
                         }
 
                         previousMonth = tick.Time.Month;

ToolBox/RandomDataGenerator/FutureSymbolGenerator.cs

@@ -0,0 +1,65 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+
+namespace QuantConnect.ToolBox.RandomDataGenerator
+{
+    /// <summary>
+    /// Generates a new random future <see cref="Symbol"/>. The generates future contract Symbol will have an
+    /// expiry between the specified time range.
+    /// </summary>
+    public class FutureSymbolGenerator : BaseSymbolGenerator
+    {
+        private readonly DateTime _minExpiry;
+        private readonly DateTime _maxExpiry;
+        private readonly string _market;
+
+        public FutureSymbolGenerator(RandomDataGeneratorSettings settings, IRandomValueGenerator random)
+            : base(settings, random)
+        {
+            _minExpiry = settings.Start;
+            _maxExpiry = settings.End;
+            _market = settings.Market;
+        }
+
+        /// <summary>
+        /// Generates a new random future <see cref="Symbol"/>. The generates future contract Symbol will have an
+        /// expiry between the specified minExpiry and maxExpiry.
+        /// </summary>
+        /// <param name="ticker">Optionally can provide a ticker that should be used</param>
+        /// <returns>A new future contract Symbol with the specified expiration parameters</returns>
+        protected override IEnumerable<Symbol> GenerateAsset(string ticker = null)
+        {
+            if (ticker == null)
+            {
+                // get a valid ticker from the Symbol properties database
+                ticker = NextTickerFromSymbolPropertiesDatabase(SecurityType.Future, _market);
+            }
+
+            var marketHours = MarketHoursDatabase.GetExchangeHours(_market, ticker, SecurityType.Future);
+            var expiry = GetRandomExpiration(marketHours, _minExpiry, _maxExpiry);
+
+            yield return Symbol.CreateFuture(ticker, _market, expiry);
+        }
+
+        /// <summary>
+        /// There is no limit for the future symbols.
+        /// </summary>
+        /// <returns>Returns int.MaxValue</returns>
+        public override int GetAvailableSymbolCount() => int.MaxValue;
+    }
+}

ToolBox/RandomDataGenerator/IPriceGenerator.cs

@@ -0,0 +1,42 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+
+namespace QuantConnect.ToolBox.RandomDataGenerator
+{
+    /// <summary>
+    /// Defines a type capable of producing random prices
+    /// </summary>
+    /// <remarks>
+    /// Any parameters referenced as a percentage value are always in 'percent space', meaning 1 is 1%.
+    /// </remarks>
+    public interface IPriceGenerator
+    {
+        /// <summary>
+        /// Generates an asset price
+        /// </summary>
+        /// <param name="maximumPercentDeviation">The maximum percent deviation. This value is in percent space,
+        ///     so a value of 1m is equal to 1%.</param>
+        /// <param name="referenceDate">date used in price calculation</param>
+        /// <returns>Returns a new decimal as price</returns>
+        public decimal NextValue(decimal maximumPercentDeviation, DateTime referenceDate);
+
+        /// <summary>
+        /// Indicates Price generator warmed up and ready to generate new values
+        /// </summary>
+        public bool WarmedUp { get; }
+    }
+}

ToolBox/RandomDataGenerator/IRandomValueGenerator.cs

@@ -1,6 +1,20 @@
-using System;
-using QuantConnect.Data.Market;
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
 using QuantConnect.Securities;
+using System;
 
 namespace QuantConnect.ToolBox.RandomDataGenerator
 {
@@ -20,12 +34,36 @@ namespace QuantConnect.ToolBox.RandomDataGenerator
         bool NextBool(double percentOddsForTrue);
 
         /// <summary>
-        /// Generates a random <see cref="string"/> within the specified lengths.
+        /// Returns a random floating-point number that is greater than or equal to 0.0, and less than 1.0
         /// </summary>
-        /// <param name="minLength">The minimum length, inclusive</param>
-        /// <param name="maxLength">The maximum length, inclusive</param>
-        /// <returns>A new upper case string within the specified lengths</returns>
-        string NextUpperCaseString(int minLength, int maxLength);
+        /// <returns>A double-precision floating point number that is greater than or equal to 0.0, and less than 1.0.</returns>
+        double NextDouble();
+
+        /// <summary>
+        /// Returns a random integer that is within a specified range.
+        /// </summary>
+        /// <param name="minValue">the inclusive lower bound of the random number returned</param>
+        /// <param name="maxValue">the exclusive upper bound of the random number returned</param>
+        /// <returns>A 32-bit signed integer greater than or equal to minValue and less than maxValue.</returns>
+        int NextInt(int minValue, int maxValue);
+
+        /// <summary>
+        /// Returns a non-negative random integer that is less than the specified maximum.
+        /// </summary>
+        /// <param name="maxValue">the exclusive upper bound of the random number to be generated.</param>
+        /// <returns>A 32-bit signed integer that is greater than or equal to 0, and less than maxValue.</returns>
+        int NextInt(int maxValue);
+
+        /// <summary>
+        /// Generates a random <see cref="DateTime"/> between the specified <paramref name="minDateTime"/> and
+        /// <paramref name="maxDateTime"/>. <paramref name="dayOfWeek"/> is optionally specified to force the
+        /// result to a particular day of the week
+        /// </summary>
+        /// <param name="minDateTime">The minimum date time, inclusive</param>
+        /// <param name="maxDateTime">The maximum date time, inclusive</param>
+        /// <param name="dayOfWeek">Optional. The day of week to force</param>
+        /// <returns>A new <see cref="DateTime"/> within the specified range and optionally of the specified day of week</returns>
+        DateTime NextDate(DateTime minDateTime, DateTime maxDateTime, DayOfWeek? dayOfWeek);
 
         /// <summary>
         /// Generates a random <see cref="decimal"/> suitable as a price. This should observe minimum price
@@ -42,101 +80,5 @@ namespace QuantConnect.ToolBox.RandomDataGenerator
         /// <returns>A new decimal suitable for usage as price within the specified deviation from the reference price</returns>
         decimal NextPrice(SecurityType securityType, string market, decimal referencePrice, decimal maximumPercentDeviation);
 
-        /// <summary>
-        /// Generates a random <see cref="DateTime"/> between the specified <paramref name="minDateTime"/> and
-        /// <paramref name="maxDateTime"/>. <paramref name="dayOfWeek"/> is optionally specified to force the
-        /// result to a particular day of the week
-        /// </summary>
-        /// <param name="minDateTime">The minimum date time, inclusive</param>
-        /// <param name="maxDateTime">The maximum date time, inclusive</param>
-        /// <param name="dayOfWeek">Optional. The day of week to force</param>
-        /// <returns>A new <see cref="DateTime"/> within the specified range and optionally of the specified day of week</returns>
-        DateTime NextDate(DateTime minDateTime, DateTime maxDateTime, DayOfWeek? dayOfWeek);
-
-        /// <summary>
-        /// Generates a random <see cref="DateTime"/> suitable for use as a tick's emit time.
-        /// If the density provided is <see cref="DataDensity.Dense"/>, then at least one tick will be generated per <paramref name="resolution"/> step.
-        /// If the density provided is <see cref="DataDensity.Sparse"/>, then at least one tick will be generated every 5 <paramref name="resolution"/> steps.
-        /// if the density provided is <see cref="DataDensity.VerySparse"/>, then at least one tick will be generated every 50 <paramref name="resolution"/> steps.
-        /// Times returned are guaranteed to be within market hours for the specified symbol
-        /// </summary>
-        /// <param name="symbol">The symbol to generate a new tick time for</param>
-        /// <param name="previous">The previous tick time</param>
-        /// <param name="resolution">The requested resolution of data</param>
-        /// <param name="density">The requested data density</param>
-        /// <returns>A new <see cref="DateTime"/> that is after <paramref name="previous"/> according to the specified <paramref name="resolution"/>
-        /// and <paramref name="density"/> specified</returns>
-        DateTime NextTickTime(Symbol symbol, DateTime previous, Resolution resolution, DataDensity density);
-
-        /// <summary>
-        /// Generates a random <see cref="Tick"/> that is at most the specified <paramref name="maximumPercentDeviation"/> away from the
-        /// <paramref name="previousValue"/> and is of the requested <paramref name="tickType"/>
-        /// </summary>
-        /// <param name="symbol">The symbol of the generated tick</param>
-        /// <param name="dateTime">The time of the generated tick</param>
-        /// <param name="tickType">The type of <see cref="Tick"/> to be generated</param>
-        /// <param name="previousValue">The previous price, used as a reference for generating
-        /// new random prices for the next time step</param>
-        /// <param name="maximumPercentDeviation">The maximum percentage to deviate from the
-        /// <paramref name="previousValue"/>, for example, 1 would indicate a maximum of 1% deviation from the
-        /// <paramref name="previousValue"/>. For a previous price of 100, this would yield a price between 99 and 101 inclusive</param>
-        /// <returns>A random <see cref="Tick"/> value that is within the specified <paramref name="maximumPercentDeviation"/>
-        /// from the <paramref name="previousValue"/></returns>
-        Tick NextTick(Symbol symbol, DateTime dateTime, TickType tickType, decimal previousValue, decimal maximumPercentDeviation);
-
-        /// <summary>
-        /// Generates a new random <see cref="Symbol"/> object of the specified security type.
-        /// All returned symbols have a matching entry in the symbol properties database.
-        /// </summary>
-        /// <remarks>
-        /// A valid implementation will keep track of generated symbol objects to ensure duplicates
-        /// are not generated.
-        /// </remarks>
-        /// <exception cref="ArgumentException">Throw when specifying <see cref="SecurityType.Option"/> or
-        /// <see cref="SecurityType.Future"/>. To generate symbols for the derivative security types, please
-        /// use <see cref="NextOption"/> and <see cref="NextFuture"/> respectively</exception>
-        /// <exception cref="NoTickersAvailableException">Thrown when there are no tickers left to use for new symbols.</exception>
-        /// <param name="securityType">The security type of the generated symbol</param>
-        /// <param name="market">The market of the generated symbol</param>
-        /// <returns>A new symbol object of the specified security type</returns>
-        Symbol NextSymbol(SecurityType securityType, string market);
-
-        /// <summary>
-        /// Generates a new random option <see cref="Symbol"/>. The generated option contract symbol will have an
-        /// expiry between the specified <paramref name="minExpiry"/> and <paramref name="maxExpiry"/>. The strike
-        /// price will be within the specified <paramref name="maximumStrikePriceDeviation"/> of the <paramref name="underlyingPrice"/>
-        /// and should be rounded to reasonable value for the given price. For example, a price of 100 dollars would round
-        /// to 5 dollar increments and a price of 5 dollars would round to 50 cent increments
-        /// </summary>
-        /// <remarks>
-        /// Standard contracts expiry on the third Friday.
-        /// Weekly contracts expiry every week on Friday
-        /// </remarks>
-        /// <param name="market">The market of the generated symbol</param>
-        /// <param name="minExpiry">The minimum expiry date, inclusive</param>
-        /// <param name="maxExpiry">The maximum expiry date, inclusive</param>
-        /// <param name="underlyingPrice">The option's current underlying price</param>
-        /// <param name="maximumStrikePriceDeviation">The strike price's maximum percent deviation from the underlying price</param>
-        /// <returns>A new option contract symbol within the specified expiration and strike price parameters</returns>
-        Symbol NextOption(string market, DateTime minExpiry, DateTime maxExpiry, decimal underlyingPrice, decimal maximumStrikePriceDeviation);
-
-        /// <summary>
-        /// Generates a new random future <see cref="Symbol"/>. The generates future contract symbol will have an
-        /// expiry between the specified <paramref name="minExpiry"/> and <paramref name="maxExpiry"/>.
-        /// </summary>
-        /// <param name="market">The market of the generated symbol</param>
-        /// <param name="minExpiry">The minimum expiry date, inclusive</param>
-        /// <param name="maxExpiry">The maximum expiry date, inclusive</param>
-        /// <returns>A new future contract symbol with the specified expiration parameters</returns>
-        Symbol NextFuture(string market, DateTime minExpiry, DateTime maxExpiry);
-
-        /// <summary>
-        /// Returns the number of symbols with the specified parameters can be generated.
-        /// Returns int.MaxValue if there is no limit for the given parameters.
-        /// </summary>
-        /// <param name="securityType">The security type of the generated symbols</param>
-        /// <param name="market">The market of the generated symbols</param>
-        /// <returns>The number of available symbols for the given parameters, or int.MaxValue if no limit</returns>
-        int GetAvailableSymbolCount(SecurityType securityType, string market);
     }
 }

ToolBox/RandomDataGenerator/ITickGenerator.cs

@@ -0,0 +1,63 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Data.Market;
+using System.Collections.Generic;
+
+namespace QuantConnect.ToolBox.RandomDataGenerator
+{
+    /// <summary>
+    /// Describes main methods for <see cref="TickGenerator"/>
+    /// </summary>
+    public interface ITickGenerator
+    {
+        /// <summary>
+        /// Generates and enumerates data points for current symbol
+        /// </summary>
+        IEnumerable<Tick> GenerateTicks();
+
+        /// <summary>
+        /// Generates a random <see cref="Tick"/> that is at most the specified <paramref name="maximumPercentDeviation"/> away from the
+        /// previous price and is of the requested <paramref name="tickType"/>
+        /// </summary>
+        /// <param name="dateTime">The time of the generated tick</param>
+        /// <param name="tickType">The type of <see cref="Tick"/> to be generated</param>
+        /// <param name="maximumPercentDeviation">The maximum percentage to deviate from the
+        /// previous price, for example, 1 would indicate a maximum of 1% deviation from the
+        /// previous price. For a previous price of 100, this would yield a price between 99 and 101 inclusive</param>
+        /// <returns>A random <see cref="Tick"/> value that is within the specified <paramref name="maximumPercentDeviation"/>
+        /// from the previous price</returns>
+        Tick NextTick(
+            DateTime dateTime,
+            TickType tickType,
+            decimal maximumPercentDeviation
+            );
+
+        /// <summary>
+        /// Generates a random <see cref="DateTime"/> suitable for use as a tick's emit time.
+        /// If the density provided is <see cref="DataDensity.Dense"/>, then at least one tick will be generated per <paramref name="resolution"/> step.
+        /// If the density provided is <see cref="DataDensity.Sparse"/>, then at least one tick will be generated every 5 <paramref name="resolution"/> steps.
+        /// if the density provided is <see cref="DataDensity.VerySparse"/>, then at least one tick will be generated every 50 <paramref name="resolution"/> steps.
+        /// Times returned are guaranteed to be within market hours for the specified Symbol
+        /// </summary>
+        /// <param name="previous">The previous tick time</param>
+        /// <param name="resolution">The requested resolution of data</param>
+        /// <param name="density">The requested data density</param>
+        /// <returns>A new <see cref="DateTime"/> that is after <paramref name="previous"/> according to the specified <paramref name="resolution"/>
+        /// and <paramref name="density"/> specified</returns>
+        DateTime NextTickTime(DateTime previous, Resolution resolution, DataDensity density);
+    }
+}

ToolBox/RandomDataGenerator/OptionPriceModelPriceGenerator.cs

@@ -0,0 +1,78 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using QuantConnect.Securities;
+using System;
+using QuantConnect.Data.Market;
+using QuantConnect.Securities.Option;
+
+namespace QuantConnect.ToolBox.RandomDataGenerator
+{
+    /// <summary>
+    /// Pricing model used to determine the fair price or theoretical value for a call or a put option price
+    /// by default using the Black-Scholes-Merton model
+    /// </summary>
+    public class OptionPriceModelPriceGenerator : IPriceGenerator
+    {
+        private readonly Option _option;
+
+        /// <summary>
+        /// <see cref="RandomPriceGenerator"/> is always ready to generate new price values as it does not depend on volatility model
+        /// </summary>
+        public bool WarmedUp => _option.PriceModel is QLOptionPriceModel optionPriceModel && optionPriceModel.VolatilityEstimatorWarmedUp || _option.PriceModel is not QLOptionPriceModel;
+
+        /// <summary>
+        /// Creates instance of <see cref="OptionPriceModelPriceGenerator"/>
+        /// </summary>
+        ///<param name="security"><see cref="Security"/> object for which to generate price data</param>
+        public OptionPriceModelPriceGenerator(Security security)
+        {
+            if (security == null)
+            {
+                throw new ArgumentNullException(nameof(security), "security cannot be null");
+            }
+
+            if (!security.Symbol.SecurityType.IsOption())
+            {
+                throw new ArgumentException($"{nameof(OptionPriceModelPriceGenerator)} model cannot be applied to non-option security.");
+            }
+
+            _option = security as Option;
+        }
+
+        /// <summary>
+        /// For Black-Scholes-Merton model price calculation relies <see cref="IOptionPriceModel"/> of the security
+        /// </summary>
+        /// <param name="maximumPercentDeviation">The maximum percent deviation. This value is in percent space,
+        ///     so a value of 1m is equal to 1%.</param>
+        /// <param name="referenceDate">current reference date</param>
+        /// <returns>A new decimal suitable for usage as new security price</returns>
+        public decimal NextValue(decimal maximumPercentDeviation, DateTime referenceDate)
+        {
+            return _option.PriceModel
+                .Evaluate(
+                    _option,
+                    null,
+                    OptionContract.Create(
+                        _option.Symbol,
+                        _option.Symbol.Underlying,
+                        referenceDate,
+                        _option,
+                        _option.Underlying.Price
+                        ))
+                .TheoreticalPrice;
+        }
+    }
+}