use compression library (#6142)

- Live subscription enumerator will clone the underlying data set when
  live mapping is being done. To avoid issues where IDQH implementations
  could reuse a data point with same configurations. Adjusting unit test
  to reproduce issue

Algorithm.CSharp/ParameterizedAlgorithm.cs

@@ -28,9 +28,9 @@ namespace QuantConnect.Algorithm.CSharp
     /// <meta name="tag" content="using quantconnect" />
     public class ParameterizedAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
-        // we place attributes on top of our fields or properties that should receive
+        // We place attributes on top of our fields or properties that should receive
         // their values from the job. The values 100 and 200 are just default values that
-        // or only used if the parameters do not exist
+        // are only used if the parameters do not exist.
         [Parameter("ema-fast")]
         public int FastPeriod = 100;
 

Algorithm.CSharp/QuantConnect.Algorithm.CSharp.csproj

@@ -35,7 +35,7 @@
     <DebugType>portable</DebugType>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Accord" Version="3.6.0" />
     <PackageReference Include="Accord.Fuzzy" Version="3.6.0" />
     <PackageReference Include="Accord.MachineLearning" Version="3.6.0" />

Algorithm.Framework/QuantConnect.Algorithm.Framework.csproj

@@ -30,7 +30,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Accord" Version="3.6.0" />
     <PackageReference Include="Accord.Math" Version="3.6.0" />
     <PackageReference Include="Accord.Statistics" Version="3.6.0" />

Algorithm.Python/QuantConnect.Algorithm.Python.csproj

@@ -37,7 +37,7 @@
     <Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

Algorithm/QCAlgorithm.cs

@@ -1680,6 +1680,12 @@ namespace QuantConnect.Algorithm
                             contractDepthOffset: contractOffset
                         );
 
+                        // let's add a MHDB entry for the continuous symbol using the associated security
+                        var continuousContractSymbol = ContinuousContractUniverse.CreateSymbol(security.Symbol);
+                        MarketHoursDatabase.SetEntry(continuousContractSymbol.ID.Market,
+                            continuousContractSymbol.ID.Symbol,
+                            continuousContractSymbol.ID.SecurityType,
+                            security.Exchange.Hours);
                         AddUniverse(new ContinuousContractUniverse(security, new UniverseSettings(settings)
                             {
                                 DataMappingMode = continuousConfigs.First().DataMappingMode,
@@ -1687,7 +1693,7 @@ namespace QuantConnect.Algorithm
                                 ContractDepthOffset = (int)continuousConfigs.First().ContractDepthOffset,
                                 SubscriptionDataTypes = dataTypes
                             }, LiveMode,
-                            new SubscriptionDataConfig(canonicalConfig, symbol: ContinuousContractUniverse.CreateSymbol(security.Symbol))));
+                            new SubscriptionDataConfig(canonicalConfig, symbol: continuousContractSymbol)));
 
                         universe = new FuturesChainUniverse((Future)security, settings);
                     }

Algorithm/QuantConnect.Algorithm.csproj

@@ -30,7 +30,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>

AlgorithmFactory/QuantConnect.AlgorithmFactory.csproj

@@ -29,7 +29,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

Brokerages/InteractiveBrokers/InteractiveBrokersBrokerage.cs

@@ -55,7 +55,7 @@ namespace QuantConnect.Brokerages.InteractiveBrokers
         /// <summary>
         /// The default gateway version to use
         /// </summary>
-        public static string DefaultVersion { get; } = "985";
+        public static string DefaultVersion { get; } = "1012";
 
         private IBAutomater.IBAutomater _ibAutomater;
 

Common/Currencies.cs

@@ -23,24 +23,34 @@ namespace QuantConnect
     public static class Currencies
     {
         /// <summary>
-        /// USD currency string
+        /// USD (United States Dollar) currency string
         /// </summary>
-        public static string USD = "USD";
+        public const string USD = "USD";
 
         /// <summary>
-        /// EUR currency string
+        /// EUR (Euro) currency string
         /// </summary>
-        public static string EUR = "EUR";
+        public const string EUR = "EUR";
 
         /// <summary>
-        /// GBP currency string
+        /// GBP (British pound sterling) currency string
         /// </summary>
-        public static string GBP = "GBP";
+        public const string GBP = "GBP";
 
         /// <summary>
-        /// INR currency string
+        /// INR (Indian rupee) currency string
         /// </summary>
-        public static string INR = "INR";
+        public const string INR = "INR";
+
+        /// <summary>
+        /// CNH (Chinese Yuan Renminbi) currency string
+        /// </summary>
+        public const string CNH = "CNH";
+
+        /// <summary>
+        /// HKD (Hong Kong dollar) currency string
+        /// </summary>
+        public const string HKD = "HKD";
 
         /// <summary>
         /// Null currency used when a real one is not required
@@ -55,19 +65,19 @@ namespace QuantConnect
         /// </remarks>
         public static readonly IReadOnlyDictionary<string, string> CurrencySymbols = new Dictionary<string, string>
         {
-            {"USD", "$"},
-            {"GBP", "₤"},
+            {USD, "$"},
+            {GBP, "₤"},
             {"JPY", "¥"},
-            {"EUR", "€"},
+            {EUR, "€"},
             {"NZD", "$"},
             {"AUD", "$"},
             {"CAD", "$"},
             {"CHF", "Fr"},
-            {"HKD", "$"},
+            {HKD, "$"},
             {"SGD", "$"},
             {"XAG", "Ag"},
             {"XAU", "Au"},
-            {"CNH", "¥"},
+            {CNH, "¥"},
             {"CNY", "¥"},
             {"CZK", "Kč"},
             {"DKK", "kr"},

Common/Market.cs

@@ -279,5 +279,13 @@ namespace QuantConnect
         {
             return !ReverseMarkets.TryGetValue(code, out var market) ? null : market;
         }
+
+        /// <summary>
+        /// Returns a list of the supported markets
+        /// </summary>
+        public static List<string> SupportedMarkets()
+        {
+            return Markets.Keys.ToList();
+        }
     }
 }

Common/Orders/Fees/InteractiveBrokersFeeModel.cs

@@ -33,9 +33,16 @@ namespace QuantConnect.Orders.Fees
         private readonly Dictionary<string, Func<decimal, decimal, CashAmount>> _optionFee =
             new Dictionary<string, Func<decimal, decimal, CashAmount>>();
 
-        private readonly Dictionary<string, CashAmount> _futureFee =
+        /// <summary>
+        /// Reference at https://www.interactivebrokers.com/en/index.php?f=commission&p=futures1
+        /// </summary>
+        private readonly Dictionary<string, Func<Security, CashAmount>> _futureFee =
             //                                                               IB fee + exchange fee
-            new Dictionary<string, CashAmount> { { Market.USA, new CashAmount(0.85m + 1, "USD") } };
+            new()
+            {
+                { Market.USA, UnitedStatesFutureFees },
+                { Market.HKFE, HongKongFutureFees }
+            };
 
         /// <summary>
         /// Initializes a new instance of the <see cref="ImmediateFillModel"/>
@@ -109,17 +116,18 @@ namespace QuantConnect.Orders.Fees
                     if (market == Market.Globex || market == Market.NYMEX
                         || market == Market.CBOT || market == Market.ICE
                         || market == Market.CFE || market == Market.COMEX
-                        || market == Market.CME || market == Market.HKFE)
+                        || market == Market.CME)
                     {
                         // just in case...
                         market = Market.USA;
                     }
 
-                    CashAmount feeRatePerContract;
-                    if (!_futureFee.TryGetValue(market, out feeRatePerContract))
+                    if (!_futureFee.TryGetValue(market, out var feeRatePerContractFunc))
                     {
                         throw new KeyNotFoundException($"InteractiveBrokersFeeModel(): unexpected future Market {market}");
                     }
+
+                    var feeRatePerContract = feeRatePerContractFunc(security);
                     feeResult = order.AbsoluteQuantity * feeRatePerContract.Amount;
                     feeCurrency = feeRatePerContract.Currency;
                     break;
@@ -129,7 +137,7 @@ namespace QuantConnect.Orders.Fees
                     switch (market)
                     {
                         case Market.USA:
-                            equityFee = new EquityFee("USD", feePerShare: 0.005m, minimumFee: 1, maximumFeeRate: 0.005m);
+                            equityFee = new EquityFee(Currencies.USD, feePerShare: 0.005m, minimumFee: 1, maximumFeeRate: 0.005m);
                             break;
                         default:
                             throw new KeyNotFoundException($"InteractiveBrokersFeeModel(): unexpected equity Market {market}");
@@ -235,6 +243,42 @@ namespace QuantConnect.Orders.Fees
             }
         }
 
+        private static CashAmount UnitedStatesFutureFees(Security security)
+        {
+            return new CashAmount(0.85m + 1, Currencies.USD);
+        }
+
+        /// <summary>
+        /// See https://www.hkex.com.hk/Services/Rules-and-Forms-and-Fees/Fees/Listed-Derivatives/Trading/Transaction?sc_lang=en
+        /// </summary>
+        private static CashAmount HongKongFutureFees(Security security)
+        {
+            if (security.Symbol.ID.Symbol.Equals("HSI", StringComparison.InvariantCultureIgnoreCase))
+            {
+                // IB fee + exchange fee
+                return new CashAmount(30 + 10, Currencies.HKD);
+            }
+
+            decimal ibFeePerContract;
+            switch (security.QuoteCurrency.Symbol)
+            {
+                case Currencies.CNH:
+                    ibFeePerContract = 13;
+                    break;
+                case Currencies.HKD:
+                    ibFeePerContract = 20;
+                    break;
+                case Currencies.USD:
+                    ibFeePerContract = 2.40m;
+                    break;
+                default:
+                    throw new ArgumentException($"Unexpected quote currency {security.QuoteCurrency.Symbol} for Hong Kong futures exchange");
+            }
+
+            // let's add a 50% extra charge for exchange fees
+            return new CashAmount(ibFeePerContract * 1.5m, security.QuoteCurrency.Symbol);
+        }
+
         /// <summary>
         /// Helper class to handle IB Equity fees
         /// </summary>

Common/QuantConnect.csproj

@@ -35,7 +35,7 @@
     <Message Text="SelectedOptimization $(SelectedOptimization)" Importance="high" />
   </Target>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="CloneExtensions" Version="1.3.0" />
     <PackageReference Include="fasterflect" Version="3.0.0" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />

Configuration/Config.cs

@@ -298,8 +298,8 @@ namespace QuantConnect.Configuration
         /// </summary>
         /// <typeparam name="T">The desired output type</typeparam>
         /// <param name="key">The configuration key</param>
-        /// <param name="value">The output value</param>
-        /// <returns>True on successful parse, false when output value is default(T)</returns>
+        /// <param name="value">The output value. If the key is found and parsed successfully, it will be the parsed value, else default(T).</param>
+        /// <returns>True on successful parse or if they key is not found. False only when key is found but fails to parse.</returns>
         public static bool TryGetValue<T>(string key, out T value)
         {
             return TryGetValue(key, default(T), out value);
@@ -312,8 +312,8 @@ namespace QuantConnect.Configuration
         /// <typeparam name="T">The desired output type</typeparam>
         /// <param name="key">The configuration key</param>
         /// <param name="defaultValue">The default value to use on key not found or unsuccessful parse</param>
-        /// <param name="value">The output value</param>
-        /// <returns>True on successful parse, false when output value is defaultValue</returns>
+        /// <param name="value">The output value. If the key is found and parsed successfully, it will be the parsed value, else defaultValue.</param>
+        /// <returns>True on successful parse or if they key is not found and using defaultValue. False only when key is found but fails to parse.</returns>
         public static bool TryGetValue<T>(string key, T defaultValue, out T value)
         {
             try

DockerfileLeanFoundation

@@ -29,10 +29,10 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test \
 
 # Install IB Gateway: Installs to /root/ibgateway
 RUN mkdir -p /root/ibgateway && \
-    wget https://cdn.quantconnect.com/interactive/ibgateway-latest-standalone-linux-x64.v985.1j.sh && \
-    chmod 777 ibgateway-latest-standalone-linux-x64.v985.1j.sh && \
-    ./ibgateway-latest-standalone-linux-x64.v985.1j.sh -q -dir /root/ibgateway && \
-    rm ibgateway-latest-standalone-linux-x64.v985.1j.sh
+    wget https://cdn.quantconnect.com/interactive/ibgateway-latest-standalone-linux-x64.v10.12.2d.sh && \
+    chmod 777 ibgateway-latest-standalone-linux-x64.v10.12.2d.sh && \
+    ./ibgateway-latest-standalone-linux-x64.v10.12.2d.sh -q -dir /root/ibgateway && \
+    rm ibgateway-latest-standalone-linux-x64.v10.12.2d.sh
 
 # Install dotnet 5 sdk & runtime
 RUN wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb && \
@@ -53,9 +53,9 @@ ENV PATH="/opt/miniconda3/bin:${PATH}"
 RUN wget https://cdn.quantconnect.com/miniconda/${CONDA} && \
     bash ${CONDA} -b -p /opt/miniconda3 && rm -rf ${CONDA} && \
     ln -s /opt/miniconda3/lib/libpython3.6m.so /usr/lib/libpython3.6m.so && \
-    conda update -y conda pip && \
-    pip install --upgrade --no-cache-dir pip && \
-    conda install -y python=3.6.8 && conda clean -y --all
+    conda install -y conda=4.10.3 && \
+    pip install --upgrade --no-cache-dir pip==21.2.2 && \
+    conda install -y python=3.6.8 && conda install -y pip=21.2.2 && conda clean -y --all
 
 # Avoid pip install read timeouts
 ENV PIP_DEFAULT_TIMEOUT=120
@@ -167,7 +167,7 @@ RUN pip install --no-cache-dir      \
     ruptures==1.1.3                 \
     simpy==4.0.1                    \
     scikit-learn-extra==0.2.0       \
-    ray==1.5.1
+    ray==1.9.1
 
 # feature_selector has overly strict dependency version ranges
 # We already installed close-enough versions of all of its dependencies above
@@ -259,10 +259,10 @@ RUN wget https://cdn.quantconnect.com/tigramite/tigramite-4.1.zip && \
     python setup.py install && cd .. && rm -rf tigramite-4.1 && rm tigramite-4.1.zip
 
 # Install H2O: https://www.h2o.ai/download/
-RUN wget https://cdn.quantconnect.com/h2o/h2o-3.30.0.3.zip && \
-    unzip -q h2o-3.30.0.3.zip && \
-    pip install h2o-3.30.0.3/python/h2o-3.30.0.3-py2.py3-none-any.whl && \
-    rm -rf h2o-3.30.0.3 && rm h2o-3.30.0.3.zip
+RUN wget https://cdn.quantconnect.com/h2o/h2o-3.34.0.7.zip && \
+    unzip -q h2o-3.34.0.7.zip && \
+    pip install h2o-3.34.0.7-py2.py3-none-any.whl && \
+    rm h2o-3.34.0.7.zip h2o-3.34.0.7-py2.py3-none-any.whl
 
 # Remove black-listed packages
 RUN pip uninstall -y s3transfer

DockerfileLeanFoundationARM

@@ -27,12 +27,12 @@ RUN apt-get update && apt-fast install -y git libgtk2.0.0 cmake bzip2 curl unzip
 # We update the install script so it doesn't use the bundled JVM
 # The bundled JVM doesn't work on ARM64, so we update it to use the JVM installed in the previous command
 RUN mkdir -p /root/ibgateway && \
-    wget https://cdn.quantconnect.com/interactive/ibgateway-latest-standalone-linux-x64.v985.1j.sh && \
+    wget https://cdn.quantconnect.com/interactive/ibgateway-latest-standalone-linux-x64.v10.12.2d.sh && \
     java_patch_version=$(java -version 2>&1 | head -n 1 | cut -d'_' -f2 | cut -d'"' -f1) && \
-    bbe -e 's|# INSTALL4J_JAVA_HOME_OVERRIDE=|INSTALL4J_JAVA_HOME_OVERRIDE="/usr/lib/jvm/java-1.8.0-openjdk-arm64"|' -e "s|-lt \"152\"|-lt \"$java_patch_version\"|" -e "s|-gt \"152\"|-gt \"$java_patch_version\"|" ibgateway-latest-standalone-linux-x64.v985.1j.sh > ibgateway-stable-standalone-linux-custom-jvm.sh && \
+    bbe -e 's|# INSTALL4J_JAVA_HOME_OVERRIDE=|INSTALL4J_JAVA_HOME_OVERRIDE="/usr/lib/jvm/java-1.8.0-openjdk-arm64"|' -e "s|-lt \"152\"|-lt \"$java_patch_version\"|" -e "s|-gt \"152\"|-gt \"$java_patch_version\"|" ibgateway-latest-standalone-linux-x64.v10.12.2d.sh > ibgateway-stable-standalone-linux-custom-jvm.sh && \
     chmod 777 ibgateway-stable-standalone-linux-custom-jvm.sh && \
     ./ibgateway-stable-standalone-linux-custom-jvm.sh -q -dir /root/ibgateway && \
-    rm ibgateway-latest-standalone-linux-x64.v985.1j.sh ibgateway-stable-standalone-linux-custom-jvm.sh
+    rm ibgateway-latest-standalone-linux-x64.v10.12.2d.sh ibgateway-stable-standalone-linux-custom-jvm.sh
 
 # Install dotnet 5 sdk & runtime
 # The .deb packages don't support ARM, the install script does
@@ -54,9 +54,9 @@ RUN wget -O Miniforge3.sh https://github.com/conda-forge/miniforge/releases/late
     rm Miniforge3.sh && \
     ln -s /opt/miniforge3/lib/libpython3.6m.so /usr/lib/libpython3.6m.so && \
     conda config --append channels https://repo.anaconda.com/pkgs/main && \
-    conda update -y conda pip && \
-    pip install --upgrade --no-cache-dir pip && \
-    conda install -y python=3.6.7 && conda clean -y --all
+    conda install -y conda=4.10.3 && \
+    pip install --upgrade --no-cache-dir pip==21.2.2 && \
+    conda install -y python=3.6.7 && conda install -y pip=21.2.2 && conda clean -y --all
 
 # Avoid pip install read timeouts
 ENV PIP_DEFAULT_TIMEOUT=120
@@ -179,10 +179,10 @@ RUN wget https://cdn.quantconnect.com/tigramite/tigramite-4.1.zip && \
     python setup.py install && cd .. && rm -rf tigramite-4.1 && rm tigramite-4.1.zip
 
 # Install H2O: https://www.h2o.ai/download/
-RUN wget https://cdn.quantconnect.com/h2o/h2o-3.30.0.3.zip && \
-    unzip -q h2o-3.30.0.3.zip && \
-    pip install h2o-3.30.0.3/python/h2o-3.30.0.3-py2.py3-none-any.whl && \
-    rm -rf h2o-3.30.0.3 && rm h2o-3.30.0.3.zip
+RUN wget https://cdn.quantconnect.com/h2o/h2o-3.34.0.7.zip && \
+    unzip -q h2o-3.34.0.7.zip && \
+    pip install h2o-3.34.0.7-py2.py3-none-any.whl && \
+    rm h2o-3.34.0.7.zip h2o-3.34.0.7-py2.py3-none-any.whl
 
 # Remove black-listed packages
 RUN pip uninstall -y s3transfer

Engine/DataFeeds/ApiDataProvider.cs

@@ -42,6 +42,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         private readonly DataPricesList _dataPrices;
         private readonly Api.Api _api;
         private readonly bool _subscribedToEquityMapAndFactorFiles;
+        private readonly bool _subscribedToFutureMapAndFactorFiles;
         private volatile bool _invalidSecurityTypeLog;
 
         /// <summary>
@@ -65,10 +66,18 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             _dataPrices = _api.ReadDataPrices(_organizationId);
             var organization = _api.ReadOrganization(_organizationId);
 
-            // Determine if the user is subscribed to map and factor files (Data product Id 37)
-            if (organization.Products.Where(x => x.Type == ProductType.Data).Any(x => x.Items.Any(x => x.Id == 37)))
+            foreach (var productItem in organization.Products.Where(x => x.Type == ProductType.Data).SelectMany(product => product.Items))
             {
-                _subscribedToEquityMapAndFactorFiles = true;
+                if (productItem.Id == 37)
+                {
+                    // Determine if the user is subscribed to Equity map and factor files (Data product Id 37)
+                    _subscribedToEquityMapAndFactorFiles = true;
+                }
+                else if (productItem.Id == 137)
+                {
+                    // Determine if the user is subscribed to Future map and factor files (Data product Id 137)
+                    _subscribedToFutureMapAndFactorFiles = true;
+                }
             }
 
             // Verify user has agreed to data provider agreements
@@ -154,23 +163,41 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             // Some security types can't be downloaded, lets attempt to extract that information
             if (LeanData.TryParseSecurityType(filePath, out SecurityType securityType) && _unsupportedSecurityType.Contains(securityType))
             {
-                if (!_invalidSecurityTypeLog)
+                // we do support future auxiliary data (map and factor files)
+                if (securityType != SecurityType.Future || !IsAuxiliaryData(filePath))
                 {
-                    // let's log this once. Will still use any existing data on disk
-                    _invalidSecurityTypeLog = true;
-                    Log.Error($"ApiDataProvider(): does not support security types: {string.Join(", ", _unsupportedSecurityType)}");
+                    if (!_invalidSecurityTypeLog)
+                    {
+                        // let's log this once. Will still use any existing data on disk
+                        _invalidSecurityTypeLog = true;
+                        Log.Error($"ApiDataProvider(): does not support security types: {string.Join(", ", _unsupportedSecurityType)}");
+                    }
+                    return false;
                 }
-                return false;
             }
 
             // Only download if it doesn't exist or is out of date.
             // Files are only "out of date" for non date based files (hour, daily, margins, etc.) because this data is stored all in one file
             var shouldDownload = !File.Exists(filePath) || filePath.IsOutOfDate();
 
-            // Final check; If we want to download and the request requires equity data we need to be sure they are subscribed to map and factor files
-            if (shouldDownload && (securityType == SecurityType.Equity || securityType == SecurityType.Option || IsEquitiesAux(filePath)))
+            if (shouldDownload)
             {
-                CheckMapFactorFileSubscription();
+                if (securityType == SecurityType.Future)
+                {
+                    if (!_subscribedToFutureMapAndFactorFiles)
+                    {
+                        throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
+                            "to download Future auxiliary data from QuantConnect. " +
+                            "Please visit https://www.quantconnect.com/datasets/quantconnect-us-futures-security-master for details.");
+                    }
+                }
+                // Final check; If we want to download and the request requires equity data we need to be sure they are subscribed to map and factor files
+                else if (!_subscribedToEquityMapAndFactorFiles && (securityType == SecurityType.Equity || securityType == SecurityType.Option || IsAuxiliaryData(filePath)))
+                {
+                    throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
+                        "to download Equity data from QuantConnect. " +
+                        "Please visit https://www.quantconnect.com/datasets/quantconnect-security-master for details.");
+                }
             }
 
             return shouldDownload;
@@ -198,29 +225,16 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         }
 
         /// <summary>
-        /// Helper method to determine if this filepath is Equity Aux data
+        /// Helper method to determine if this filepath is auxiliary data
         /// </summary>
-        /// <param name="filepath"></param>
-        /// <returns>True if this file is EquitiesAux</returns>
-        private static bool IsEquitiesAux(string filepath)
+        /// <param name="filepath">The target file path</param>
+        /// <returns>True if this file is of auxiliary data</returns>
+        private static bool IsAuxiliaryData(string filepath)
         {
             return filepath.Contains("map_files", StringComparison.InvariantCulture)
                 || filepath.Contains("factor_files", StringComparison.InvariantCulture)
                 || filepath.Contains("fundamental", StringComparison.InvariantCulture)
                 || filepath.Contains("shortable", StringComparison.InvariantCulture);
         }
-
-        /// <summary>
-        /// Helper to check map and factor file subscription, throws if not subscribed.
-        /// </summary>
-        private void CheckMapFactorFileSubscription()
-        {
-            if(!_subscribedToEquityMapAndFactorFiles)
-            {
-                throw new ArgumentException("ApiDataProvider(): Must be subscribed to map and factor files to use the ApiDataProvider " +
-                    "to download Equity data from QuantConnect. " + 
-                    "Please visit https://www.quantconnect.com/datasets/quantconnect-security-master for details.");
-            }
-        }
     }
 }

Engine/DataFeeds/Enumerators/LiveSubscriptionEnumerator.cs

@@ -29,13 +29,20 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
     /// </summary>
     public class LiveSubscriptionEnumerator : IEnumerator<BaseData>
     {
+        private BaseData _current;
         private readonly Symbol _requestedSymbol;
         private SubscriptionDataConfig _currentConfig;
         private IEnumerator<BaseData> _previousEnumerator;
         private IEnumerator<BaseData> _underlyingEnumerator;
 
-        public BaseData Current => _underlyingEnumerator.Current;
+        /// <summary>
+        /// The current data object instance
+        /// </summary>
+        public BaseData Current => _current;
 
+        /// <summary>
+        /// The current data object instance
+        /// </summary>
         object IEnumerator.Current => Current;
 
         /// <summary>
@@ -73,9 +80,14 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators
             }
 
             var result = _underlyingEnumerator.MoveNext();
-            if (Current != null)
+            _current = _underlyingEnumerator.Current;
+
+            if (_current != null && _current.Symbol != _requestedSymbol)
             {
-                Current.Symbol = _requestedSymbol;
+                // if we've done some mapping at this layer let's clone the underlying and set the requested symbol,
+                // don't trust the IDQH implementations for data uniqueness, since the configuration could be shared
+                _current = _current.Clone();
+                _current.Symbol = _requestedSymbol;
             }
 
             return result;

Engine/QuantConnect.Lean.Engine.csproj

@@ -43,7 +43,7 @@
     <Message Text="SelectedOptimization $(SelectedOptimization)" Importance="high" />
   </Target>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="fasterflect" Version="3.0.0" />
     <PackageReference Include="FSharp.Core" Version="4.5.2" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />

Indicators/QuantConnect.Indicators.csproj

@@ -32,7 +32,7 @@
     <Message Text="SelectedOptimization $(SelectedOptimization)" Importance="high" />
   </Target>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>

Report/QuantConnect.Report.csproj

@@ -42,7 +42,7 @@
     <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Deedle" Version="2.1.0" />
     <PackageReference Include="MathNet.Numerics" Version="4.15.0" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">

Research/QuantConnect.Research.csproj

@@ -32,7 +32,7 @@
     <Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Microsoft.CodeAnalysis.NetAnalyzers" Version="5.0.3">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

Tests/Algorithm/AlgorithmAddSecurityTests.cs

@@ -17,15 +17,18 @@
 using System;
 using NUnit.Framework;
 using QuantConnect.Algorithm;
-using QuantConnect.Data.Custom.AlphaStreams;
-using QuantConnect.Lean.Engine.DataFeeds;
+using QuantConnect.Securities;
+using System.Collections.Generic;
 using QuantConnect.Securities.Cfd;
 using QuantConnect.Securities.Crypto;
 using QuantConnect.Securities.Equity;
 using QuantConnect.Securities.Forex;
 using QuantConnect.Securities.Future;
 using QuantConnect.Securities.Option;
+using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Tests.Engine.DataFeeds;
+using QuantConnect.Securities.IndexOption;
+using QuantConnect.Data.Custom.AlphaStreams;
 using Index = QuantConnect.Securities.Index.Index;
 
 namespace QuantConnect.Tests.Algorithm
@@ -80,6 +83,9 @@ namespace QuantConnect.Tests.Algorithm
                     case SecurityType.Index:
                         var index = (Index)security;
                         break;
+                    case SecurityType.IndexOption:
+                        var indexOption = (IndexOption)security;
+                        break;
                     case SecurityType.Crypto:
                         var crypto = (Crypto)security;
                         break;
@@ -102,7 +108,7 @@ namespace QuantConnect.Tests.Algorithm
         {
             get
             {
-                return new[]
+                var result = new List<TestCaseData>()
                 {
                     new TestCaseData(Symbols.SPY, null),
                     new TestCaseData(Symbols.EURUSD, null),
@@ -117,6 +123,21 @@ namespace QuantConnect.Tests.Algorithm
                     new TestCaseData(Symbol.Create("CustomData", SecurityType.Base, Market.Binance), null),
                     new TestCaseData(Symbol.Create("CustomData2", SecurityType.Base, Market.COMEX), null)
                 };
+
+                foreach (var market in Market.SupportedMarkets())
+                {
+                    foreach (var kvp in SymbolPropertiesDatabase.FromDataFolder().GetSymbolPropertiesList(market))
+                    {
+                        var securityDatabaseKey = kvp.Key;
+                        if (securityDatabaseKey.SecurityType != SecurityType.FutureOption)
+                        {
+                            result.Add(new TestCaseData(Symbol.Create(securityDatabaseKey.Symbol, securityDatabaseKey.SecurityType,
+                                securityDatabaseKey.Market), null));
+                        }
+                    }
+                }
+
+                return result.ToArray();
             }
         }
     }

Tests/Common/Orders/Fees/InteractiveBrokersFeeModelTests.cs

@@ -92,6 +92,39 @@ namespace QuantConnect.Tests.Common.Orders.Fees
             Assert.AreEqual(1000 * 1.85m, fee.Value.Amount);
         }
 
+        [TestCase(false)]
+        [TestCase(true)]
+        public void HongKongFutureFee(bool canonical)
+        {
+            var symbol = Symbols.CreateFutureSymbol(Futures.Indices.HangSeng, SecurityIdentifier.DefaultDate);
+            if (!canonical)
+            {
+                symbol = Symbols.CreateFutureSymbol(Futures.Indices.HangSeng,
+                    FuturesExpiryFunctions.FuturesExpiryFunction(symbol)(new DateTime(2021, 12, 1)));
+            }
+            var entry = MarketHoursDatabase.FromDataFolder().GetEntry(symbol.ID.Market, symbol, symbol.SecurityType);
+            var properties = SymbolPropertiesDatabase.FromDataFolder()
+                .GetSymbolProperties(symbol.ID.Market, symbol, symbol.SecurityType, null);
+            var security = new Future(symbol, entry.ExchangeHours,
+                new Cash(properties.QuoteCurrency, 0, 0),
+                properties,
+                ErrorCurrencyConverter.Instance,
+                RegisteredSecurityDataTypesProvider.Null,
+                new SecurityCache()
+            );
+            security.SetMarketPrice(new Tick(DateTime.UtcNow, security.Symbol, 100, 100));
+
+            var fee = _feeModel.GetOrderFee(
+                new OrderFeeParameters(
+                    security,
+                    new MarketOrder(security.Symbol, 1000, DateTime.UtcNow)
+                )
+            );
+
+            Assert.AreEqual(Currencies.HKD, fee.Value.Currency);
+            Assert.AreEqual(1000 * 40m, fee.Value.Amount);
+        }
+
         [Test]
         public void USAOptionFee()
         {

Tests/Engine/DataFeeds/Enumerators/LiveSubscriptionEnumeratorTests.cs

@@ -15,6 +15,7 @@
 
 using System;
 using NodaTime;
+using System.Linq;
 using NUnit.Framework;
 using QuantConnect.Data;
 using QuantConnect.Packets;
@@ -35,12 +36,12 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
             var canonical = Symbols.Fut_SPY_Feb19_2016.Canonical;
             var dataQueue = new TestDataQueueHandler
             {
-                DataPerSymbol = new Dictionary<Symbol, IEnumerator<BaseData>>
+                DataPerSymbol = new Dictionary<Symbol, List<BaseData>>
                 {
                     { Symbols.Fut_SPY_Feb19_2016,
-                        new List<BaseData>{ new Tick(Time.BeginningOfTime, Symbols.Fut_SPY_Feb19_2016, 1, 1)}.GetEnumerator() },
+                        new List<BaseData>{ new Tick(Time.BeginningOfTime, Symbols.Fut_SPY_Feb19_2016, 1, 1)} },
                     { Symbols.Fut_SPY_Mar19_2016,
-                        new List<BaseData>{ new Tick(Time.BeginningOfTime, Symbols.Fut_SPY_Mar19_2016, 2, 2)}.GetEnumerator() },
+                        new List<BaseData>{ new Tick(Time.BeginningOfTime, Symbols.Fut_SPY_Mar19_2016, 2, 2)} },
                 }
             };
             var config = new SubscriptionDataConfig(typeof(Tick), canonical, Resolution.Tick,
@@ -67,6 +68,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
             Assert.IsTrue(data.MoveNext());
             Assert.AreEqual(2, (data.Current as Tick).AskPrice);
             Assert.AreEqual(canonical, (data.Current as Tick).Symbol);
+            Assert.AreNotEqual(canonical, dataQueue.DataPerSymbol[Symbols.Fut_SPY_Mar19_2016].Single().Symbol);
 
             Assert.IsFalse(data.MoveNext());
             Assert.IsNull(data.Current);
@@ -81,13 +83,13 @@ namespace QuantConnect.Tests.Engine.DataFeeds.Enumerators
         {
             public bool IsConnected => true;
 
-            public Dictionary<Symbol, IEnumerator<BaseData>> DataPerSymbol;
+            public Dictionary<Symbol, List<BaseData>> DataPerSymbol;
 
             public IEnumerator<BaseData> Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler)
             {
-                if (DataPerSymbol.TryGetValue(dataConfig.Symbol, out var enumerator))
+                if (DataPerSymbol.TryGetValue(dataConfig.Symbol, out var baseDatas))
                 {
-                    return enumerator;
+                    return baseDatas.GetEnumerator();
                 }
                 throw new Exception($"Failed to find a data enumerator for symbol {dataConfig.Symbol}!");
             }

Tests/Engine/DataProviders/ApiDataProviderTests.cs

@@ -80,7 +80,7 @@ namespace QuantConnect.Tests.Engine.DataProviders
         [TestCase("forex/oanda/minute/eurusd/20140501_quote.zip", 10, false)]       // Date based Forex minute data
         [TestCase("forex/oanda/second/eurusd/20140501_quote.zip", 10, false)]       // Date based Forex second data
         [TestCase("forex/oanda/tick/eurusd/20140501_quote.zip", 10, false)]         // Date based Forex tick data
-        // Future Cases ** All False because Unsupported
+        // Price Future Cases False because Unsupported
         [TestCase("future/cboe/margins/VX.csv", 0, false)]                           // Fresh Margins data
         [TestCase("future/cboe/margins/VX.csv", 10, false)]                          // Stale Margins data
         [TestCase("future/comex/minute/gc/20131007_openinterest.zip", 10, false)]   // Date based minute data
@@ -127,9 +127,12 @@ namespace QuantConnect.Tests.Engine.DataProviders
         [TestCase("option/usa/minute/aapl/20100603_quote_american.zip")]
         // Forex
         [TestCase("forex/oanda/minute/eurusd/20020516_quote.zip")]
-        // Futures          * False because unsupported
+        // Price Futures          * False because unsupported
         [TestCase("future/cbot/minute/zs/20090501_trade.zip", false)]
         [TestCase("future/sgx/margins/IN.csv", false)]
+        // Auxiliary Data Future Cases true
+        [TestCase("future/comex/map_files/map_files_20211225.zip", true)]
+        [TestCase("future/cme/factor_files/factor_files_20211225.zip", true)]
         // Future Options   * False because unsupported
         [TestCase("futureoption/comex/minute/og/20120227/20120105_quote_american.zip", false)]
         // Index            * False because unsupported

Tests/QuantConnect.Tests.csproj

@@ -33,7 +33,7 @@
   </PropertyGroup>
   <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="Accord" Version="3.6.0" />
     <PackageReference Include="Accord.Math" Version="3.6.0" />
     <PackageReference Include="Common.Logging" Version="3.4.1" />

ToolBox/FactorFileGenerator.cs

@@ -298,22 +298,11 @@ namespace QuantConnect.ToolBox
         /// <returns>A list of <see cref="TradeBar"/> read from file</returns>
         private static List<TradeBar> ReadDailyEquityData(string pathForDailyEquityData)
         {
-            using (var zipToOpen = new FileStream(pathForDailyEquityData, FileMode.Open, FileAccess.Read, FileShare.Read))
-            {
-                using (var archive = new ZipArchive(zipToOpen, ZipArchiveMode.Read))
-                {
-                    foreach (var entry in archive.Entries)
-                    {
-                        var parser = new LeanParser();
-                        var stream = entry.Open();
-                        return parser.Parse(pathForDailyEquityData, stream)
-                                     .OrderByDescending(x => x.Time)
-                                     .Select(x => (TradeBar)x)
-                                     .ToList();
-                    }
-                }
-            }
-            return new List<TradeBar>();
+            var dataReader = new LeanDataReader(pathForDailyEquityData);
+            var bars = dataReader.Parse();
+            return bars.OrderByDescending(x => x.Time)
+                         .Select(x => (TradeBar)x)
+                         .ToList();
         }
 
         /// <summary>

ToolBox/QuantConnect.ToolBox.csproj

@@ -40,7 +40,7 @@
     <Compile Include="..\Common\Properties\SharedAssemblyInfo.cs" Link="Properties\SharedAssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.10" />
+    <PackageReference Include="QuantConnect.pythonnet" Version="2.0.11" />
     <PackageReference Include="CoinAPI.WebSocket.V1" Version="1.6.7" />
     <PackageReference Include="Common.Logging" Version="3.4.1" />
     <PackageReference Include="Common.Logging.Core" Version="3.4.1" />