Revert some changes and adds pandas memory allocator

* Updates CI for Arrow
  * Updates DockerfileLeanFoundation
  * Adds Arrow to Python packages tests

.gitignore

@@ -144,7 +144,6 @@ $tf/
 # ReSharper is a .NET coding add-in
 _ReSharper*/
 *.[Rr]e[Ss]harper
-*.DotSettings
 *.DotSettings.user
 
 # JustCode is a .NET coding addin-in

.travis.yml

@@ -5,7 +5,7 @@ mono:
 solution: QuantConnect.Lean.sln
 before_install:
   - export PATH="$HOME/miniconda3/bin:$PATH"
-  - wget -q https://cdn.quantconnect.com/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh
+  - wget https://cdn.quantconnect.com/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh
   - bash Miniconda3-4.5.12-Linux-x86_64.sh -b
   - rm -rf Miniconda3-4.5.12-Linux-x86_64.sh
   - sudo ln -s $HOME/miniconda3/lib/libpython3.6m.so /usr/lib/libpython3.6m.so
@@ -16,11 +16,12 @@ before_install:
   - conda install -y cython=0.29.15
   - conda install -y scipy=1.4.1
   - conda install -y wrapt=1.12.1
+  - pip install pyarrow==1.0.1
 install:
-  - nuget restore QuantConnect.Lean.sln -v quiet
+  - nuget restore QuantConnect.Lean.sln
   - nuget install NUnit.Runners -Version 3.11.1 -OutputDirectory testrunner
 script:
-  - msbuild /p:Configuration=Release /p:VbcToolExe=vbnc.exe /v:quiet /p:WarningLevel=1 QuantConnect.Lean.sln
-  - mono ./testrunner/NUnit.ConsoleRunner.3.11.1/tools/nunit3-console.exe ./Tests/bin/Release/QuantConnect.Tests.dll --where "cat != TravisExclude" --labels=Off --params:log-handler=ConsoleErrorLogHandler
+  - msbuild /p:Configuration=Release /p:VbcToolExe=vbnc.exe QuantConnect.Lean.sln
+  - mono ./testrunner/NUnit.ConsoleRunner.3.11.1/tools/nunit3-console.exe ./Tests/bin/Release/QuantConnect.Tests.dll --where "cat != TravisExclude" --labels=Off
   - chmod +x ci_build_stubs.sh
   - sudo -E ./ci_build_stubs.sh -d -t -g -p

.vscode/readme.md

@@ -13,7 +13,7 @@ Before anything we need to ensure a few things have been done:
 
 
 1. Get [Visual Studio Code](https://code.visualstudio.com/download)
-    *   Get the Extension [Mono Debug **15.8**](https://marketplace.visualstudio.com/items?itemName=ms-vscode.mono-debug) for C# Debugging
+    *   Get the Extension [Mono Debug](https://marketplace.visualstudio.com/items?itemName=ms-vscode.mono-debug) for C# Debugging
     *   Get the Extension [Python](https://marketplace.visualstudio.com/items?itemName=ms-python.python) for Python Debugging
 
 2. Get [Docker](https://docs.docker.com/get-docker/):
@@ -35,8 +35,7 @@ Before anything we need to ensure a few things have been done:
     *   Download the repo or clone it using: _git clone[ https://github.com/QuantConnect/Lean](https://github.com/QuantConnect/Lean)_
     *   Open the folder using VS Code
 
-**NOTES**: 
-- Mono Extension Version 16 and greater fails to debug the docker container remotely, please install **Version 15.8**. To install an older version from within VS Code go to the extensions tab, search "Mono Debug", and select "Install Another Version...".
+
 <br />
 
 <h1>Develop Algorithms Locally, Run in Container</h1>
@@ -113,12 +112,6 @@ In VS Code click on the debug/run icon on the left toolbar, at the top you shoul
 
 As defaults these are all great! Feel free to change them as needed for your setup.
 
-**NOTE:** VSCode may try and throw errors when launching this way regarding build on `QuantConnect.csx` and `Config.json` these errors can be ignored by selecting "*Debug Anyway*". To stop this error message in the future select "*Remember my choice in user settings*". 
-
-If using C# algorithms ensure that msbuild can build them successfully.  
-
-
-
 <br />
 
 <h3>Option 2</h3>
@@ -201,6 +194,4 @@ _Figure 2: Python Debugger Messages_
 <h1>Common Issues</h1>
 Here we will cover some common issues with setting this up. This section will expand as we get user feedback!
 
-*   Any error messages about building in VSCode that point to comments in JSON. Either select **ignore** or follow steps described [here](https://stackoverflow.com/questions/47834825/in-vs-code-disable-error-comments-are-not-permitted-in-json) to remove the errors entirely.
-*   `Errors exist after running preLaunchTask 'run-docker'`This VSCode error appears to warn you of CSharp errors when trying to use `Debug in Container` select "Debug Anyway" as the errors are false flags for JSON comments as well as `QuantConnect.csx` not finding references. Neither of these will impact your debugging. 
-*   `The container name "/LeanEngine" is already in use by container "****"` This Docker error implies that another instance of lean is already running under the container name /LeanEngine. If this error appears either use Docker Desktop to delete the container or use `docker kill LeanEngine` from the command line.
+*   Error messages about build in VSCode points to comments in JSON. Either select **ignore** or follow steps described [here](https://stackoverflow.com/questions/47834825/in-vs-code-disable-error-comments-are-not-permitted-in-json) to remove the errors entirely.

.vscode/tasks.json

@@ -20,34 +20,6 @@
             },
             "problemMatcher": "$msCompile"
         },
-        {
-            "label": "rebuild",
-            "type": "shell",
-            "command": "msbuild",
-            "args": [
-                "/p:Configuration=Debug",
-                "/p:DebugType=portable",
-                "/t:rebuild",
-            ],
-            "group": "build",
-            "presentation": {
-                "reveal": "silent"
-            },
-            "problemMatcher": "$msCompile"
-        },
-        {
-            "label": "clean",
-            "type": "shell",
-            "command": "msbuild",
-            "args": [
-                "/t:clean",
-            ],
-            "group": "build",
-            "presentation": {
-                "reveal": "silent"
-            },
-            "problemMatcher": "$msCompile"
-        },
         {
             "label": "force build linux",
             "type": "shell",

Algorithm.CSharp/DelistedFutureLiquidateRegressionAlgorithm.cs

@@ -132,7 +132,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "490786648"}
+            {"OrderListHash", "-1252326142"}
         };
     }
 }

Algorithm.CSharp/FutureOptionBuySellCallIntradayRegressionAlgorithm.cs

@@ -124,30 +124,30 @@ namespace QuantConnect.Algorithm.CSharp
         public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
             {"Total Trades", "6"},
-            {"Average Win", "2.93%"},
+            {"Average Win", "2.94%"},
             {"Average Loss", "-4.15%"},
-            {"Compounding Annual Return", "-5.663%"},
-            {"Drawdown", "5.700%"},
-            {"Expectancy", "-0.148"},
-            {"Net Profit", "-2.802%"},
-            {"Sharpe Ratio", "-0.495"},
-            {"Probabilistic Sharpe Ratio", "10.470%"},
+            {"Compounding Annual Return", "-5.601%"},
+            {"Drawdown", "5.600%"},
+            {"Expectancy", "-0.146"},
+            {"Net Profit", "-2.771%"},
+            {"Sharpe Ratio", "-0.49"},
+            {"Probabilistic Sharpe Ratio", "10.583%"},
             {"Loss Rate", "50%"},
             {"Win Rate", "50%"},
-            {"Profit-Loss Ratio", "0.70"},
+            {"Profit-Loss Ratio", "0.71"},
             {"Alpha", "-0.043"},
             {"Beta", "-0.001"},
             {"Annual Standard Deviation", "0.087"},
             {"Annual Variance", "0.008"},
-            {"Information Ratio", "0.957"},
+            {"Information Ratio", "0.96"},
             {"Tracking Error", "0.192"},
-            {"Treynor Ratio", "57.633"},
+            {"Treynor Ratio", "58.394"},
             {"Total Fees", "$14.80"},
             {"Fitness Score", "0.018"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "-0.097"},
-            {"Return Over Maximum Drawdown", "-0.999"},
+            {"Sortino Ratio", "-0.096"},
+            {"Return Over Maximum Drawdown", "-0.993"},
             {"Portfolio Turnover", "0.043"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
@@ -162,7 +162,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "-1863159170"}
+            {"OrderListHash", "-290004562"}
         };
     }
 }

Algorithm.CSharp/FutureOptionCallITMExpiryRegressionAlgorithm.cs

@@ -132,10 +132,12 @@ namespace QuantConnect.Algorithm.CSharp
 
         private void AssertFutureOptionOrderExercise(OrderEvent orderEvent, Security future, Security optionContract)
         {
-            // For unknown reasons, the delisting happens two minutes after the market open. Most likely
-            // stems from the placement of the ProcessDelistedSymbols and HandleDelistedSymbols methods in relation
-            // to the algorithm time update and the brokerage ProcessSynchronousEvents.
-            var expectedLiquidationTimeUtc = new DateTime(2020, 6, 19, 13, 32, 0);
+            // We expect the liquidation to occur on the day of the delisting (while the market is open),
+            // but currently we liquidate at the next market open (AAPL open) which happens to be
+            // at 9:30:00 Eastern Time. For unknown reasons, the delisting happens two minutes after the
+            // market open.
+            // Read more about the issue affecting this test here: https://github.com/QuantConnect/Lean/issues/4980
+            var expectedLiquidationTimeUtc = new DateTime(2020, 6, 22, 13, 32, 0);
 
             if (orderEvent.Direction == OrderDirection.Sell && future.Holdings.Quantity != 0)
             {
@@ -212,13 +214,13 @@ namespace QuantConnect.Algorithm.CSharp
         public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
             {"Total Trades", "3"},
-            {"Average Win", "1.22%"},
+            {"Average Win", "1.25%"},
             {"Average Loss", "-7.42%"},
-            {"Compounding Annual Return", "-12.461%"},
+            {"Compounding Annual Return", "-12.413%"},
             {"Drawdown", "6.300%"},
-            {"Expectancy", "-0.417"},
-            {"Net Profit", "-6.282%"},
-            {"Sharpe Ratio", "-1.324"},
+            {"Expectancy", "-0.416"},
+            {"Net Profit", "-6.257%"},
+            {"Sharpe Ratio", "-1.325"},
             {"Probabilistic Sharpe Ratio", "0.004%"},
             {"Loss Rate", "50%"},
             {"Win Rate", "50%"},
@@ -227,14 +229,14 @@ namespace QuantConnect.Algorithm.CSharp
             {"Beta", "-0.003"},
             {"Annual Standard Deviation", "0.076"},
             {"Annual Variance", "0.006"},
-            {"Information Ratio", "0.671"},
+            {"Information Ratio", "0.673"},
             {"Tracking Error", "0.188"},
-            {"Treynor Ratio", "33.52"},
+            {"Treynor Ratio", "33.559"},
             {"Total Fees", "$7.40"},
             {"Fitness Score", "0.008"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "-0.204"},
+            {"Sortino Ratio", "-0.205"},
             {"Return Over Maximum Drawdown", "-1.983"},
             {"Portfolio Turnover", "0.023"},
             {"Total Insights Generated", "0"},
@@ -250,7 +252,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "1442219241"}
+            {"OrderListHash", "23301049"}
         };
     }
 }

Algorithm.CSharp/FutureOptionCallITMGreeksExpiryRegressionAlgorithm.cs

@@ -166,31 +166,31 @@ namespace QuantConnect.Algorithm.CSharp
         public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
             {"Total Trades", "3"},
-            {"Average Win", "27.44%"},
+            {"Average Win", "28.04%"},
             {"Average Loss", "-62.81%"},
-            {"Compounding Annual Return", "-78.376%"},
-            {"Drawdown", "52.600%"},
-            {"Expectancy", "-0.282"},
-            {"Net Profit", "-52.604%"},
-            {"Sharpe Ratio", "-0.864"},
+            {"Compounding Annual Return", "-78.165%"},
+            {"Drawdown", "52.400%"},
+            {"Expectancy", "-0.277"},
+            {"Net Profit", "-52.379%"},
+            {"Sharpe Ratio", "-0.865"},
             {"Probabilistic Sharpe Ratio", "0.019%"},
             {"Loss Rate", "50%"},
             {"Win Rate", "50%"},
-            {"Profit-Loss Ratio", "0.44"},
-            {"Alpha", "-0.598"},
-            {"Beta", "-0.032"},
-            {"Annual Standard Deviation", "0.684"},
-            {"Annual Variance", "0.467"},
+            {"Profit-Loss Ratio", "0.45"},
+            {"Alpha", "-0.596"},
+            {"Beta", "-0.031"},
+            {"Annual Standard Deviation", "0.681"},
+            {"Annual Variance", "0.463"},
             {"Information Ratio", "-0.514"},
-            {"Tracking Error", "0.706"},
-            {"Treynor Ratio", "18.718"},
+            {"Tracking Error", "0.703"},
+            {"Treynor Ratio", "18.748"},
             {"Total Fees", "$66.60"},
-            {"Fitness Score", "0.158"},
+            {"Fitness Score", "0.157"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
             {"Sortino Ratio", "-0.133"},
-            {"Return Over Maximum Drawdown", "-1.489"},
-            {"Portfolio Turnover", "0.413"},
+            {"Return Over Maximum Drawdown", "-1.492"},
+            {"Portfolio Turnover", "0.411"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
             {"Total Insights Analysis Completed", "0"},
@@ -204,7 +204,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "891799117"}
+            {"OrderListHash", "151392833"}
         };
     }
 }

Algorithm.CSharp/FutureOptionCallOTMExpiryRegressionAlgorithm.cs

@@ -26,13 +26,11 @@ namespace QuantConnect.Algorithm.CSharp
 {
     /// <summary>
     /// This regression algorithm tests Out of The Money (OTM) future option expiry for calls.
-    /// We expect 2 orders from the algorithm, which are:
+    /// We expect 1 order from the algorithm, which are:
     ///
     ///   * Initial entry, buy ES Call Option (expiring OTM)
     ///     - contract expires worthless, not exercised, so never opened a position in the underlying
     ///
-    ///   * Liquidation of worthless ES call option (expiring OTM)
-    ///
     /// Additionally, we test delistings for future options and assert that our
     /// portfolio holdings reflect the orders the algorithm has submitted.
     /// </summary>
@@ -221,7 +219,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "1061918870"}
+            {"OrderListHash", "-1116221764"}
         };
     }
 }

Algorithm.CSharp/FutureOptionPutITMExpiryRegressionAlgorithm.cs

@@ -132,10 +132,12 @@ namespace QuantConnect.Algorithm.CSharp
 
         private void AssertFutureOptionOrderExercise(OrderEvent orderEvent, Security future, Security optionContract)
         {
-            // For unknown reasons, the delisting happens two minutes after the market open. Most likely
-            // stems from the placement of the ProcessDelistedSymbols and HandleDelistedSymbols methods in relation
-            // to the algorithm time update and the brokerage ProcessSynchronousEvents.
-            var expectedLiquidationTimeUtc = new DateTime(2020, 6, 19, 13, 32, 0);
+            // We expect the liquidation to occur on the day of the delisting (while the market is open),
+            // but currently we liquidate at the next market open (AAPL open) which happens to be
+            // at 9:30:00 Eastern Time. For unknown reasons, the delisting happens two minutes after the
+            // market open.
+            // Read more about the issue affecting this test here: https://github.com/QuantConnect/Lean/issues/4980
+            var expectedLiquidationTimeUtc = new DateTime(2020, 6, 22, 13, 32, 0);
 
             if (orderEvent.Direction == OrderDirection.Buy && future.Holdings.Quantity != 0)
             {
@@ -212,24 +214,24 @@ namespace QuantConnect.Algorithm.CSharp
         public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
             {"Total Trades", "3"},
-            {"Average Win", "4.15%"},
+            {"Average Win", "4.18%"},
             {"Average Loss", "-8.27%"},
-            {"Compounding Annual Return", "-8.928%"},
-            {"Drawdown", "4.500%"},
-            {"Expectancy", "-0.249"},
-            {"Net Profit", "-4.457%"},
-            {"Sharpe Ratio", "-1.389"},
+            {"Compounding Annual Return", "-8.879%"},
+            {"Drawdown", "4.400%"},
+            {"Expectancy", "-0.247"},
+            {"Net Profit", "-4.432%"},
+            {"Sharpe Ratio", "-1.391"},
             {"Probabilistic Sharpe Ratio", "0.002%"},
             {"Loss Rate", "50%"},
             {"Win Rate", "50%"},
-            {"Profit-Loss Ratio", "0.50"},
+            {"Profit-Loss Ratio", "0.51"},
             {"Alpha", "-0.073"},
             {"Beta", "-0.002"},
             {"Annual Standard Deviation", "0.052"},
             {"Annual Variance", "0.003"},
-            {"Information Ratio", "0.861"},
+            {"Information Ratio", "0.863"},
             {"Tracking Error", "0.179"},
-            {"Treynor Ratio", "38.365"},
+            {"Treynor Ratio", "38.46"},
             {"Total Fees", "$7.40"},
             {"Fitness Score", "0.008"},
             {"Kelly Criterion Estimate", "0"},
@@ -250,7 +252,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "-1705374528"}
+            {"OrderListHash", "-675079082"}
         };
     }
 }

Algorithm.CSharp/FutureOptionPutOTMExpiryRegressionAlgorithm.cs

@@ -26,13 +26,11 @@ namespace QuantConnect.Algorithm.CSharp
 {
     /// <summary>
     /// This regression algorithm tests Out of The Money (OTM) future option expiry for puts.
-    /// We expect 2 orders from the algorithm, which are:
+    /// We expect 1 order from the algorithm, which are:
     ///
     ///   * Initial entry, buy ES Put Option (expiring OTM)
     ///     - contract expires worthless, not exercised, so never opened a position in the underlying
     ///
-    ///   * Liquidation of worthless ES Put OTM contract
-    ///
     /// Additionally, we test delistings for future options and assert that our
     /// portfolio holdings reflect the orders the algorithm has submitted.
     /// </summary>
@@ -220,7 +218,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "-312857564"}
+            {"OrderListHash", "515984318"}
         };
     }
 }

Algorithm.CSharp/FutureOptionShortCallITMExpiryRegressionAlgorithm.cs

@@ -194,29 +194,29 @@ namespace QuantConnect.Algorithm.CSharp
         {
             {"Total Trades", "3"},
             {"Average Win", "10.05%"},
-            {"Average Loss", "-5.63%"},
-            {"Compounding Annual Return", "8.067%"},
+            {"Average Loss", "-5.60%"},
+            {"Compounding Annual Return", "8.121%"},
             {"Drawdown", "0.500%"},
-            {"Expectancy", "0.393"},
-            {"Net Profit", "3.855%"},
-            {"Sharpe Ratio", "1.191"},
-            {"Probabilistic Sharpe Ratio", "58.149%"},
+            {"Expectancy", "0.396"},
+            {"Net Profit", "3.880%"},
+            {"Sharpe Ratio", "1.192"},
+            {"Probabilistic Sharpe Ratio", "58.203%"},
             {"Loss Rate", "50%"},
             {"Win Rate", "50%"},
             {"Profit-Loss Ratio", "1.79"},
-            {"Alpha", "0.068"},
+            {"Alpha", "0.069"},
             {"Beta", "0.003"},
             {"Annual Standard Deviation", "0.057"},
             {"Annual Variance", "0.003"},
-            {"Information Ratio", "1.64"},
+            {"Information Ratio", "1.641"},
             {"Tracking Error", "0.18"},
-            {"Treynor Ratio", "22.061"},
+            {"Treynor Ratio", "22.101"},
             {"Total Fees", "$7.40"},
             {"Fitness Score", "0.021"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
             {"Sortino Ratio", "79228162514264337593543950335"},
-            {"Return Over Maximum Drawdown", "17.142"},
+            {"Return Over Maximum Drawdown", "17.255"},
             {"Portfolio Turnover", "0.021"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
@@ -231,7 +231,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "-991138464"}
+            {"OrderListHash", "1118389718"}
         };
     }
 }

Algorithm.CSharp/FutureOptionShortCallOTMExpiryRegressionAlgorithm.cs

@@ -26,13 +26,11 @@ namespace QuantConnect.Algorithm.CSharp
 {
     /// <summary>
     /// This regression algorithm tests Out of The Money (OTM) future option expiry for short calls.
-    /// We expect 2 orders from the algorithm, which are:
+    /// We expect 1 order from the algorithm, which are:
     ///
     ///   * Initial entry, sell ES Call Option (expiring OTM)
     ///     - Profit the option premium, since the option was not assigned.
     ///
-    ///   * Liquidation of ES call OTM contract on the last trade date
-    ///
     /// Additionally, we test delistings for future options and assert that our
     /// portfolio holdings reflect the orders the algorithm has submitted.
     /// </summary>
@@ -214,7 +212,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "1847291350"}
+            {"OrderListHash", "1364902860"}
         };
     }
 }

Algorithm.CSharp/FutureOptionShortPutITMExpiryRegressionAlgorithm.cs

@@ -191,29 +191,29 @@ namespace QuantConnect.Algorithm.CSharp
         {
             {"Total Trades", "3"},
             {"Average Win", "10.18%"},
-            {"Average Loss", "-8.05%"},
-            {"Compounding Annual Return", "2.721%"},
+            {"Average Loss", "-8.02%"},
+            {"Compounding Annual Return", "2.773%"},
             {"Drawdown", "0.500%"},
-            {"Expectancy", "0.133"},
-            {"Net Profit", "1.318%"},
-            {"Sharpe Ratio", "0.934"},
-            {"Probabilistic Sharpe Ratio", "46.618%"},
+            {"Expectancy", "0.135"},
+            {"Net Profit", "1.343%"},
+            {"Sharpe Ratio", "0.939"},
+            {"Probabilistic Sharpe Ratio", "46.842%"},
             {"Loss Rate", "50%"},
             {"Win Rate", "50%"},
             {"Profit-Loss Ratio", "1.27"},
             {"Alpha", "0.023"},
             {"Beta", "0.002"},
-            {"Annual Standard Deviation", "0.024"},
+            {"Annual Standard Deviation", "0.025"},
             {"Annual Variance", "0.001"},
-            {"Information Ratio", "1.448"},
+            {"Information Ratio", "1.45"},
             {"Tracking Error", "0.173"},
-            {"Treynor Ratio", "14.482"},
+            {"Treynor Ratio", "14.62"},
             {"Total Fees", "$7.40"},
             {"Fitness Score", "0.021"},
             {"Kelly Criterion Estimate", "0"},
             {"Kelly Criterion Probability Value", "0"},
             {"Sortino Ratio", "79228162514264337593543950335"},
-            {"Return Over Maximum Drawdown", "5.706"},
+            {"Return Over Maximum Drawdown", "5.815"},
             {"Portfolio Turnover", "0.022"},
             {"Total Insights Generated", "0"},
             {"Total Insights Closed", "0"},
@@ -228,7 +228,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "777632049"}
+            {"OrderListHash", "980293281"}
         };
     }
 }

Algorithm.CSharp/FutureOptionShortPutOTMExpiryRegressionAlgorithm.cs

@@ -26,13 +26,11 @@ namespace QuantConnect.Algorithm.CSharp
 {
     /// <summary>
     /// This regression algorithm tests Out of The Money (OTM) future option expiry for short puts.
-    /// We expect 2 order from the algorithm, which are:
+    /// We expect 1 order from the algorithm, which are:
     ///
     ///   * Initial entry, sell ES Put Option (expiring OTM)
     ///     - Profit the option premium, since the option was not assigned.
     ///
-    ///   * Liquidation of ES put OTM contract on the last trade date
-    ///
     /// Additionally, we test delistings for future options and assert that our
     /// portfolio holdings reflect the orders the algorithm has submitted.
     /// </summary>
@@ -213,7 +211,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "1003680014"}
+            {"OrderListHash", "-418839052"}
         };
     }
 }

Algorithm.CSharp/FuturesAndFuturesOptionsExpiryTimeAndLiquidationRegressionAlgorithm.cs

@@ -1,202 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using System.Collections.Generic;
-using QuantConnect.Data;
-using QuantConnect.Interfaces;
-using QuantConnect.Orders;
-using QuantConnect.Securities;
-
-namespace QuantConnect.Algorithm.CSharp
-{
-    /// <summary>
-    /// Tests delistings for Futures and Futures Options to ensure that they are delisted at the expected times.
-    /// </summary>
-    public class FuturesAndFuturesOptionsExpiryTimeAndLiquidationRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
-    {
-        private bool _invested;
-        private int _liquidated;
-        private int _delistingsReceived;
-
-        private Symbol _esFuture;
-        private Symbol _esFutureOption;
-
-        private readonly DateTime _expectedExpiryWarningTime = new DateTime(2020, 6, 19);
-        private readonly DateTime _expectedExpiryDelistingTime = new DateTime(2020, 6, 20);
-        private readonly DateTime _expectedLiquidationTime = new DateTime(2020, 6, 19, 9, 32, 0);
-
-        public override void Initialize()
-        {
-            SetStartDate(2020, 1, 5);
-            SetEndDate(2020, 12, 1);
-            SetCash(100000);
-
-            // To ensure that the expiry liquidations are ran for the Futures and FOPs, we
-            // add AAPL to pump a data point through on liquidation date so that the liquidation goes through
-            // at AAPL market open. See issue for more details: https://github.com/QuantConnect/Lean/issues/4872
-            AddEquity("AAPL", Resolution.Daily);
-
-            var es = QuantConnect.Symbol.CreateFuture(
-                Futures.Indices.SP500EMini,
-                Market.CME,
-                new DateTime(2020, 6, 19));
-
-            var esOption = QuantConnect.Symbol.CreateOption(
-                es,
-                Market.CME,
-                OptionStyle.American,
-                OptionRight.Put,
-                3400m,
-                new DateTime(2020, 6, 19));
-
-            _esFuture = AddFutureContract(es, Resolution.Minute).Symbol;
-            _esFutureOption = AddFutureOptionContract(esOption, Resolution.Minute).Symbol;
-        }
-
-        public override void OnData(Slice data)
-        {
-            foreach (var delisting in data.Delistings.Values)
-            {
-                // Two warnings and two delisted events should be received for a grand total of 4 events.
-                _delistingsReceived++;
-
-                if (delisting.Type == DelistingType.Warning &&
-                    delisting.Time != _expectedExpiryWarningTime)
-                {
-                    throw new Exception($"Expiry warning with time {delisting.Time} but is expected to be {_expectedExpiryWarningTime}");
-                }
-                if (delisting.Type == DelistingType.Warning && delisting.Time != Time.Date)
-                {
-                    throw new Exception($"Delisting warning received at an unexpected date: {Time} - expected {delisting.Time}");
-                }
-                if (delisting.Type == DelistingType.Delisted &&
-                    delisting.Time != _expectedExpiryDelistingTime)
-                {
-                    throw new Exception($"Delisting occurred at unexpected time: {delisting.Time} - expected: {_expectedExpiryDelistingTime}");
-                }
-                if (delisting.Type == DelistingType.Delisted &&
-                    delisting.Time != Time.Date)
-                {
-                    throw new Exception($"Delisting notice received at an unexpected date: {Time} - expected {delisting.Time}");
-                }
-            }
-
-            if (!_invested &&
-                (data.Bars.ContainsKey(_esFuture) || data.QuoteBars.ContainsKey(_esFuture)) &&
-                (data.Bars.ContainsKey(_esFutureOption) || data.QuoteBars.ContainsKey(_esFutureOption)))
-            {
-                _invested = true;
-
-                MarketOrder(_esFuture, 1);
-                MarketOrder(_esFutureOption, 1);
-            }
-        }
-
-        public override void OnOrderEvent(OrderEvent orderEvent)
-        {
-            if (orderEvent.Direction != OrderDirection.Sell || orderEvent.Status != OrderStatus.Filled)
-            {
-                return;
-            }
-
-            // * Future Liquidation
-            // * Future Option Exercise
-            // * Underlying Future Liquidation
-            _liquidated++;
-            if (orderEvent.Symbol.SecurityType == SecurityType.FutureOption && _expectedLiquidationTime != Time)
-            {
-                throw new Exception($"Expected to liquidate option {orderEvent.Symbol} at {_expectedLiquidationTime}, instead liquidated at {Time}");
-            }
-            if (orderEvent.Symbol.SecurityType == SecurityType.Future && _expectedLiquidationTime.AddMinutes(-1) != Time && _expectedLiquidationTime != Time)
-            {
-                throw new Exception($"Expected to liquidate future {orderEvent.Symbol} at {_expectedLiquidationTime} (+1 minute), instead liquidated at {Time}");
-            }
-        }
-
-        public override void OnEndOfAlgorithm()
-        {
-            if (!_invested)
-            {
-                throw new Exception("Never invested in ES futures and FOPs");
-            }
-            if (_delistingsReceived != 4)
-            {
-                throw new Exception($"Expected 4 delisting events received, found: {_delistingsReceived}");
-            }
-            if (_liquidated != 3)
-            {
-                throw new Exception($"Expected 3 liquidation events, found {_liquidated}");
-            }
-        }
-
-        /// <summary>
-        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
-        /// </summary>
-        public bool CanRunLocally { get; } = true;
-
-        /// <summary>
-        /// This is used by the regression test system to indicate which languages this algorithm is written in.
-        /// </summary>
-        public Language[] Languages { get; } = { Language.CSharp, Language.Python };
-
-        /// <summary>
-        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
-        /// </summary>
-        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
-        {
-            {"Total Trades", "4"},
-            {"Average Win", "0.13%"},
-            {"Average Loss", "-11.33%"},
-            {"Compounding Annual Return", "-2.634%"},
-            {"Drawdown", "2.400%"},
-            {"Expectancy", "-0.494"},
-            {"Net Profit", "-2.399%"},
-            {"Sharpe Ratio", "-0.933"},
-            {"Probabilistic Sharpe Ratio", "0.003%"},
-            {"Loss Rate", "50%"},
-            {"Win Rate", "50%"},
-            {"Profit-Loss Ratio", "0.01"},
-            {"Alpha", "-0.017"},
-            {"Beta", "-0.001"},
-            {"Annual Standard Deviation", "0.018"},
-            {"Annual Variance", "0"},
-            {"Information Ratio", "0.886"},
-            {"Tracking Error", "0.127"},
-            {"Treynor Ratio", "30.534"},
-            {"Total Fees", "$9.25"},
-            {"Fitness Score", "0.007"},
-            {"Kelly Criterion Estimate", "0"},
-            {"Kelly Criterion Probability Value", "0"},
-            {"Sortino Ratio", "-0.177"},
-            {"Return Over Maximum Drawdown", "-1.098"},
-            {"Portfolio Turnover", "0.018"},
-            {"Total Insights Generated", "0"},
-            {"Total Insights Closed", "0"},
-            {"Total Insights Analysis Completed", "0"},
-            {"Long Insight Count", "0"},
-            {"Short Insight Count", "0"},
-            {"Long/Short Ratio", "100%"},
-            {"Estimated Monthly Alpha Value", "$0"},
-            {"Total Accumulated Estimated Alpha Value", "$0"},
-            {"Mean Population Estimated Insight Value", "$0"},
-            {"Mean Population Direction", "0%"},
-            {"Mean Population Magnitude", "0%"},
-            {"Rolling Averaged Population Direction", "0%"},
-            {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "542517089"}
-        };
-    }
-}

Algorithm.CSharp/QuantConnect.Algorithm.CSharp.csproj

@@ -148,7 +148,6 @@
     <Compile Include="AddOptionContractExpiresRegressionAlgorithm.cs" />
     <Compile Include="AltData\QuiverWallStreetBetsDataAlgorithm.cs" />
     <Compile Include="FutureOptionCallITMGreeksExpiryRegressionAlgorithm.cs" />
-    <Compile Include="FuturesAndFuturesOptionsExpiryTimeAndLiquidationRegressionAlgorithm.cs" />
     <Compile Include="OnOrderEventExceptionRegression.cs" />
     <Compile Include="FutureOptionCallITMExpiryRegressionAlgorithm.cs" />
     <Compile Include="FutureOptionCallOTMExpiryRegressionAlgorithm.cs" />

Algorithm.Python/FutureOptionCallITMExpiryRegressionAlgorithm.py

@@ -107,7 +107,7 @@ class FutureOptionCallITMExpiryRegressionAlgorithm(QCAlgorithm):
         # at 9:30:00 Eastern Time. For unknown reasons, the delisting happens two minutes after the
         # market open.
         # Read more about the issue affecting this test here: https://github.com/QuantConnect/Lean/issues/4980
-        expectedLiquidationTimeUtc = datetime(2020, 6, 19, 13, 32, 0)
+        expectedLiquidationTimeUtc = datetime(2020, 6, 22, 13, 32, 0)
 
         if orderEvent.Direction == OrderDirection.Sell and future.Holdings.Quantity != 0:
             # We expect the contract to have been liquidated immediately

Algorithm.Python/FutureOptionCallOTMExpiryRegressionAlgorithm.py

@@ -28,13 +28,11 @@ from QuantConnect import Market
 
 ### <summary>
 ### This regression algorithm tests Out of The Money (OTM) future option expiry for calls.
-### We expect 2 orders from the algorithm, which are:
+### We expect 1 order from the algorithm, which are:
 ###
 ###   * Initial entry, buy ES Call Option (expiring OTM)
 ###     - contract expires worthless, not exercised, so never opened a position in the underlying
 ###
-###   * Liquidation of worthless ES call option (expiring OTM)
-###
 ### Additionally, we test delistings for future options and assert that our
 ### portfolio holdings reflect the orders the algorithm has submitted.
 ### </summary>

Algorithm.Python/FutureOptionPutITMExpiryRegressionAlgorithm.py

@@ -106,7 +106,7 @@ class FutureOptionPutITMExpiryRegressionAlgorithm(QCAlgorithm):
         # at 9:30:00 Eastern Time. For unknown reasons, the delisting happens two minutes after the
         # market open.
         # Read more about the issue affecting this test here: https://github.com/QuantConnect/Lean/issues/4980
-        expectedLiquidationTimeUtc = datetime(2020, 6, 19, 13, 32, 0)
+        expectedLiquidationTimeUtc = datetime(2020, 6, 22, 13, 32, 0)
 
         if orderEvent.Direction == OrderDirection.Buy and future.Holdings.Quantity != 0:
             # We expect the contract to have been liquidated immediately

Algorithm.Python/FutureOptionPutOTMExpiryRegressionAlgorithm.py

@@ -28,12 +28,10 @@ from QuantConnect import Market
 
 ### <summary>
 ### This regression algorithm tests Out of The Money (OTM) future option expiry for puts.
-### We expect 2 orders from the algorithm, which are:
+### We expect 1 order from the algorithm, which are:
 ###
 ###   * Initial entry, buy ES Put Option (expiring OTM)
 ###     - contract expires worthless, not exercised, so never opened a position in the underlying
-###   
-###   * Liquidation of worthless ES Put OTM contract 
 ###
 ### Additionally, we test delistings for future options and assert that our
 ### portfolio holdings reflect the orders the algorithm has submitted.

Algorithm.Python/FutureOptionShortCallOTMExpiryRegressionAlgorithm.py

@@ -28,13 +28,11 @@ from QuantConnect import Market
 
 ### <summary>
 ### This regression algorithm tests Out of The Money (OTM) future option expiry for short calls.
-### We expect 2 orders from the algorithm, which are:
+### We expect 1 order from the algorithm, which are:
 ###
 ###   * Initial entry, sell ES Call Option (expiring OTM)
 ###     - Profit the option premium, since the option was not assigned.
 ###
-###   * Liquidation of ES call OTM contract on the last trade date
-###
 ### Additionally, we test delistings for future options and assert that our
 ### portfolio holdings reflect the orders the algorithm has submitted.
 ### </summary>

Algorithm.Python/FutureOptionShortPutOTMExpiryRegressionAlgorithm.py

@@ -28,13 +28,11 @@ from QuantConnect import Market
 
 ### <summary>
 ### This regression algorithm tests Out of The Money (OTM) future option expiry for short puts.
-### We expect 2 orders from the algorithm, which are:
+### We expect 1 order from the algorithm, which are:
 ###
 ###   * Initial entry, sell ES Put Option (expiring OTM)
 ###     - Profit the option premium, since the option was not assigned.
 ###
-###   * Liquidation of ES put OTM contract on the last trade date
-###
 ### Additionally, we test delistings for future options and assert that our
 ### portfolio holdings reflect the orders the algorithm has submitted.
 ### </summary>

Algorithm.Python/FuturesAndFuturesOptionsExpiryTimeAndLiquidationRegressionAlgorithm.py

@@ -1,100 +0,0 @@
-from datetime import datetime, timedelta
-
-from QuantConnect.Algorithm import *
-from QuantConnect.Data import *
-from QuantConnect.Data.Market import *
-from QuantConnect.Orders import *
-from QuantConnect import *
-
-### <summary>
-### Tests delistings for Futures and Futures Options to ensure that they are delisted at the expected times.
-### </summary>
-class FuturesAndFuturesOptionsExpiryTimeAndLiquidationRegressionAlgorithm(QCAlgorithm):
-    def Initialize(self):
-        self.invested = False
-        self.liquidated = 0
-        self.delistingsReceived = 0
-
-        self.expectedExpiryWarningTime = datetime(2020, 6, 19)
-        self.expectedExpiryDelistingTime = datetime(2020, 6, 20)
-        self.expectedLiquidationTime = datetime(2020, 6, 19, 9, 32, 0)
-
-        self.SetStartDate(2020, 1, 5)
-        self.SetEndDate(2020, 12, 1)
-        self.SetCash(100000)
-
-        # To ensure that the expiry liquidations are ran for the Futures and FOPs, we
-        # add AAPL to pump a data point through on liquidation date so that the liquidation goes through
-        # at AAPL market open. See issue for more details: https://github.com/QuantConnect/Lean/issues/4872
-        self.AddEquity("AAPL", Resolution.Daily)
-
-        es = Symbol.CreateFuture(
-            "ES",
-            Market.CME,
-            datetime(2020, 6, 19)
-        )
-
-        esOption = Symbol.CreateOption(
-            es,
-            Market.CME,
-            OptionStyle.American,
-            OptionRight.Put,
-            3400.0,
-            datetime(2020, 6, 19)
-        )
-
-        self.esFuture = self.AddFutureContract(es, Resolution.Minute).Symbol
-        self.esFutureOption = self.AddFutureOptionContract(esOption, Resolution.Minute).Symbol
-
-    def OnData(self, data: Slice):
-        for delisting in data.Delistings.Values:
-            self.delistingsReceived += 1
-
-            if delisting.Type == DelistingType.Warning and delisting.Time != self.expectedExpiryWarningTime:
-                raise AssertionError(f"Expiry warning with time {delisting.Time} but is expected to be {self.expectedExpiryWarningTime}")
-
-            if delisting.Type == DelistingType.Warning and delisting.Time != datetime(self.Time.year, self.Time.month, self.Time.day):
-                raise AssertionError(f"Delisting warning received at an unexpected date: {self.Time} - expected {delisting.Time}")
-
-            if delisting.Type == DelistingType.Delisted and delisting.Time != self.expectedExpiryDelistingTime:
-                raise AssertionError(f"Delisting occurred at unexpected time: {delisting.Time} - expected: {self.expectedExpiryDelistingTime}")
-
-            if delisting.Type == DelistingType.Delisted and delisting.Time != datetime(self.Time.year, self.Time.month, self.Time.day):
-                raise AssertionError(f"Delisting notice received at an unexpected date: {self.Time} - expected {delisting.Time}")
-
-        if not self.invested and \
-            (self.esFuture in data.Bars or self.esFuture in data.QuoteBars) and \
-            (self.esFutureOption in data.Bars or self.esFutureOption in data.QuoteBars):
-
-            self.invested = True
-
-            self.MarketOrder(self.esFuture, 1)
-            self.MarketOrder(self.esFutureOption, 1)
-
-    def OnOrderEvent(self, orderEvent: OrderEvent):
-        if orderEvent.Direction != OrderDirection.Sell or orderEvent.Status != OrderStatus.Filled:
-            return
-
-        # * Future Liquidation
-        # * Future Option Exercise
-        # * Underlying Future Liquidation
-        self.liquidated += 1
-        if orderEvent.Symbol.SecurityType == SecurityType.FutureOption and self.expectedLiquidationTime != self.Time:
-            raise AssertionError(f"Expected to liquidate option {orderEvent.Symbol} at {self.expectedLiquidationTime}, instead liquidated at {self.Time}")
-
-        if orderEvent.Symbol.SecurityType == SecurityType.Future and \
-            (self.expectedLiquidationTime - timedelta(minutes=1)) != self.Time and \
-            self.expectedLiquidationTime != self.Time:
-
-            raise AssertionError(f"Expected to liquidate future {orderEvent.Symbol} at {self.expectedLiquidationTime} (+1 minute), instead liquidated at {self.Time}")
-
-
-    def OnEndOfAlgorithm(self):
-        if not self.invested:
-            raise AssertionError("Never invested in ES futures and FOPs")
-
-        if self.delistingsReceived != 4:
-            raise AssertionError(f"Expected 4 delisting events received, found: {self.delistingsReceived}")
-
-        if self.liquidated != 3:
-            raise AssertionError(f"Expected 3 liquidation events, found {self.liquidated}")

Algorithm.Python/QuantConnect.Algorithm.Python.csproj

@@ -109,7 +109,6 @@
     <Content Include="FutureOptionShortCallOTMExpiryRegressionAlgorithm.py" />
     <Content Include="FutureOptionShortPutITMExpiryRegressionAlgorithm.py" />
     <Content Include="FutureOptionShortPutOTMExpiryRegressionAlgorithm.py" />
-    <Content Include="FuturesAndFuturesOptionsExpiryTimeAndLiquidationRegressionAlgorithm.py" />
     <Content Include="KerasNeuralNetworkAlgorithm.py" />
     <Content Include="CustomDataUsingMapFileRegressionAlgorithm.py" />
     <Content Include="LiquidETFUniverseFrameworkAlgorithm.py" />

Algorithm.Python/readme.md

@@ -33,20 +33,22 @@ Before we enable python support, follow the [installation instructions](https://
         - Value of the variable: python installation path.
 4. Install [pandas=0.25.3](https://pandas.pydata.org/) and its [dependencies](https://pandas.pydata.org/pandas-docs/stable/install.html#dependencies).
 5. Install [wrapt=1.11.2](https://pypi.org/project/wrapt/) module.
-6. Reboot computer to ensure changes are propogated.
+6. Install [pyarrow=1.0.1](https://arrow.apache.org/install/) module.
+7. Reboot computer to ensure changes are propagated.
 
 #### [macOS](https://github.com/QuantConnect/Lean#macos)
 
 1. Use the macOS x86-64 package installer from [Anaconda](https://repo.anaconda.com/archive/Anaconda3-5.2.0-MacOSX-x86_64.pkg) and follow "[Installing on macOS](https://docs.anaconda.com/anaconda/install/mac-os)" instructions from Anaconda documentation page.
 2. Install [pandas=0.25.3](https://pandas.pydata.org/) and its [dependencies](https://pandas.pydata.org/pandas-docs/stable/install.html#dependencies).
 3. Install [wrapt=1.11.2](https://pypi.org/project/wrapt/) module.
+4. Install [pyarrow=1.0.1](https://arrow.apache.org/install/) module.
 
-*Note:* If you encounter the "System.DllNotFoundException: python3.6m" runtime error when running Python algorithms, or generating reports, on macOS:
+*Note:* If you encounter the "System.DllNotFoundException: python3.6m" runtime error when running Python algorithms on macOS:
 1. Find `libpython3.6m.dylib` in your Python installation folder. If you installed Python with Anaconda, it may be found at
     ```
     /Users/{your_user_name}/anaconda3/lib/libpython3.6m.dylib
     ```
-2. Open `Lean/Common/Python/Python.Runtime.dll.config`, add the following text under `<configuration> ... </configuration>` and save:
+2. Open `Lean/Launcher/bin/Debug/Python.Runtime.dll.config`, add the following text under `<configuration> ... </configuration>` and save:
     ```
         <dllmap dll="python3.6m" target="{the path in step 1 including libpython3.6m.dylib}" os="osx"/>
     ```
@@ -64,6 +66,7 @@ conda update -y python conda pip
 conda install -y cython=0.29.11
 conda install -y pandas=0.25.3
 conda install -y wrapt=1.11.2
+pip install pyarrow==1.0.1
 ```
 
 *Note 1:* There is a [known issue](https://github.com/pythonnet/pythonnet/issues/609) with python 3.6.5 that prevents pythonnet installation, please upgrade python to version 3.6.8:
@@ -81,7 +84,7 @@ conda install -y wrapt=1.11.2
    conda create -n qc_environment python=3.6.8 cython=0.29.11 pandas=0.25.3 wrapt=1.11.2
 
    ```
-2. Open `Lean/Common/Python/Python.Runtime.dll.config`, add the following text under `<configuration> ... </configuration>` and save:
+2. Open `Lean/Launcher/bin/Debug/Python.Runtime.dll.config`, add the following text under `<configuration> ... </configuration>` and save:
     ```
         <dllmap dll="python3.6m" target="{the path in step 1 including libpython3.6m.so}" os="linux"/>
     ```

Algorithm/QCAlgorithm.Framework.cs

@@ -14,7 +14,6 @@
 */
 
 using System;
-using System.Collections.Generic;
 using System.Linq;
 using QuantConnect.Algorithm.Framework.Alphas;
 using QuantConnect.Algorithm.Framework.Alphas.Analysis;
@@ -32,7 +31,6 @@ namespace QuantConnect.Algorithm
     {
         private readonly ISecurityValuesProvider _securityValuesProvider;
         private bool _isEmitWarmupInsightWarningSent;
-        private bool _isEmitDelistedInsightWarningSent;
 
         /// <summary>
         /// Enables additional logging of framework models including:
@@ -149,8 +147,7 @@ namespace QuantConnect.Algorithm
             // only fire insights generated event if we actually have insights
             if (insights.Length != 0)
             {
-                insights = InitializeInsights(insights);
-                OnInsightsGenerated(insights);
+                OnInsightsGenerated(insights.Select(InitializeInsightFields));
             }
 
             ProcessInsights(insights);
@@ -229,7 +226,7 @@ namespace QuantConnect.Algorithm
                     Log($"{Time}: RISK ADJUSTED TARGETS: {string.Join(" | ", riskAdjustedTargets.Select(t => t.ToString()).OrderBy(t => t))}");
                 }
             }
-
+            
             Execution.Execute(this, riskAdjustedTargets);
         }
 
@@ -385,8 +382,7 @@ namespace QuantConnect.Algorithm
                 return;
             }
 
-            insights = InitializeInsights(insights);
-            OnInsightsGenerated(insights);
+            OnInsightsGenerated(insights.Select(InitializeInsightFields));
             ProcessInsights(insights);
         }
 
@@ -398,52 +394,7 @@ namespace QuantConnect.Algorithm
         /// <param name="insight">The insight to be emitted</param>
         public void EmitInsights(Insight insight)
         {
-            EmitInsights(new[] { insight });
-        }
-
-        /// <summary>
-        /// Helper method used to validate insights and prepare them to be emitted
-        /// </summary>
-        /// <param name="insights">insights preparing to be emitted</param>
-        /// <returns>Validated insights</returns>
-        private Insight[] InitializeInsights(Insight[] insights)
-        {
-            List<Insight> validInsights = null;
-            for (var i = 0; i < insights.Length; i++)
-            {
-                if (Securities[insights[i].Symbol].IsDelisted)
-                {
-                    if (!_isEmitDelistedInsightWarningSent)
-                    {
-                        Error($"QCAlgorithm.EmitInsights(): Warning: cannot emit insights for delisted securities, these will be discarded");
-                        _isEmitDelistedInsightWarningSent = true;
-                    }
-
-                    // If this is our first invalid insight, create the list and fill it with previous values
-                    if (validInsights == null)
-                    {
-                        validInsights = new List<Insight>() {};
-                        for (var j = 0; j < i; j++)
-                        {
-                            validInsights.Add(insights[j]);
-                        }
-                    }
-                }
-                else
-                {
-                    // Initialize the insight fields
-                    insights[i] = InitializeInsightFields(insights[i]);
-
-                    // If we already had an invalid insight, this will have been initialized storing the valid ones.
-                    if (validInsights != null)
-                    {
-                        validInsights.Add(insights[i]);
-                    }
-                }
-            }
-
-            return validInsights == null ? insights : validInsights.ToArray();
-
+            EmitInsights(new []{insight});
         }
 
         /// <summary>

Algorithm/QCAlgorithm.Indicators.cs

@@ -126,28 +126,6 @@ namespace QuantConnect.Algorithm
             return averageDirectionalIndex;
         }
 
-        /// <summary>
-        /// Creates a new Awesome Oscillator from the specified periods.
-        /// </summary>
-        /// <param name="symbol">The symbol whose Awesome Oscillator we seek</param>
-        /// <param name="resolution">The resolution.</param>
-        /// <param name="fastPeriod">The period of the fast moving average associated with the AO</param>
-        /// <param name="slowPeriod">The period of the slow moving average associated with the AO</param>
-        /// <param name="type">The type of moving average used when computing the fast and slow term. Defaults to simple moving average.</param>
-        public AwesomeOscillator AO(Symbol symbol, int slowPeriod, int fastPeriod, MovingAverageType type, Resolution? resolution = null, Func<IBaseData, IBaseDataBar> selector = null)
-        {
-            var name = CreateIndicatorName(symbol, $"AO({fastPeriod},{slowPeriod},{type})", resolution);
-            var awesomeOscillator = new AwesomeOscillator(name, fastPeriod, slowPeriod, type);
-            RegisterIndicator(symbol, awesomeOscillator, resolution, selector);
-
-            if (EnableAutomaticIndicatorWarmUp)
-            {
-                WarmUpIndicator(symbol, awesomeOscillator, resolution);
-            }
-
-            return awesomeOscillator;
-        }
-        
         /// <summary>
         /// Creates a new AverageDirectionalMovementIndexRating indicator.
         /// </summary>
@@ -376,28 +354,6 @@ namespace QuantConnect.Algorithm
 
             return commodityChannelIndex;
         }
-        
-        /// <summary>
-        /// Creates a new ChaikinMoneyFlow indicator.
-        /// </summary>
-        /// <param name="symbol">The symbol whose CMF we want</param>
-        /// <param name="period">The period over which to compute the CMF</param>
-        /// <param name="resolution">The resolution</param>
-        /// <param name="selector">Selects a value from the BaseData to send into the indicator, if null defaults to casting the input value to a TradeBar</param>
-        /// <returns>The ChaikinMoneyFlow indicator for the requested symbol over the specified period</returns>
-        public ChaikinMoneyFlow CMF(Symbol symbol, int period, Resolution? resolution = null, Func<IBaseData, TradeBar> selector = null)
-        {
-            var name = CreateIndicatorName(symbol, $"CMF({period})", resolution);
-            var chaikinMoneyFlow = new ChaikinMoneyFlow(name, period);
-            RegisterIndicator(symbol, chaikinMoneyFlow, resolution, selector);
-            if (EnableAutomaticIndicatorWarmUp)
-            {
-                WarmUpIndicator(symbol, chaikinMoneyFlow, resolution);
-            }
-
-            return chaikinMoneyFlow;
-            
-        }
 
         /// <summary>
         /// Creates a new ChandeMomentumOscillator indicator.
@@ -420,31 +376,7 @@ namespace QuantConnect.Algorithm
 
             return chandeMomentumOscillator;
         }
-        
-        ///<summary>
-        /// Creates a new DeMarker Indicator (DEM), an oscillator-type indicator measuring changes in terms of an asset's
-        /// High and Low tradebar values. 
-        ///</summary>
-        /// <param name="symbol">The symbol whose DEM we seek.</param>
-        /// <param name="period">The period of the moving average implemented</param>
-        /// <param name="movingAverageType">Specifies the type of moving average to be used</param>
-        /// <param name="resolution">The resolution.</param>
-        /// <param name="selector">Selects a value from the BaseData to send into the indicator, if null defaults to casting the input value to a TradeBar</param>
-        /// <returns>The DeMarker indicator for the requested symbol.</returns>
-        public DeMarkerIndicator DEM(Symbol symbol, int period, MovingAverageType type, Resolution? resolution = null, Func<IBaseData, TradeBar> selector = null)
-        {
-            var name = CreateIndicatorName(symbol, $"DEM({period},{type})", resolution);
-            var deMarkerIndicator = new DeMarkerIndicator(name, period, type);
-            RegisterIndicator(symbol, deMarkerIndicator, resolution, selector);
 
-            if (EnableAutomaticIndicatorWarmUp)
-            {
-                WarmUpIndicator(symbol, deMarkerIndicator, resolution);
-            }
-
-            return deMarkerIndicator;
-        }
-        
         /// <summary>
         /// Creates a new Donchian Channel indicator which will compute the Upper Band and Lower Band.
         /// The indicator will be automatically updated on the given resolution.
@@ -2611,4 +2543,4 @@ namespace QuantConnect.Algorithm
             return new BaseDataConsolidator(calendar);
         }
     }
-}
+}

Algorithm/QCAlgorithm.cs

@@ -2286,15 +2286,17 @@ namespace QuantConnect.Algorithm
         /// </summary>
         /// <param name="insights">The collection of insights generaed at the current time step</param>
         /// <param name="clone">Will emit a clone of the generated insights</param>
-        private void OnInsightsGenerated(Insight[] insights, bool clone = true)
+        private void OnInsightsGenerated(IEnumerable<Insight> insights, bool clone = true)
         {
+            var insightCollection = insights.ToArray();
+
             // debug printing of generated insights
             if (DebugMode)
             {
-                Log($"{Time}: ALPHA: {string.Join(" | ", insights.Select(i => i.ToString()).OrderBy(i => i))}");
+                Log($"{Time}: ALPHA: {string.Join(" | ", insightCollection.Select(i => i.ToString()).OrderBy(i => i))}");
             }
 
-            InsightsGenerated?.Invoke(this, new GeneratedInsightsCollection(UtcTime, insights, clone: clone));
+            InsightsGenerated?.Invoke(this, new GeneratedInsightsCollection(UtcTime, insightCollection, clone: clone));
         }
 
         /// <summary>

Api/Api.cs

@@ -20,7 +20,6 @@ using System.Net;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using QuantConnect.Interfaces;
-using QuantConnect.Logging;
 using QuantConnect.Orders;
 using RestSharp;
 using RestSharp.Extensions;
@@ -384,15 +383,9 @@ namespace QuantConnect.Api
                 backtestName
             }), ParameterType.RequestBody);
 
-            BacktestResponseWrapper result;
+            Backtest result;
             ApiConnection.TryRequest(request, out result);
-
-            // Use API Response values for Backtest Values
-            result.Backtest.Success = result.Success;
-            result.Backtest.Errors = result.Errors;
-
-            // Return only the backtest object
-            return result.Backtest;
+            return result;
         }
 
         /// <summary>
@@ -415,58 +408,9 @@ namespace QuantConnect.Api
                 backtestId
             }), ParameterType.RequestBody);
 
-            BacktestResponseWrapper result;
+            Backtest result;
             ApiConnection.TryRequest(request, out result);
-
-            // Go fetch the charts if the backtest is completed
-            if (result.Backtest.Completed)
-            {
-                // For storing our collected charts
-                var updatedCharts = new Dictionary<string, Chart>();
-
-                // Create backtest requests for each chart that is empty
-                foreach (var chart in result.Backtest.Charts)
-                {
-                    if (!chart.Value.Series.IsNullOrEmpty())
-                    {
-                        continue;
-                    }
-
-                    var chartRequest = new RestRequest("backtests/read", Method.POST)
-                    {
-                        RequestFormat = DataFormat.Json
-                    };
-
-                    chartRequest.AddParameter("application/json", JsonConvert.SerializeObject(new
-                    {
-                        projectId,
-                        backtestId,
-                        chart = chart.Key.Replace(' ', '+')
-                    }), ParameterType.RequestBody);
-
-                    BacktestResponseWrapper chartResponse;
-                    ApiConnection.TryRequest(chartRequest, out chartResponse);
-
-                    // Add this chart to our updated collection
-                    if (chartResponse.Success)
-                    {
-                        updatedCharts.Add(chart.Key, chartResponse.Backtest.Charts[chart.Key]);
-                    }
-                }
-
-                // Update our result
-                foreach(var updatedChart in updatedCharts)
-                {
-                    result.Backtest.Charts[updatedChart.Key] = updatedChart.Value;
-                }
-            }
-
-            // Use API Response values for Backtest Values
-            result.Backtest.Success = result.Success;
-            result.Backtest.Errors = result.Errors;
-
-            // Return only the backtest object
-            return result.Backtest;
+            return result;
         }
 
         /// <summary>
@@ -817,17 +761,8 @@ namespace QuantConnect.Api
             var uri     = new Uri(link.DataLink);
             var client  = new RestClient(uri.Scheme + "://" + uri.Host);
             var request = new RestRequest(uri.PathAndQuery, Method.GET);
+            client.DownloadData(request).SaveAs(path);
 
-            // If the response is not a zip then it is not data, don't write it.
-            var response = client.Execute(request);
-            if (response.ContentType != "application/zip")
-            {
-                var message = JObject.Parse(response.Content)["message"].Value<string>();
-                Log.Error($"Api.DownloadData(): Failed to download zip for {symbol} {resolution} data for date {date}, Api response: {message}");
-                return false;
-            }
-            
-            response.RawBytes.SaveAs(path);
             return true;
         }
 

Brokerages/Bitfinex/BitfinexBrokerage.Messaging.cs

@@ -28,7 +28,6 @@ using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Linq;
-using System.Net;
 using QuantConnect.Brokerages.Bitfinex.Messages;
 using Order = QuantConnect.Orders.Order;
 
@@ -54,9 +53,6 @@ namespace QuantConnect.Brokerages.Bitfinex
         private readonly object _clientOrderIdLocker = new object();
         private long _nextClientOrderId;
 
-        // map Bitfinex currency to LEAN currency
-        private readonly Dictionary<string, string> _currencyMap;
-
         /// <summary>
         /// Locking object for the Ticks list in the data queue handler
         /// </summary>
@@ -93,15 +89,6 @@ namespace QuantConnect.Brokerages.Bitfinex
             _algorithm = algorithm;
             _aggregator = aggregator;
 
-            // load currency map
-            using (var wc = new WebClient())
-            {
-                var json = wc.DownloadString("https://api-pub.bitfinex.com/v2/conf/pub:map:currency:sym");
-                var rows = JsonConvert.DeserializeObject<List<List<List<string>>>>(json)[0];
-                _currencyMap = rows
-                    .ToDictionary(row => row[0], row => row[1].ToUpperInvariant());
-            }
-
             WebSocket.Open += (sender, args) =>
             {
                 SubscribeAuth();
@@ -397,7 +384,7 @@ namespace QuantConnect.Brokerages.Bitfinex
                 var fillQuantity = update.ExecAmount;
                 var direction = fillQuantity < 0 ? OrderDirection.Sell : OrderDirection.Buy;
                 var updTime = Time.UnixMillisecondTimeStampToDateTime(update.MtsCreate);
-                var orderFee = new OrderFee(new CashAmount(Math.Abs(update.Fee), GetLeanCurrency(update.FeeCurrency)));
+                var orderFee = new OrderFee(new CashAmount(Math.Abs(update.Fee), update.FeeCurrency));
 
                 var status = OrderStatus.Filled;
                 if (fillQuantity != order.Quantity)
@@ -453,17 +440,6 @@ namespace QuantConnect.Brokerages.Bitfinex
             }
         }
 
-        private string GetLeanCurrency(string brokerageCurrency)
-        {
-            string currency;
-            if (!_currencyMap.TryGetValue(brokerageCurrency.ToUpperInvariant(), out currency))
-            {
-                currency = brokerageCurrency.ToUpperInvariant();
-            }
-
-            return currency;
-        }
-
         /// <summary>
         /// Emit stream tick
         /// </summary>

Brokerages/Bitfinex/BitfinexBrokerage.cs

@@ -280,7 +280,7 @@ namespace QuantConnect.Brokerages.Bitfinex
             {
                 if (item.Balance > 0)
                 {
-                    list.Add(new CashAmount(item.Balance, GetLeanCurrency(item.Currency)));
+                    list.Add(new CashAmount(item.Balance, item.Currency.ToUpperInvariant()));
                 }
             }
 

Brokerages/InteractiveBrokers/InteractiveBrokersBrokerage.cs

@@ -2250,7 +2250,6 @@ namespace QuantConnect.Brokerages.InteractiveBrokers
             var market = InteractiveBrokersBrokerageModel.DefaultMarketMap[securityType];
             var isFutureOption = contract.SecType == IB.SecurityType.FutureOption;
 
-
             // Handle future options as a Future, up until we actually return the future.
             if (isFutureOption || securityType == SecurityType.Future)
             {
@@ -2261,7 +2260,6 @@ namespace QuantConnect.Brokerages.InteractiveBrokers
                     market = defaultMarket;
                 }
 
-                Log.Trace($"Mapping contract {contract} with ticker `{leanSymbol}` and SecurityType `{securityType.ToString()}` with expiry: {contract.LastTradeDateOrContractMonth} to Symbol");
                 var contractExpiryDate = DateTime.ParseExact(contract.LastTradeDateOrContractMonth, DateFormat.EightCharacter, CultureInfo.InvariantCulture);
 
                 if (!isFutureOption)
@@ -2283,7 +2281,6 @@ namespace QuantConnect.Brokerages.InteractiveBrokers
             }
             if (securityType == SecurityType.Option)
             {
-                Log.Trace($"Mapping contract {contract} with ticker `{ibSymbol}` and SecurityType `{securityType.ToString()}` with expiry: {contract.LastTradeDateOrContractMonth} to Symbol");
                 var expiryDate = DateTime.ParseExact(contract.LastTradeDateOrContractMonth, DateFormat.EightCharacter, CultureInfo.InvariantCulture);
                 var right = contract.Right == IB.RightType.Call ? OptionRight.Call : OptionRight.Put;
                 var strike = Convert.ToDecimal(contract.Strike);
@@ -3223,7 +3220,7 @@ namespace QuantConnect.Brokerages.InteractiveBrokers
         // these are warning messages from IB
         private static readonly HashSet<int> WarningCodes = new HashSet<int>
         {
-            102, 104, 105, 106, 107, 109, 110, 111, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 129, 131, 132, 133, 134, 135, 136, 137, 140, 141, 146, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 201, 303,313,314,315,319,325,328,329,334,335,336,337,338,339,340,341,342,343,345,347,348,349,350,352,353,355,356,358,359,360,361,362,363,364,367,368,369,370,371,372,373,374,375,376,377,378,379,380,382,383,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,402,403,404,405,406,407,408,409,410,411,412,413,417,418,419,420,421,422,423,424,425,426,427,428,429,430,433,434,435,436,437,439,440,441,442,443,444,445,446,447,448,449,450,10002,10003,10006,10007,10008,10009,10010,10011,10012,10014,10018,10019,10020,10052,10147,10148,10149,2100,2101,2102,2109,2148
+            102, 104, 105, 106, 107, 109, 110, 111, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 129, 131, 132, 133, 134, 135, 136, 137, 140, 141, 146, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 201, 303,313,314,315,319,325,328,329,334,335,336,337,338,339,340,341,342,343,345,347,348,349,350,352,353,355,356,358,359,360,361,362,363,364,367,368,369,370,371,372,373,374,375,376,377,378,379,380,382,383,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,402,403,404,405,406,407,408,409,410,411,412,413,417,418,419,420,421,422,423,424,425,426,427,428,429,430,433,434,435,436,437,439,440,441,442,443,444,445,446,447,448,449,450,1100,10002,10003,10006,10007,10008,10009,10010,10011,10012,10014,10018,10019,10020,10052,10147,10148,10149,1101,1102,2100,2101,2102,2103,2105,2109,2110,2148
         };
 
         // these require us to issue invalidated order events

Common/AlphaRuntimeStatistics.cs

@@ -19,7 +19,6 @@ using System.Collections.Generic;
 using Newtonsoft.Json;
 using QuantConnect.Algorithm.Framework.Alphas;
 using QuantConnect.Interfaces;
-using QuantConnect.Util;
 
 namespace QuantConnect
 {
@@ -184,7 +183,7 @@ namespace QuantConnect
         /// It is calculated by taking a portfolio's annualized rate of return and subtracting the risk free rate of return.
         /// </summary>
         /// <remarks>For performance we only truncate when the value is gotten</remarks>
-        [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore), JsonConverter(typeof(StringDecimalJsonConverter), true)]
+        [JsonProperty(DefaultValueHandling = DefaultValueHandling.Ignore)]
         public decimal SortinoRatio
         {
             get

Common/Api/Backtest.cs

@@ -16,7 +16,7 @@
 using System;
 using System.Collections.Generic;
 using Newtonsoft.Json;
-using QuantConnect.Statistics;
+using QuantConnect.Packets;
 
 namespace QuantConnect.Api
 {
@@ -55,6 +55,12 @@ namespace QuantConnect.Api
         [JsonProperty(PropertyName = "progress")]
         public decimal Progress;
 
+        /// <summary>
+        /// Result packet for the backtest
+        /// </summary>
+        [JsonProperty(PropertyName = "result")]
+        public BacktestResult Result;
+
         /// <summary>
         /// Backtest error message
         /// </summary>
@@ -72,56 +78,6 @@ namespace QuantConnect.Api
         /// </summary>
         [JsonProperty(PropertyName = "created")]
         public DateTime Created;
-
-        /// <summary>
-        /// Rolling window detailed statistics.
-        /// </summary>
-        [JsonProperty(PropertyName = "rollingWindow", NullValueHandling = NullValueHandling.Ignore)]
-        public Dictionary<string, AlgorithmPerformance> RollingWindow;
-
-        /// <summary>
-        /// Rolling window detailed statistics.
-        /// </summary>
-        [JsonProperty(PropertyName = "totalPerformance", NullValueHandling = NullValueHandling.Ignore)]
-        public AlgorithmPerformance TotalPerformance;
-
-        /// <summary>
-        /// Contains population averages scores over the life of the algorithm
-        /// </summary>
-        [JsonProperty(PropertyName = "alphaRuntimeStatistics", NullValueHandling = NullValueHandling.Ignore)]
-        public AlphaRuntimeStatistics AlphaRuntimeStatistics;
-
-        /// <summary>
-        /// Charts updates for the live algorithm since the last result packet
-        /// </summary>
-        [JsonProperty(PropertyName = "charts", NullValueHandling = NullValueHandling.Ignore)]
-        public IDictionary<string, Chart> Charts;
-
-        /// <summary>
-        /// Statistics information sent during the algorithm operations.
-        /// </summary>
-        /// <remarks>Intended for update mode -- send updates to the existing statistics in the result GUI. If statistic key does not exist in GUI, create it</remarks>
-        [JsonProperty(PropertyName = "statistics", NullValueHandling = NullValueHandling.Ignore)]
-        public IDictionary<string, string> Statistics;
-
-        /// <summary>
-        /// Runtime banner/updating statistics in the title banner of the live algorithm GUI.
-        /// </summary>
-        [JsonProperty(PropertyName = "runtimeStatistics", NullValueHandling = NullValueHandling.Ignore)]
-        public IDictionary<string, string> RuntimeStatistics;
-    }
-
-    /// <summary>
-    /// Wrapper class for Backtest/* endpoints JSON response
-    /// Currently used by Backtest/Read and Backtest/Create
-    /// </summary>
-    public class BacktestResponseWrapper : RestResponse
-    {
-        /// <summary>
-        /// Backtest Object
-        /// </summary>
-        [JsonProperty(PropertyName = "backtest")]
-        public Backtest Backtest;
     }
 
     /// <summary>

Common/Data/Market/DataDictionary.cs

@@ -54,17 +54,8 @@ namespace QuantConnect.Data.Market
         /// <param name="time">The time this data was emitted.</param>
         public DataDictionary(DateTime time)
         {
-#pragma warning disable 618 // This assignment is left here until the Time property is removed.
-            Time = time;
-#pragma warning restore 618
         }
 
-        /// <summary>
-        /// Gets or sets the time associated with this collection of data
-        /// </summary>
-        [Obsolete("The DataDictionary<T> Time property is now obsolete. All algorithms should use algorithm.Time instead.")]
-        public DateTime Time { get; set; }
-
         /// <summary>
         /// Returns an enumerator that iterates through the collection.
         /// </summary>
@@ -300,4 +291,4 @@ namespace QuantConnect.Data.Market
             dictionary.Add(data.Symbol, data);
         }
     }
-}
+}

Common/Data/Market/Delistings.cs

@@ -47,7 +47,7 @@ namespace QuantConnect.Data.Market
         /// </returns>
         /// <param name="ticker">The ticker of the element to get or set.</param>
         /// <remarks>Wraps the base implementation to enable indexing in python algorithms due to pythonnet limitations</remarks>
-        public new Delisting this[string ticker] { get { return base[ticker]; } set { base[ticker] = value; } }
+        public new Delisting this[string ticker] { get { return base[ticker]; } internal set { base[ticker] = value; } }
 
         /// <summary>
         /// Gets or sets the Delisting with the specified Symbol.
@@ -57,6 +57,6 @@ namespace QuantConnect.Data.Market
         /// </returns>
         /// <param name="symbol">The Symbol of the element to get or set.</param>
         /// <remarks>Wraps the base implementation to enable indexing in python algorithms due to pythonnet limitations</remarks>
-        public new Delisting this[Symbol symbol] { get { return base[symbol]; } set { base[symbol] = value; } }
+        public new Delisting this[Symbol symbol] { get { return base[symbol]; } internal set { base[symbol] = value; } }
     }
 }

Common/Data/Market/Dividends.cs

@@ -47,7 +47,7 @@ namespace QuantConnect.Data.Market
         /// </returns>
         /// <param name="ticker">The ticker of the element to get or set.</param>
         /// <remarks>Wraps the base implementation to enable indexing in python algorithms due to pythonnet limitations</remarks>
-        public new Dividend this[string ticker] { get { return base[ticker]; } set { base[ticker] = value; } }
+        public new Dividend this[string ticker] { get { return base[ticker]; } internal set { base[ticker] = value; } }
 
         /// <summary>
         /// Gets or sets the Dividend with the specified Symbol.
@@ -57,6 +57,6 @@ namespace QuantConnect.Data.Market
         /// </returns>
         /// <param name="symbol">The Symbol of the element to get or set.</param>
         /// <remarks>Wraps the base implementation to enable indexing in python algorithms due to pythonnet limitations</remarks>
-        public new Dividend this[Symbol symbol] { get { return base[symbol]; } set { base[symbol] = value; } }
+        public new Dividend this[Symbol symbol] { get { return base[symbol]; } internal set { base[symbol] = value; } }
     }
 }

Common/Data/Market/FuturesChains.cs

@@ -45,7 +45,7 @@ namespace QuantConnect.Data.Market
         /// </returns>
         /// <param name="ticker">The ticker of the element to get or set.</param>
         /// <remarks>Wraps the base implementation to enable indexing in python algorithms due to pythonnet limitations</remarks>
-        public new FuturesChain this[string ticker] { get { return base[ticker]; } set { base[ticker] = value; } }
+        public new FuturesChain this[string ticker] { get { return base[ticker]; } internal set { base[ticker] = value; } }
 
         /// <summary>
         /// Gets or sets the FuturesChain with the specified Symbol.
@@ -55,6 +55,6 @@ namespace QuantConnect.Data.Market
         /// </returns>
         /// <param name="symbol">The Symbol of the element to get or set.</param>
         /// <remarks>Wraps the base implementation to enable indexing in python algorithms due to pythonnet limitations</remarks>
-        public new FuturesChain this[Symbol symbol] { get { return base[symbol]; } set { base[symbol] = value; } }
+        public new FuturesChain this[Symbol symbol] { get { return base[symbol]; } internal set { base[symbol] = value; } }
     }
-}
+}

Common/Data/Market/OptionChains.cs

@@ -45,7 +45,7 @@ namespace QuantConnect.Data.Market
         /// </returns>
         /// <param name="ticker">The ticker of the element to get or set.</param>
         /// <remarks>Wraps the base implementation to enable indexing in python algorithms due to pythonnet limitations</remarks>
-        public new OptionChain this[string ticker] { get { return base[ticker]; } set { base[ticker] = value; } }
+        public new OptionChain this[string ticker] { get { return base[ticker]; } internal set { base[ticker] = value; } }
 
         /// <summary>
         /// Gets or sets the OptionChain with the specified Symbol.
@@ -55,6 +55,6 @@ namespace QuantConnect.Data.Market
         /// </returns>
         /// <param name="symbol">The Symbol of the element to get or set.</param>
         /// <remarks>Wraps the base implementation to enable indexing in python algorithms due to pythonnet limitations</remarks>
-        public new OptionChain this[Symbol symbol] { get { return base[symbol]; } set { base[symbol] = value; } }
+        public new OptionChain this[Symbol symbol] { get { return base[symbol]; } internal set { base[symbol] = value; } }
     }
-}
+}

Common/Data/Market/QuoteBars.cs

@@ -45,7 +45,7 @@ namespace QuantConnect.Data.Market
         /// </returns>
         /// <param name="ticker">The ticker of the element to get or set.</param>
         /// <remarks>Wraps the base implementation to enable indexing in python algorithms due to pythonnet limitations</remarks>
-        public new QuoteBar this[string ticker] { get { return base[ticker]; } set { base[ticker] = value; } }
+        public new QuoteBar this[string ticker] { get { return base[ticker]; } internal set { base[ticker] = value; } }
 
         /// <summary>
         /// Gets or sets the QuoteBar with the specified Symbol.
@@ -55,6 +55,6 @@ namespace QuantConnect.Data.Market
         /// </returns>
         /// <param name="symbol">The Symbol of the element to get or set.</param>
         /// <remarks>Wraps the base implementation to enable indexing in python algorithms due to pythonnet limitations</remarks>
-        public new QuoteBar this[Symbol symbol] { get { return base[symbol]; } set { base[symbol] = value; } }
+        public new QuoteBar this[Symbol symbol] { get { return base[symbol]; } internal set { base[symbol] = value; } }
     }
-}
+}

Common/Data/Market/Splits.cs

@@ -47,7 +47,7 @@ namespace QuantConnect.Data.Market
         /// </returns>
         /// <param name="ticker">The ticker of the element to get or set.</param>
         /// <remarks>Wraps the base implementation to enable indexing in python algorithms due to pythonnet limitations</remarks>
-        public new Split this[string ticker] { get { return base[ticker]; } set { base[ticker] = value; } }
+        public new Split this[string ticker] { get { return base[ticker]; } internal set { base[ticker] = value; } }
 
         /// <summary>
         /// Gets or sets the Split with the specified Symbol.
@@ -57,6 +57,6 @@ namespace QuantConnect.Data.Market
         /// </returns>
         /// <param name="symbol">The Symbol of the element to get or set.</param>
         /// <remarks>Wraps the base implementation to enable indexing in python algorithms due to pythonnet limitations</remarks>
-        public new Split this[Symbol symbol] { get { return base[symbol]; } set { base[symbol] = value; } }
+        public new Split this[Symbol symbol] { get { return base[symbol]; } internal set { base[symbol] = value; } }
     }
-}
+}

Common/Data/Market/SymbolChangedEvents.cs

@@ -47,7 +47,7 @@ namespace QuantConnect.Data.Market
         /// </returns>
         /// <param name="ticker">The ticker of the element to get or set.</param>
         /// <remarks>Wraps the base implementation to enable indexing in python algorithms due to pythonnet limitations</remarks>
-        public new SymbolChangedEvent this[string ticker] { get { return base[ticker]; } set { base[ticker] = value; } }
+        public new SymbolChangedEvent this[string ticker] { get { return base[ticker]; } internal set { base[ticker] = value; } }
 
         /// <summary>
         /// Gets or sets the SymbolChangedEvent with the specified Symbol.
@@ -57,6 +57,6 @@ namespace QuantConnect.Data.Market
         /// </returns>
         /// <param name="symbol">The Symbol of the element to get or set.</param>
         /// <remarks>Wraps the base implementation to enable indexing in python algorithms due to pythonnet limitations</remarks>
-        public new SymbolChangedEvent this[Symbol symbol] { get { return base[symbol]; } set { base[symbol] = value; } }
+        public new SymbolChangedEvent this[Symbol symbol] { get { return base[symbol]; } internal set { base[symbol] = value; } }
     }
 }

Common/Data/Market/Ticks.cs

@@ -48,7 +48,7 @@ namespace QuantConnect.Data.Market
         /// </returns>
         /// <param name="ticker">The ticker of the element to get or set.</param>
         /// <remarks>Wraps the base implementation to enable indexing in python algorithms due to pythonnet limitations</remarks>
-        public new List<Tick> this[string ticker] { get { return base[ticker]; } set { base[ticker] = value; } }
+        public new List<Tick> this[string ticker] { get { return base[ticker]; } internal set { base[ticker] = value; } }
 
         /// <summary>
         /// Gets or sets the list of Tick with the specified Symbol.
@@ -58,6 +58,6 @@ namespace QuantConnect.Data.Market
         /// </returns>
         /// <param name="symbol">The Symbol of the element to get or set.</param>
         /// <remarks>Wraps the base implementation to enable indexing in python algorithms due to pythonnet limitations</remarks>
-        public new List<Tick> this[Symbol symbol] { get { return base[symbol]; } set { base[symbol] = value; } }
+        public new List<Tick> this[Symbol symbol] { get { return base[symbol]; } internal set { base[symbol] = value; } }
     }
 }

Common/Data/Market/TradeBars.cs

@@ -46,7 +46,7 @@ namespace QuantConnect.Data.Market
         /// </returns>
         /// <param name="ticker">The ticker of the element to get or set.</param>
         /// <remarks>Wraps the base implementation to enable indexing in python algorithms due to pythonnet limitations</remarks>
-        public new TradeBar this[string ticker] { get { return base[ticker]; } set { base[ticker] = value; } }
+        public new TradeBar this[string ticker] { get { return base[ticker]; } internal set { base[ticker] = value; } }
 
         /// <summary>
         /// Gets or sets the TradeBar with the specified Symbol.
@@ -56,6 +56,6 @@ namespace QuantConnect.Data.Market
         /// </returns>
         /// <param name="symbol">The Symbol of the element to get or set.</param>
         /// <remarks>Wraps the base implementation to enable indexing in python algorithms due to pythonnet limitations</remarks>
-        public new TradeBar this[Symbol symbol] { get { return base[symbol]; } set { base[symbol] = value; } }
+        public new TradeBar this[Symbol symbol] { get { return base[symbol]; } internal set { base[symbol] = value; } }
     }
-}
+}

Common/Data/Slice.cs

@@ -203,15 +203,15 @@ namespace QuantConnect.Data
         /// <param name="time">The timestamp for this slice of data</param>
         /// <param name="data">The raw data in this slice</param>
         public Slice(DateTime time, List<BaseData> data)
-            : this(time, data, CreateCollection<TradeBars, TradeBar>(time, data),
-                CreateCollection<QuoteBars, QuoteBar>(time, data),
-                CreateTicksCollection(time, data),
-                CreateCollection<OptionChains, OptionChain>(time, data),
-                CreateCollection<FuturesChains, FuturesChain>(time, data),
-                CreateCollection<Splits, Split>(time, data),
-                CreateCollection<Dividends, Dividend>(time, data),
-                CreateCollection<Delistings, Delisting>(time, data),
-                CreateCollection<SymbolChangedEvents, SymbolChangedEvent>(time, data))
+            : this(time, data, CreateCollection<TradeBars, TradeBar>(data),
+                CreateCollection<QuoteBars, QuoteBar>(data),
+                CreateTicksCollection(data),
+                CreateCollection<OptionChains, OptionChain>(data),
+                CreateCollection<FuturesChains, FuturesChain>(data),
+                CreateCollection<Splits, Split>(data),
+                CreateCollection<Dividends, Dividend>(data),
+                CreateCollection<Delistings, Delisting>(data),
+                CreateCollection<SymbolChangedEvents, SymbolChangedEvent>(data))
         {
         }
 
@@ -500,9 +500,9 @@ namespace QuantConnect.Data
         /// <summary>
         /// Dynamically produces a <see cref="Ticks"/> data dictionary using the provided data
         /// </summary>
-        private static Ticks CreateTicksCollection(DateTime time, IEnumerable<BaseData> data)
+        private static Ticks CreateTicksCollection(IEnumerable<BaseData> data)
         {
-            var ticks = new Ticks(time);
+            var ticks = new Ticks();
             foreach (var tick in data.OfType<Tick>())
             {
                 List<Tick> listTicks;
@@ -523,16 +523,11 @@ namespace QuantConnect.Data
         /// <param name="time">The current slice time</param>
         /// <param name="data">The data to create the collection</param>
         /// <returns>The data dictionary of <typeparamref name="TItem"/> containing all the data of that type in this slice</returns>
-        private static T CreateCollection<T, TItem>(DateTime time, IEnumerable<BaseData> data)
+        private static T CreateCollection<T, TItem>(IEnumerable<BaseData> data)
             where T : DataDictionary<TItem>, new()
             where TItem : BaseData
         {
-            var collection = new T
-            {
-#pragma warning disable 618 // This assignment is left here until the Time property is removed.
-                Time = time
-#pragma warning restore 618
-            };
+            var collection = new T();
             foreach (var item in data.OfType<TItem>())
             {
                 collection[item.Symbol] = item;

Common/Extensions.cs

@@ -118,6 +118,10 @@ namespace QuantConnect
         {
             Guids.Add(guid);
         }
+        
+	/// Unix epoch (1970-01-01 00:00:00.000000000Z)
+        /// </summary>
+        public static DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
 
         /// <summary>
         /// Serialize a list of ticks using protobuf
@@ -1298,14 +1302,13 @@ namespace QuantConnect
         /// <param name="to">The time zone to be converted to</param>
         /// <param name="strict">True for strict conversion, this will throw during ambiguitities, false for lenient conversion</param>
         /// <returns>The time in terms of the to time zone</returns>
-        public static DateTime ConvertTo(this DateTime time, DateTimeZone from, DateTimeZone to, bool strict = false)
+        public static DateTime ConvertTo(this DateTime time, DateTimeZone from, DateTimeZone to)
         {
-            if (strict)
-            {
-                return from.AtStrictly(LocalDateTime.FromDateTime(time)).WithZone(to).ToDateTimeUnspecified();
-            }
+            var instant = new Instant(time.Ticks - UnixEpoch.Ticks);
+            var fromOffset = from.GetUtcOffset(instant).ToTimeSpan();
+            var toOffset = to.GetUtcOffset(instant).ToTimeSpan();
 
-            return from.AtLeniently(LocalDateTime.FromDateTime(time)).WithZone(to).ToDateTimeUnspecified();
+            return time - (fromOffset - toOffset);
         }
 
         /// <summary>
@@ -1315,11 +1318,12 @@ namespace QuantConnect
         /// <param name="to">The destinatio time zone</param>
         /// <param name="strict">True for strict conversion, this will throw during ambiguitities, false for lenient conversion</param>
         /// <returns>The time in terms of the <paramref name="to"/> time zone</returns>
-        public static DateTime ConvertFromUtc(this DateTime time, DateTimeZone to, bool strict = false)
+        public static DateTime ConvertFromUtc(this DateTime time, DateTimeZone to)
         {
-            return time.ConvertTo(TimeZones.Utc, to, strict);
+            return time + to.GetUtcOffset(new Instant(time.Ticks - UnixEpoch.Ticks)).ToTimeSpan();
         }
 
+
         /// <summary>
         /// Converts the specified time from the <paramref name="from"/> time zone to <see cref="TimeZones.Utc"/>
         /// </summary>
@@ -1327,14 +1331,9 @@ namespace QuantConnect
         /// <param name="from">The time zone the specified <paramref name="time"/> is in</param>
         /// <param name="strict">True for strict conversion, this will throw during ambiguitities, false for lenient conversion</param>
         /// <returns>The time in terms of the to time zone</returns>
-        public static DateTime ConvertToUtc(this DateTime time, DateTimeZone from, bool strict = false)
+        public static DateTime ConvertToUtc(this DateTime time, DateTimeZone from)
         {
-            if (strict)
-            {
-                return from.AtStrictly(LocalDateTime.FromDateTime(time)).ToDateTimeUtc();
-            }
-
-            return from.AtLeniently(LocalDateTime.FromDateTime(time)).ToDateTimeUtc();
+            return time.Subtract(from.GetUtcOffset(Instant.FromTicksSinceUnixEpoch((time.Ticks - UnixEpoch.Ticks))).ToTimeSpan());
         }
 
         /// <summary>

Common/Packets/AlgorithmNodePacket.cs

@@ -33,12 +33,6 @@ namespace QuantConnect.Packets
             : base(type)
         { }
 
-        /// <summary>
-        /// The host name to use if any
-        /// </summary>
-        [JsonProperty(PropertyName = "sHostName")]
-        public string HostName;
-
         /// <summary>
         /// User Id placing request
         /// </summary>

Common/Packets/AlgorithmStatusPacket.cs

@@ -49,12 +49,6 @@ namespace QuantConnect.Packets
         [JsonProperty(PropertyName = "sAlgorithmID")]
         public string AlgorithmId;
 
-        /// <summary>
-        /// OptimizationId for this result packet if any
-        /// </summary>
-        [JsonProperty(PropertyName = "sOptimizationID")]
-        public string OptimizationId;
-
         /// <summary>
         /// Project Id associated with this status packet
         /// </summary>

Common/Packets/BacktestNodePacket.cs

@@ -44,12 +44,6 @@ namespace QuantConnect.Packets
         [JsonProperty(PropertyName = "sBacktestID")]
         public string BacktestId = DefaultId;
 
-        /// <summary>
-        /// Optimization Id for this task
-        /// </summary>
-        [JsonProperty(PropertyName = "sOptimizationID")]
-        public string OptimizationId;
-
         /// <summary>
         /// Backtest start-date as defined in the Initialize() method.
         /// </summary>

Common/Packets/BacktestResultPacket.cs

@@ -52,12 +52,6 @@ namespace QuantConnect.Packets
         [JsonProperty(PropertyName = "sBacktestID")]
         public string BacktestId = "";
 
-        /// <summary>
-        /// OptimizationId for this result packet if any
-        /// </summary>
-        [JsonProperty(PropertyName = "sOptimizationID")]
-        public string OptimizationId;
-
         /// <summary>
         /// Compile Id for the algorithm which generated this result packet.
         /// </summary>
@@ -153,7 +147,6 @@ namespace QuantConnect.Packets
                 Results             = packet.Results;
                 ProcessingTime      = packet.ProcessingTime;
                 TradeableDates      = packet.TradeableDates;
-                OptimizationId      = packet.OptimizationId;
             }
             catch (Exception err)
             {
@@ -182,7 +175,6 @@ namespace QuantConnect.Packets
                 CompileId = job.CompileId;
                 Channel = job.Channel;
                 BacktestId = job.BacktestId;
-                OptimizationId = job.OptimizationId;
                 Results = results;
                 Name = job.Name;
                 UserId = job.UserId;

Common/Packets/Controls.cs

@@ -146,8 +146,8 @@ namespace QuantConnect.Packets
         /// <summary>
         /// The cost associated with running this job
         /// </summary>
-        [JsonProperty(PropertyName = "dCreditCost")]
-        public decimal CreditCost;
+        [JsonProperty(PropertyName = "iCreditCost")]
+        public uint CreditCost;
 
         /// <summary>
         /// Initializes a new default instance of the <see cref="Controls"/> class

Common/Packets/Packet.cs

@@ -162,9 +162,6 @@ namespace QuantConnect.Packets
         OptimizationEstimate,
 
         /// Optimization work status update
-        OptimizationStatus,
-
-        /// Optimization work result
-        OptimizationResult
+        OptimizationStatus
     }
 }

Common/Python/PandasArrowMemoryAllocator.cs

@@ -0,0 +1,189 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Buffers;
+using System.Collections.Generic;
+using System.Linq;
+using Apache.Arrow.Memory;
+
+namespace QuantConnect.Python
+{
+    public class PandasArrowMemoryAllocator : NativeMemoryAllocator, IDisposable
+    {
+        private bool _disposed;
+        private readonly List<PandasMemoryOwner> _free = new List<PandasMemoryOwner>();
+        private readonly List<PandasMemoryOwner> _used = new List<PandasMemoryOwner>();
+
+        public PandasArrowMemoryAllocator() : base()
+        {
+        }
+
+        protected override IMemoryOwner<byte> AllocateInternal(int length, out int bytesAllocated)
+        {
+            PandasMemoryOwner owner;
+            var memoryResizeIndexes = new List<KeyValuePair<int, int>>();
+
+            for (var i = 0; i < _free.Count; i++)
+            {
+                var memory = _free[i];
+                if (length > memory.Original.Memory.Length)
+                {
+                    memoryResizeIndexes.Add(new KeyValuePair<int, int>(i, memory.Original.Memory.Length));
+                    continue;
+                }
+
+                owner = memory;
+                bytesAllocated = 0;
+
+                _free.Remove(owner);
+                _used.Add(owner);
+                owner.Reset();
+
+                if (length != memory.Original.Memory.Length)
+                {
+                    owner.Slice(0, length);
+                }
+
+                return owner;
+            }
+
+            if (memoryResizeIndexes.Count != 0)
+            {
+                // Get the smallest resizable instance, and reallocate a larger buffer.
+                var resizeIndex = memoryResizeIndexes.OrderBy(x => x.Value).First();
+                var resizable = _free[resizeIndex.Key];
+
+                resizable.Resize(base.AllocateInternal(length, out bytesAllocated));
+
+                _used.Add(resizable);
+                _free.RemoveAt(resizeIndex.Key);
+
+                return resizable;
+            }
+
+            // New allocation, should only be called a few times when we start using the allocator
+            owner = new PandasMemoryOwner(base.AllocateInternal(length, out bytesAllocated));
+            _used.Add(owner);
+
+            return owner;
+        }
+
+        /// <summary>
+        /// Frees the underlying memory buffers so that they can be re-used
+        /// </summary>
+        public void Free()
+        {
+            foreach (var used in _used)
+            {
+                _free.Add(used);
+            }
+
+            _used.Clear();
+        }
+
+        private class PandasMemoryOwner : IMemoryOwner<byte>
+        {
+            private bool _disposed;
+
+            /// <summary>
+            /// Original memory owner containing the full-length byte-array
+            /// we initially allocated.
+            /// </summary>
+            public IMemoryOwner<byte> Original { get; private set; }
+
+            /// <summary>
+            /// Slice of the original memory owner containing the contents of
+            /// the buffer Arrow will use. We slice the original memory so
+            /// that Arrow doesn't panic when it receives a slice with a length
+            /// longer than it expects when serializing its internal buffer.
+            /// </summary>
+            public Memory<byte> Memory { get; private set; }
+
+            public PandasMemoryOwner(IMemoryOwner<byte> memory)
+            {
+                Original = memory;
+                Memory = Original.Memory;
+            }
+
+            /// <summary>
+            /// Creates a slice of the original MemoryOwner and stores the result in <see cref="Memory"/>
+            /// </summary>
+            /// <param name="start">Index start of the slice</param>
+            /// <param name="length">Length of the slice</param>
+            public void Slice(int start, int length)
+            {
+                Memory = Original.Memory.Slice(start, length);
+            }
+
+            /// <summary>
+            /// Restores the <see cref="Memory"/> slice to its initial value
+            /// </summary>
+            public void Reset()
+            {
+                Memory = null;
+                Memory = Original.Memory;
+            }
+
+            /// <summary>
+            /// Resizes the instance to the new memory size
+            /// </summary>
+            /// <param name="newMemory"></param>
+            public void Resize(IMemoryOwner<byte> newMemory)
+            {
+                Original.Dispose();
+                Original = newMemory;
+                Memory = null;
+                Memory = Original.Memory;
+            }
+
+            public void Free()
+            {
+                Original.Dispose();
+                Memory = null;
+                Original = null;
+            }
+
+            /// <summary>
+            /// no-op dispose because we want to re-use the MemoryOwner instance after we dispose of a RecordBatch.
+            /// To dispose of the resources this class owns, use <see cref="Free"/>
+            /// </summary>
+            public void Dispose()
+            {
+            }
+        }
+
+        public void Dispose()
+        {
+            if (_disposed)
+            {
+                throw new ObjectDisposedException("PandasArrowMemoryAllocator has already been disposed");
+            }
+            foreach (var free in _free)
+            {
+                free.Free();
+            }
+            foreach (var used in _used)
+            {
+                used.Free();
+            }
+
+            _free.Clear();
+            _used.Clear();
+
+            _disposed = true;
+        }
+    }
+}

Common/Python/PandasData.cs

@@ -1,612 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using Python.Runtime;
-using QuantConnect.Data;
-using QuantConnect.Data.Market;
-using QuantConnect.Util;
-using System;
-using System.Collections;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
-using System.Linq;
-using System.Reflection;
-
-namespace QuantConnect.Python
-{
-    /// <summary>
-    /// Organizes a list of data to create pandas.DataFrames
-    /// </summary>
-    public class PandasData
-    {
-        private static dynamic _pandas;
-        private readonly static HashSet<string> _baseDataProperties = typeof(BaseData).GetProperties().ToHashSet(x => x.Name.ToLowerInvariant());
-        private readonly static ConcurrentDictionary<Type, List<MemberInfo>> _membersByType = new ConcurrentDictionary<Type, List<MemberInfo>>();
-
-        private readonly Symbol _symbol;
-        private readonly Dictionary<string, Tuple<List<DateTime>, List<object>>> _series;
-
-        private readonly List<MemberInfo> _members;
-
-        /// <summary>
-        /// Gets true if this is a custom data request, false for normal QC data
-        /// </summary>
-        public bool IsCustomData { get; }
-
-        /// <summary>
-        /// Implied levels of a multi index pandas.Series (depends on the security type)
-        /// </summary>
-        public int Levels { get; } = 2;
-
-        /// <summary>
-        /// Initializes an instance of <see cref="PandasData"/>
-        /// </summary>
-        public PandasData(object data)
-        {
-            if (_pandas == null)
-            {
-                using (Py.GIL())
-                {
-                    // this python Remapper class will work as a proxy and adjust the
-                    // input to its methods using the provided 'mapper' callable object
-                    _pandas = PythonEngine.ModuleFromString("remapper",
-                        @"import pandas as pd
-from pandas.core.resample import Resampler, DatetimeIndexResampler, PeriodIndexResampler, TimedeltaIndexResampler
-from pandas.core.groupby.generic import DataFrameGroupBy, SeriesGroupBy
-from pandas.core.indexes.frozen import FrozenList as pdFrozenList
-from pandas.core.window import Expanding, EWM, Rolling, Window
-from pandas.core.computation.ops import UndefinedVariableError
-from inspect import getmembers, isfunction, isgenerator
-from functools import partial
-from sys import modules
-
-from clr import AddReference
-AddReference(""QuantConnect.Common"")
-from QuantConnect import *
-
-def mapper(key):
-    '''Maps a Symbol object or a Symbol Ticker (string) to the string representation of
-    Symbol SecurityIdentifier. If cannot map, returns the object
-    '''
-    keyType = type(key)
-    if keyType is Symbol:
-        return str(key.ID)
-    if keyType is str:
-        reserved = ['high', 'low', 'open', 'close']
-        if key in reserved:
-            return key
-        kvp = SymbolCache.TryGetSymbol(key, None)
-        if kvp[0]:
-            return str(kvp[1].ID)
-    if keyType is list:
-        return [mapper(x) for x in key]
-    if keyType is tuple:
-        return tuple([mapper(x) for x in key])
-    if keyType is dict:
-        return {k:mapper(v) for k,v in key.items()}
-    return key
-
-def try_wrap_as_index(obj):
-    '''Tries to wrap object if it is one of pandas' index objects.'''
-
-    objType = type(obj)
-
-    if objType is pd.Index:
-        return True, Index(obj)
-
-    if objType is pd.MultiIndex:
-        result = object.__new__(MultiIndex)
-        result._set_levels(obj.levels, copy=obj.copy, validate=False)
-        result._set_codes(obj.codes, copy=obj.copy, validate=False)
-        result._set_names(obj.names)
-        result.sortorder = obj.sortorder
-        return True, result
-
-    if objType is pdFrozenList:
-        return True, FrozenList(obj)
-
-    return False, obj
-
-def try_wrap_as_pandas(obj):
-    '''Tries to wrap object if it is a pandas' object.'''
-
-    success, obj = try_wrap_as_index(obj)
-    if success:
-        return success, obj
-
-    objType = type(obj)
-
-    if objType is pd.DataFrame:
-        return True, DataFrame(data=obj)
-
-    if objType is pd.Series:
-        return True, Series(data=obj)
-
-    if objType is tuple:
-        anySuccess = False
-        results = list()
-        for item in obj:
-            success, result = try_wrap_as_pandas(item)
-            anySuccess |= success
-            results.append(result)
-        if anySuccess:
-            return True, tuple(results)
-
-    return False, obj
-
-def try_wrap_resampler(obj, self):
-    '''Tries to wrap object if it is a pandas' Resampler object.'''
-
-    if not isinstance(obj, Resampler):
-        return False, obj
-
-    klass = CreateWrapperClass(type(obj))
-    return True, klass(self, groupby=obj.groupby, kind=obj.kind, axis=obj.axis)
-
-def wrap_function(f):
-    '''Wraps function f with g.
-    Function g converts the args/kwargs to use alternative index keys
-    and the result of the f function call to the wrapper objects
-    '''
-    def g(*args, **kwargs):
-
-        if len(args) > 1:
-            args = mapper(args)
-        if len(kwargs) > 0:
-            kwargs = mapper(kwargs)
-
-        try:
-            result = f(*args, **kwargs)
-        except UndefinedVariableError as e:
-            # query/eval methods needs to look for a scope variable at a higher level
-            # since the wrapper classes are children of pandas classes
-            kwargs['level'] = kwargs.pop('level', 0) + 1
-            result = f(*args, **kwargs)
-
-        success, result = try_wrap_as_pandas(result)
-        if success:
-            return result
-
-        success, result = try_wrap_resampler(result, args[0])
-        if success:
-            return result
-
-        if isgenerator(result):
-            return ( (k, try_wrap_as_pandas(v)[1]) for k, v in result)
-
-        return result
-
-    g.__name__ = f.__name__
-    return g
-
-def wrap_special_function(name, cls, fcls, gcls = None):
-    '''Replaces the special function of a given class by g that wraps fcls
-    This is how pandas implements them.
-    gcls represents an alternative for fcls
-    if the keyword argument has 'win_type' key for the Rolling/Window case
-    '''
-    fcls = CreateWrapperClass(fcls)
-    if gcls is not None:
-        gcls = CreateWrapperClass(fcls)
-
-    def g(*args, **kwargs):
-        if kwargs.get('win_type', None):
-            return gcls(*args, **kwargs)
-        return fcls(*args, **kwargs)
-    g.__name__ = name
-    setattr(cls, g.__name__, g)
-
-def CreateWrapperClass(cls: type):
-    '''Creates wrapper classes.
-    Members of the original class are wrapped to allow alternative index look-up
-    '''
-    # Define a new class
-    klass = type(f'{cls.__name__}', (cls,) + cls.__bases__, dict(cls.__dict__))
-
-    def g(self, name):
-        '''Wrap '__getattribute__' to handle indices
-        Only need to wrap columns, index and levels attributes
-        '''
-        attr = object.__getattribute__(self, name)
-        if name in ['columns', 'index', 'levels']:
-            _, attr = try_wrap_as_index(attr)
-        return attr
-    g.__name__ =  '__getattribute__'
-    g.__qualname__ =  g.__name__
-    setattr(klass, g.__name__, g)
-
-    def wrap_union(f):
-        '''Wraps function f (union) with g.
-        Special case: The union method from index objects needs to
-        receive pandas' index objects to avoid infity recursion.
-        Function g converts the args/kwargs objects to one of pandas index objects
-        and the result of the f function call back to wrapper indexes objects
-        '''
-        def unwrap_index(obj):
-            '''Tries to unwrap object if it is one of this module wrapper's index objects.'''
-            objType = type(obj)
-
-            if objType is Index:
-                return pd.Index(obj)
-
-            if objType is MultiIndex:
-                result = object.__new__(pd.MultiIndex)
-                result._set_levels(obj.levels, copy=obj.copy, validate=False)
-                result._set_codes(obj.codes, copy=obj.copy, validate=False)
-                result._set_names(obj.names)
-                result.sortorder = obj.sortorder
-                return result
-
-            if objType is FrozenList:
-                return pdFrozenList(obj)
-
-            return obj
-
-        def g(*args, **kwargs):
-
-            args = tuple([unwrap_index(x) for x in args])
-            result = f(*args, **kwargs)
-            _, result = try_wrap_as_index(result)
-            return result
-
-        g.__name__ = f.__name__
-        return g
-
-    # We allow the wraopping of slot methods that are not inherited from object
-    # It will include operation methods like __add__ and __contains__
-    allow_list = set(x for x in dir(klass) if x.startswith('__')) - set(dir(object))
-
-    # Wrap class members of the newly created class
-    for name, member in getmembers(klass):
-        if name.startswith('_') and name not in allow_list:
-            continue
-
-        if isfunction(member):
-            if name == 'union':
-                member = wrap_union(member)
-            else:
-                member = wrap_function(member)
-            setattr(klass, name, member)
-
-        elif type(member) is property:
-            if type(member.fget) is partial:
-                func = CreateWrapperClass(member.fget.func)
-                fget = partial(func, name)
-            else:
-                fget = wrap_function(member.fget)
-            member = property(fget, member.fset, member.fdel, member.__doc__)
-            setattr(klass, name, member)
-
-    return klass
-
-FrozenList = CreateWrapperClass(pdFrozenList)
-Index = CreateWrapperClass(pd.Index)
-MultiIndex = CreateWrapperClass(pd.MultiIndex)
-Series = CreateWrapperClass(pd.Series)
-DataFrame = CreateWrapperClass(pd.DataFrame)
-
-wrap_special_function('groupby', Series, SeriesGroupBy)
-wrap_special_function('groupby', DataFrame, DataFrameGroupBy)
-wrap_special_function('ewm', Series, EWM)
-wrap_special_function('ewm', DataFrame, EWM)
-wrap_special_function('expanding', Series, Expanding)
-wrap_special_function('expanding', DataFrame, Expanding)
-wrap_special_function('rolling', Series, Rolling, Window)
-wrap_special_function('rolling', DataFrame, Rolling, Window)
-
-CreateSeries = pd.Series
-
-setattr(modules[__name__], 'concat', wrap_function(pd.concat))");
-                }
-            }
-
-            var enumerable = data as IEnumerable;
-            if (enumerable != null)
-            {
-                foreach (var item in enumerable)
-                {
-                    data = item;
-                }
-            }
-
-            var type = data.GetType();
-            IsCustomData = type.Namespace != typeof(Bar).Namespace;
-            _members = new List<MemberInfo>();
-            _symbol = ((IBaseData)data).Symbol;
-
-            if (_symbol.SecurityType == SecurityType.Future) Levels = 3;
-            if (_symbol.SecurityType == SecurityType.Option || _symbol.SecurityType == SecurityType.FutureOption) Levels = 5;
-
-            var columns = new HashSet<string>
-            {
-                   "open",    "high",    "low",    "close", "lastprice",  "volume",
-                "askopen", "askhigh", "asklow", "askclose",  "askprice", "asksize", "quantity", "suspicious",
-                "bidopen", "bidhigh", "bidlow", "bidclose",  "bidprice", "bidsize", "exchange", "openinterest"
-            };
-
-            if (IsCustomData)
-            {
-                var keys = (data as DynamicData)?.GetStorageDictionary().ToHashSet(x => x.Key);
-
-                // C# types that are not DynamicData type
-                if (keys == null)
-                {
-                    if (_membersByType.TryGetValue(type, out _members))
-                    {
-                        keys = _members.ToHashSet(x => x.Name.ToLowerInvariant());
-                    }
-                    else
-                    {
-                        var members = type.GetMembers().Where(x => x.MemberType == MemberTypes.Field || x.MemberType == MemberTypes.Property).ToList();
-
-                        var duplicateKeys = members.GroupBy(x => x.Name.ToLowerInvariant()).Where(x => x.Count() > 1).Select(x => x.Key);
-                        foreach (var duplicateKey in duplicateKeys)
-                        {
-                            throw new ArgumentException($"PandasData.ctor(): More than one \'{duplicateKey}\' member was found in \'{type.FullName}\' class.");
-                        }
-
-                        // If the custom data derives from a Market Data (e.g. Tick, TradeBar, QuoteBar), exclude its keys
-                        keys = members.ToHashSet(x => x.Name.ToLowerInvariant());
-                        keys.ExceptWith(_baseDataProperties);
-                        keys.ExceptWith(GetPropertiesNames(typeof(QuoteBar), type));
-                        keys.ExceptWith(GetPropertiesNames(typeof(TradeBar), type));
-                        keys.ExceptWith(GetPropertiesNames(typeof(Tick), type));
-                        keys.Add("value");
-
-                        _members = members.Where(x => keys.Contains(x.Name.ToLowerInvariant())).ToList();
-                        _membersByType.TryAdd(type, _members);
-                    }
-                }
-
-                columns.Add("value");
-                columns.UnionWith(keys);
-            }
-
-            _series = columns.ToDictionary(k => k, v => Tuple.Create(new List<DateTime>(), new List<object>()));
-        }
-
-        /// <summary>
-        /// Adds security data object to the end of the lists
-        /// </summary>
-        /// <param name="baseData"><see cref="IBaseData"/> object that contains security data</param>
-        public void Add(object baseData)
-        {
-            foreach (var member in _members)
-            {
-                var key = member.Name.ToLowerInvariant();
-                var endTime = ((IBaseData) baseData).EndTime;
-                var propertyMember = member as PropertyInfo;
-                if (propertyMember != null)
-                {
-                    AddToSeries(key, endTime, propertyMember.GetValue(baseData));
-                    continue;
-                }
-                var fieldMember = member as FieldInfo;
-                if (fieldMember != null)
-                {
-                    AddToSeries(key, endTime, fieldMember.GetValue(baseData));
-                }
-            }
-
-            var storage = (baseData as DynamicData)?.GetStorageDictionary();
-            if (storage != null)
-            {
-                var endTime = ((IBaseData) baseData).EndTime;
-                var value = ((IBaseData) baseData).Value;
-                AddToSeries("value", endTime, value);
-
-                foreach (var kvp in storage)
-                {
-                    AddToSeries(kvp.Key, endTime, kvp.Value);
-                }
-            }
-            else
-            {
-                var ticks = new List<Tick> { baseData as Tick };
-                var tradeBar = baseData as TradeBar;
-                var quoteBar = baseData as QuoteBar;
-                Add(ticks, tradeBar, quoteBar);
-            }
-        }
-
-        /// <summary>
-        /// Adds Lean data objects to the end of the lists
-        /// </summary>
-        /// <param name="ticks">List of <see cref="Tick"/> object that contains tick information of the security</param>
-        /// <param name="tradeBar"><see cref="TradeBar"/> object that contains trade bar information of the security</param>
-        /// <param name="quoteBar"><see cref="QuoteBar"/> object that contains quote bar information of the security</param>
-        public void Add(IEnumerable<Tick> ticks, TradeBar tradeBar, QuoteBar quoteBar)
-        {
-            if (tradeBar != null)
-            {
-                var time = tradeBar.EndTime;
-                AddToSeries("open", time, tradeBar.Open);
-                AddToSeries("high", time, tradeBar.High);
-                AddToSeries("low", time, tradeBar.Low);
-                AddToSeries("close", time, tradeBar.Close);
-                AddToSeries("volume", time, tradeBar.Volume);
-            }
-            if (quoteBar != null)
-            {
-                var time = quoteBar.EndTime;
-                if (tradeBar == null)
-                {
-                    AddToSeries("open", time, quoteBar.Open);
-                    AddToSeries("high", time, quoteBar.High);
-                    AddToSeries("low", time, quoteBar.Low);
-                    AddToSeries("close", time, quoteBar.Close);
-                }
-                if (quoteBar.Ask != null)
-                {
-                    AddToSeries("askopen", time, quoteBar.Ask.Open);
-                    AddToSeries("askhigh", time, quoteBar.Ask.High);
-                    AddToSeries("asklow", time, quoteBar.Ask.Low);
-                    AddToSeries("askclose", time, quoteBar.Ask.Close);
-                    AddToSeries("asksize", time, quoteBar.LastAskSize);
-                }
-                if (quoteBar.Bid != null)
-                {
-                    AddToSeries("bidopen", time, quoteBar.Bid.Open);
-                    AddToSeries("bidhigh", time, quoteBar.Bid.High);
-                    AddToSeries("bidlow", time, quoteBar.Bid.Low);
-                    AddToSeries("bidclose", time, quoteBar.Bid.Close);
-                    AddToSeries("bidsize", time, quoteBar.LastBidSize);
-                }
-            }
-            if (ticks != null)
-            {
-                foreach (var tick in ticks)
-                {
-                    if (tick == null) continue;
-
-                    var time = tick.EndTime;
-                    var column = tick.TickType == TickType.OpenInterest
-                        ? "openinterest"
-                        : "lastprice";
-
-                    if (tick.TickType == TickType.Quote)
-                    {
-                        AddToSeries("askprice", time, tick.AskPrice);
-                        AddToSeries("asksize", time, tick.AskSize);
-                        AddToSeries("bidprice", time, tick.BidPrice);
-                        AddToSeries("bidsize", time, tick.BidSize);
-                    }
-                    AddToSeries("exchange", time, tick.Exchange);
-                    AddToSeries("suspicious", time, tick.Suspicious);
-                    AddToSeries("quantity", time, tick.Quantity);
-                    AddToSeries(column, time, tick.LastPrice);
-                }
-            }
-        }
-
-        /// <summary>
-        /// Get the pandas.DataFrame of the current <see cref="PandasData"/> state
-        /// </summary>
-        /// <param name="levels">Number of levels of the multi index</param>
-        /// <returns>pandas.DataFrame object</returns>
-        public PyObject ToPandasDataFrame(int levels = 2)
-        {
-            var empty = new PyString(string.Empty);
-            var list = Enumerable.Repeat<PyObject>(empty, 5).ToList();
-            list[3] = _symbol.ID.ToString().ToPython();
-
-            if (_symbol.SecurityType == SecurityType.Future)
-            {
-                list[0] = _symbol.ID.Date.ToPython();
-                list[3] = _symbol.ID.ToString().ToPython();
-            }
-            if (_symbol.SecurityType == SecurityType.Option || _symbol.SecurityType == SecurityType.FutureOption)
-            {
-                list[0] = _symbol.ID.Date.ToPython();
-                list[1] = _symbol.ID.StrikePrice.ToPython();
-                list[2] = _symbol.ID.OptionRight.ToString().ToPython();
-                list[3] = _symbol.ID.ToString().ToPython();
-            }
-
-            // Create the index labels
-            var names = "expiry,strike,type,symbol,time";
-            if (levels == 2)
-            {
-                names = "symbol,time";
-                list.RemoveRange(0, 3);
-            }
-            if (levels == 3)
-            {
-                names = "expiry,symbol,time";
-                list.RemoveRange(1, 2);
-            }
-
-            Func<object, bool> filter = x =>
-            {
-                var isNaNOrZero = x is double && ((double)x).IsNaNOrZero();
-                var isNullOrWhiteSpace = x is string && string.IsNullOrWhiteSpace((string)x);
-                var isFalse = x is bool && !(bool)x;
-                return x == null || isNaNOrZero || isNullOrWhiteSpace || isFalse;
-            };
-            Func<DateTime, PyTuple> selector = x =>
-            {
-                list[list.Count - 1] = x.ToPython();
-                return new PyTuple(list.ToArray());
-            };
-            // creating the pandas MultiIndex is expensive so we keep a cash
-            var indexCache = new Dictionary<List<DateTime>, dynamic>(new ListComparer<DateTime>());
-            using (Py.GIL())
-            {
-                // Returns a dictionary keyed by column name where values are pandas.Series objects
-                var pyDict = new PyDict();
-                var splitNames = names.Split(',');
-                foreach (var kvp in _series)
-                {
-                    var values = kvp.Value.Item2;
-                    if (values.All(filter)) continue;
-
-                    dynamic index;
-                    if (!indexCache.TryGetValue(kvp.Value.Item1, out index))
-                    {
-                        var tuples = kvp.Value.Item1.Select(selector).ToArray();
-                        index = _pandas.MultiIndex.from_tuples(tuples, names: splitNames);
-                        indexCache[kvp.Value.Item1] = index;
-                    }
-
-                    // Adds pandas.Series value keyed by the column name
-                    // CreateSeries will create an original pandas.Series
-                    // We are not using the wrapper class to avoid unnecessary and expensive
-                    // index wrapping operations when the Series are packed into a DataFrame
-                    pyDict.SetItem(kvp.Key, _pandas.CreateSeries(values, index));
-                }
-                _series.Clear();
-
-                // Create a DataFrame with wrapper class.
-                // This is the starting point. The types of all DataFrame and Series that result from any operation will
-                // be wrapper classes. Index and MultiIndex will be converted when required by index operations such as
-                // stack, unstack, merge, union, etc.
-                return _pandas.DataFrame(pyDict);
-            }
-        }
-
-        /// <summary>
-        /// Adds data to dictionary
-        /// </summary>
-        /// <param name="key">The key of the value to get</param>
-        /// <param name="time"><see cref="DateTime"/> object to add to the value associated with the specific key</param>
-        /// <param name="input"><see cref="Object"/> to add to the value associated with the specific key. Can be null.</param>
-        private void AddToSeries(string key, DateTime time, object input)
-        {
-            Tuple<List<DateTime>, List<object>> value;
-            if (_series.TryGetValue(key, out value))
-            {
-                value.Item1.Add(time);
-                value.Item2.Add(input is decimal ? input.ConvertInvariant<double>() : input);
-            }
-            else
-            {
-                throw new ArgumentException($"PandasData.AddToSeries(): {key} key does not exist in series dictionary.");
-            }
-        }
-
-        /// <summary>
-        /// Get the lower-invariant name of properties of the type that a another type is assignable from
-        /// </summary>
-        /// <param name="baseType">The type that is assignable from</param>
-        /// <param name="type">The type that is assignable by</param>
-        /// <returns>List of string. Empty list if not assignable from</returns>
-        private static IEnumerable<string> GetPropertiesNames(Type baseType, Type type)
-        {
-            return baseType.IsAssignableFrom(type)
-                ? baseType.GetProperties().Select(x => x.Name.ToLowerInvariant())
-                : Enumerable.Empty<string>();
-        }
-    }
-}

Common/QuantConnect.csproj

@@ -25,6 +25,7 @@
     <NuGetPackageImportStamp>
     </NuGetPackageImportStamp>
     <CodeAnalysisRuleSet>..\QuantConnect.ruleset</CodeAnalysisRuleSet>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
     <DebugSymbols>true</DebugSymbols>
@@ -106,6 +107,9 @@
     <CodeAnalysisRuleSet>..\QuantConnect.ruleset</CodeAnalysisRuleSet>
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="Apache.Arrow, Version=2.0.0.0, Culture=neutral, PublicKeyToken=204f54e5a45c07df, processorArchitecture=MSIL">
+      <HintPath>..\packages\gsalaz98.Unofficial.Apache.Arrow.2.0.1-SNAPSHOT\lib\net461\Apache.Arrow.dll</HintPath>
+    </Reference>
     <Reference Include="CloneExtensions, Version=1.3.0.0, Culture=neutral, processorArchitecture=MSIL">
       <HintPath>..\packages\CloneExtensions.1.3.0\lib\net461\CloneExtensions.dll</HintPath>
     </Reference>
@@ -124,6 +128,11 @@
     <Reference Include="Microsoft.IO.RecyclableMemoryStream, Version=1.3.5.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <HintPath>..\packages\Microsoft.IO.RecyclableMemoryStream.1.3.5\lib\net46\Microsoft.IO.RecyclableMemoryStream.dll</HintPath>
     </Reference>
+    <Reference Include="Microsoft.Win32.Primitives, Version=4.0.2.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
+      <HintPath>..\packages\Microsoft.Win32.Primitives.4.3.0\lib\net46\Microsoft.Win32.Primitives.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
+    <Reference Include="mscorlib" />
     <Reference Include="Newtonsoft.Json, Version=10.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
       <HintPath>..\packages\Newtonsoft.Json.10.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
     </Reference>
@@ -166,6 +175,10 @@
     <Reference Include="System.ServiceModel.Primitives, Version=4.7.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
       <HintPath>..\packages\System.ServiceModel.Primitives.4.7.0\lib\net461\System.ServiceModel.Primitives.dll</HintPath>
     </Reference>
+    <Reference Include="System.Threading.Tasks.Extensions, Version=4.2.0.1, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51">
+      <HintPath>..\packages\System.Threading.Tasks.Extensions.4.5.4\lib\net461\System.Threading.Tasks.Extensions.dll</HintPath>
+      <Private>True</Private>
+    </Reference>
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />
     <Reference Include="Microsoft.CSharp" />
@@ -401,6 +414,7 @@
     <Compile Include="Packets\LeakyBucketControlParameters.cs" />
     <Compile Include="Packets\LiveResultParameters.cs" />
     <Compile Include="Python\BrokerageMessageHandlerPythonWrapper.cs" />
+    <Compile Include="Python\PandasArrowMemoryAllocator.cs" />
     <Compile Include="Python\PythonConsolidator.cs" />
     <Compile Include="Python\MarginCallModelPythonWrapper.cs" />
     <Compile Include="Python\PythonInitializer.cs" />
@@ -515,7 +529,6 @@
     <Compile Include="Python\PythonActivator.cs" />
     <Compile Include="Python\PythonSlice.cs" />
     <Compile Include="Python\VolatilityModelPythonWrapper.cs" />
-    <Compile Include="Python\PandasData.cs" />
     <Compile Include="Python\SecurityInitializerPythonWrapper.cs" />
     <Compile Include="Python\SlippageModelPythonWrapper.cs" />
     <Compile Include="Python\FillModelPythonWrapper.cs" />
@@ -935,9 +948,6 @@
   </ItemGroup>
   <ItemGroup>
     <None Include="packages.config" />
-    <None Include="Python\Python.Runtime.dll.config" Link="Python.Runtime.dll.config">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
   </ItemGroup>
   <ItemGroup>
     <Content Include="decimal.py">
@@ -975,4 +985,4 @@
   <Target Name="AfterBuild">
   </Target>
   -->
-</Project>
+</Project>

Common/SecurityIdentifier.cs

@@ -240,7 +240,7 @@ namespace QuantConnect
                 {
                     if (SecurityType != SecurityType.Option && SecurityType != SecurityType.FutureOption)
                     {
-                        throw new InvalidOperationException("StrikePrice is only defined for SecurityType.Option and SecurityType.FutureOption");
+                        throw new InvalidOperationException("OptionType is only defined for SecurityType.Option and SecurityType.FutureOption");
                     }
 
                     // performance: lets calculate strike price once

Common/SeriesSampler.cs

@@ -41,9 +41,8 @@ namespace QuantConnect
         /// <param name="series">The series to be sampled</param>
         /// <param name="start">The date to start sampling, if before start of data then start of data will be used</param>
         /// <param name="stop">The date to stop sampling, if after stop of data, then stop of data will be used</param>
-        /// <param name="truncateValues">True will truncate values to integers</param>
         /// <returns>The sampled series</returns>
-        public Series Sample(Series series, DateTime start, DateTime stop, bool truncateValues = false)
+        public Series Sample(Series series, DateTime start, DateTime stop)
         {
             var sampled = new Series(series.Name, series.SeriesType, series.Index, series.Unit);
 
@@ -60,13 +59,7 @@ namespace QuantConnect
                 {
                     if (point.x >= nextSample && point.x <= unixStopDate)
                     {
-                        var samplePoint = point;
-                        if (truncateValues)
-                        {
-                            // let's not modify the original
-                            samplePoint = new ChartPoint(samplePoint) { y = Math.Truncate(samplePoint.y) };
-                        }
-                        sampled.Values.Add(samplePoint);
+                        sampled.Values.Add(point);
                     }
                 }
                 return sampled;
@@ -113,12 +106,7 @@ namespace QuantConnect
                 while (nextSample <= current.x && nextSample <= unixStopDate)
                 {
                     var value = Interpolate(previous, current, (long) nextSample);
-                    var point = new ChartPoint {x = (long) nextSample, y = value};
-                    if (truncateValues)
-                    {
-                        point.y = Math.Truncate(point.y);
-                    }
-                    sampled.Values.Add(point);
+                    sampled.Values.Add(new ChartPoint {x = (long) nextSample, y = value});
                     nextSample += _seconds;
                 }
 

Common/Symbol.cs

@@ -307,10 +307,6 @@ namespace QuantConnect
                 throw new ArgumentNullException(nameof(value));
             }
             ID = sid;
-            if (ID.HasUnderlying)
-            {
-                Underlying = new Symbol(ID.Underlying, ID.Underlying.Symbol);
-            }
 
             Value = value.LazyToUpper();
         }

Common/packages.config

@@ -3,6 +3,7 @@
   <package id="CloneExtensions" version="1.3.0" targetFramework="net462" />
   <package id="DotNetZip" version="1.13.3" targetFramework="net452" />
   <package id="fasterflect" version="2.1.3" targetFramework="net45" />
+  <package id="gsalaz98.Unofficial.Apache.Arrow" version="2.0.1-SNAPSHOT" targetFramework="net462" />
   <package id="MathNet.Numerics" version="3.19.0" targetFramework="net452" />
   <package id="Microsoft.CodeAnalysis.FxCopAnalyzers" version="2.9.3" targetFramework="net452" />
   <package id="Microsoft.CodeAnalysis.VersionCheckAnalyzer" version="2.9.3" targetFramework="net452" />
@@ -24,4 +25,5 @@
   <package id="System.Reflection.Emit.Lightweight" version="4.3.0" targetFramework="net452" />
   <package id="System.Runtime.CompilerServices.Unsafe" version="4.5.3" targetFramework="net462" />
   <package id="System.ServiceModel.Primitives" version="4.7.0" targetFramework="net462" />
+  <package id="System.Threading.Tasks.Extensions" version="4.5.4" targetFramework="net462" />
 </packages>

DockerfileLeanFoundation

@@ -66,6 +66,9 @@ RUN conda install -y                \
     pandas=0.25.3                   \
     wrapt=1.12.1
 
+# Install pyarrow to improve DataFrame construction performance
+RUN pip install pyarrow==1.0.1
+
 # Install non-math packages
 RUN conda install -y                \
     astropy=4.0.1.post1             \
@@ -242,4 +245,4 @@ RUN conda remove --force-remove -y s3transfer
 RUN conda clean -y --all
 
 # List all packages
-RUN conda list
+RUN conda list

Engine/AlgorithmManager.cs

@@ -1030,26 +1030,19 @@ namespace QuantConnect.Lean.Engine
         {
             for (var i = delistings.Count - 1; i >= 0; i--)
             {
-                var delisting = delistings[i];
-                var security = algorithm.Securities[delisting.Symbol];
+                // check if we are holding position
+                var security = algorithm.Securities[delistings[i].Symbol];
                 if (security.Holdings.Quantity == 0)
                 {
                     continue;
                 }
 
-                var delistingTime = delisting.Time;
-                if (!security.Exchange.Hours.IsOpen(delistingTime, false))
-                {
-                    // This exists as a defensive measure to ensure that the delisting time
-                    // does not get moved if the market is open. If the market is closed,
-                    // we get the next market open, which will be on the same day if the delisting
-                    // date is a trading day. If the delisting date is after market close, then
-                    // the delisting will be adjusted to the next market open.
-                    delistingTime = security.Exchange.Hours.GetNextMarketOpen(delistingTime, false);
-                    delistingTime = security.Exchange.Hours.GetNextMarketClose(delistingTime, false);
-                }
+                // check if the time has come for delisting
+                var delistingTime = delistings[i].Time;
+                var nextMarketOpen = security.Exchange.Hours.GetNextMarketOpen(delistingTime, false);
+                var nextMarketClose = security.Exchange.Hours.GetNextMarketClose(nextMarketOpen, false);
 
-                if (security.LocalTime < delistingTime)
+                if (security.LocalTime < nextMarketClose)
                 {
                     continue;
                 }

Engine/DataFeeds/ApiDataProvider.cs

@@ -31,7 +31,6 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         private readonly int _uid = Config.GetInt("job-user-id", 0);
         private readonly string _token = Config.Get("api-access-token", "1");
         private readonly string _dataPath = Config.Get("data-folder", "../../../Data/");
-        private static readonly int DownloadPeriod = Config.GetInt("api-data-update-period", 5);
         private readonly Api.Api _api;
 
         /// <summary>
@@ -52,70 +51,40 @@ namespace QuantConnect.Lean.Engine.DataFeeds
         /// <returns>A <see cref="Stream"/> of the data requested</returns>
         public Stream Fetch(string key)
         {
+            if (File.Exists(key))
+            {
+                return new FileStream(key, FileMode.Open, FileAccess.Read, FileShare.Read);
+            }
+
+            // If the file cannot be found on disc, attempt to retrieve it from the API
             Symbol symbol;
             DateTime date;
             Resolution resolution;
 
-            // Fetch the details of this data request
             if (LeanData.TryParsePath(key, out symbol, out date, out resolution))
             {
-                if (!File.Exists(key) || IsOutOfDate(resolution, key))
-                {
-                    return DownloadData(key, symbol, date, resolution);
-                }
-
-                // Use the file already on the disk
-                return new FileStream(key, FileMode.Open, FileAccess.Read, FileShare.Read);
-            }
-
-            Log.Error("ApiDataProvider.Fetch(): failed to parse key {0}", key);
-            return null;
-        }
-
-        /// <summary>
-        /// Determine if the file is out of date based on configuration and needs to be updated
-        /// </summary>
-        /// <param name="resolution">Data resolution</param>
-        /// <param name="filepath">Path to the file</param>
-        /// <returns>True if the file is out of date</returns>
-        /// <remarks>Files are only "out of date" for Hourly/Daily data because this data is stored all in one file</remarks>
-        public static bool IsOutOfDate(Resolution resolution, string filepath)
-        {
-            return resolution >= Resolution.Hour &&
-                (DateTime.Now - TimeSpan.FromDays(DownloadPeriod)) > File.GetLastWriteTime(filepath);
-        }
-
-        /// <summary>
-        /// Attempt to download data using the Api for and return a FileStream of that data.
-        /// </summary>
-        /// <param name="key"></param>
-        /// <param name="symbol"></param>
-        /// <param name="date"></param>
-        /// <param name="resolution"></param>
-        /// <returns>A FileStream of the data</returns>
-        private FileStream DownloadData(string filepath, Symbol symbol, DateTime date, Resolution resolution)
-        {
-            Log.Trace("ApiDataProvider.Fetch(): Attempting to get data from QuantConnect.com's data library for symbol({0}), resolution({1}) and date({2}).",
-                symbol.Value,
-                resolution,
-                date.Date.ToShortDateString());
-
-            var downloadSuccessful = _api.DownloadData(symbol, resolution, date);
-
-            if (downloadSuccessful)
-            {
-                Log.Trace("ApiDataProvider.Fetch(): Successfully retrieved data for symbol({0}), resolution({1}) and date({2}).",
+                Log.Trace("ApiDataProvider.Fetch(): Attempting to get data from QuantConnect.com's data library for symbol({0}), resolution({1}) and date({2}).",
                     symbol.Value,
                     resolution,
                     date.Date.ToShortDateString());
 
-                return new FileStream(filepath, FileMode.Open, FileAccess.Read, FileShare.Read);
+                var downloadSuccessful = _api.DownloadData(symbol, resolution, date);
+
+                if (downloadSuccessful)
+                {
+                    Log.Trace("ApiDataProvider.Fetch(): Successfully retrieved data for symbol({0}), resolution({1}) and date({2}).",
+                        symbol.Value,
+                        resolution,
+                        date.Date.ToShortDateString());
+
+                    return new FileStream(key, FileMode.Open, FileAccess.Read, FileShare.Read);
+                }
             }
 
-            // Failed to download
             Log.Error("ApiDataProvider.Fetch(): Unable to remotely retrieve data for path {0}. " +
-                "Please make sure you have the necessary data in your online QuantConnect data library.",
-                filepath);
+                      "Please make sure you have the necessary data in your online QuantConnect data library.",
+                       key);
+
             return null;
         }
     }

Engine/DataFeeds/TimeSliceFactory.cs

@@ -15,6 +15,7 @@
 */
 
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using NodaTime;
 using QuantConnect.Data;
@@ -31,19 +32,20 @@ namespace QuantConnect.Lean.Engine.DataFeeds
     public class TimeSliceFactory
     {
         private readonly DateTimeZone _timeZone;
+        private static readonly ConcurrentDictionary<Symbol, Symbol> _canonicalSymbols = new ConcurrentDictionary<Symbol,Symbol>();
 
         // performance: these collections are not always used so keep a reference to an empty
         // instance to use and avoid unnecessary constructors and allocations
-        private readonly List<UpdateData<ISecurityPrice>> _emptyCustom = new List<UpdateData<ISecurityPrice>>();
-        private readonly TradeBars _emptyTradeBars = new TradeBars();
-        private readonly QuoteBars _emptyQuoteBars = new QuoteBars();
-        private readonly Ticks _emptyTicks = new Ticks();
-        private readonly Splits _emptySplits = new Splits();
-        private readonly Dividends _emptyDividends = new Dividends();
-        private readonly Delistings _emptyDelistings = new Delistings();
-        private readonly OptionChains _emptyOptionChains = new OptionChains();
-        private readonly FuturesChains _emptyFuturesChains = new FuturesChains();
-        private readonly SymbolChangedEvents _emptySymbolChangedEvents = new SymbolChangedEvents();
+        private static List<UpdateData<ISecurityPrice>> _emptyCustom = new List<UpdateData<ISecurityPrice>>(0);
+        private static TradeBars _emptyTradeBars = new TradeBars();
+        private static QuoteBars _emptyQuoteBars = new QuoteBars();
+        private static Ticks _emptyTicks = new Ticks();
+        private static Splits _emptySplits = new Splits();
+        private static Dividends _emptyDividends = new Dividends();
+        private static Delistings _emptyDelistings = new Delistings();
+        private static OptionChains _emptyOptionChains = new OptionChains();
+        private static FuturesChains _emptyFuturesChains = new FuturesChains();
+        private static SymbolChangedEvents _emptySymbolChangedEvents = new SymbolChangedEvents();
 
         /// <summary>
         /// Creates a new instance
@@ -118,8 +120,6 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             FuturesChains futuresChains = null;
             SymbolChangedEvents symbolChanges = null;
 
-            UpdateEmptyCollections(algorithmTime);
-
             if (universeData.Count > 0)
             {
                 // count universe data
@@ -362,36 +362,19 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             return new TimeSlice(utcDateTime, count, slice, data, security, consolidator, custom ?? _emptyCustom, changes, universeData);
         }
 
-        private void UpdateEmptyCollections(DateTime algorithmTime)
-        {
-            // just in case
-            _emptyTradeBars.Clear();
-            _emptyQuoteBars.Clear();
-            _emptyTicks.Clear();
-            _emptySplits.Clear();
-            _emptyDividends.Clear();
-            _emptyDelistings.Clear();
-            _emptyOptionChains.Clear();
-            _emptyFuturesChains.Clear();
-            _emptySymbolChangedEvents.Clear();
-
-            _emptyTradeBars.Time
-                = _emptyQuoteBars.Time
-                = _emptyTicks.Time
-                = _emptySplits.Time
-                = _emptyDividends.Time
-                = _emptyDelistings.Time
-                = _emptyOptionChains.Time
-                = _emptyFuturesChains.Time
-                = _emptySymbolChangedEvents.Time = algorithmTime;
-        }
-
         private bool HandleOptionData(DateTime algorithmTime, BaseData baseData, OptionChains optionChains, ISecurityPrice security, Lazy<Slice> sliceFuture, IReadOnlyDictionary<Symbol, BaseData> optionUnderlyingUpdates)
         {
             var symbol = baseData.Symbol;
 
             OptionChain chain;
-            var canonical = Symbol.CreateOption(symbol.Underlying, symbol.ID.Market, default(OptionStyle), default(OptionRight), 0, SecurityIdentifier.DefaultDate);
+            Symbol canonical;
+
+            if (!_canonicalSymbols.TryGetValue(symbol, out canonical))
+            {
+                canonical = Symbol.CreateOption(symbol.Underlying, symbol.ID.Market, default(OptionStyle), default(OptionRight), 0, SecurityIdentifier.DefaultDate);
+                _canonicalSymbols.TryAdd(symbol, canonical);
+            }
+
             if (!optionChains.TryGetValue(canonical, out chain))
             {
                 chain = new OptionChain(canonical, algorithmTime);
@@ -498,7 +481,14 @@ namespace QuantConnect.Lean.Engine.DataFeeds
             var symbol = baseData.Symbol;
 
             FuturesChain chain;
-            var canonical = Symbol.Create(symbol.ID.Symbol, SecurityType.Future, symbol.ID.Market);
+            Symbol canonical;
+
+            if (!_canonicalSymbols.TryGetValue(symbol, out canonical))
+            {
+                canonical = Symbol.Create(symbol.ID.Symbol, SecurityType.Future, symbol.ID.Market);
+                _canonicalSymbols.TryAdd(symbol, canonical);
+            }
+
             if (!futuresChains.TryGetValue(canonical, out chain))
             {
                 chain = new FuturesChain(canonical, algorithmTime);

Engine/Results/BacktestingResultHandler.cs

@@ -653,7 +653,7 @@ namespace QuantConnect.Lean.Engine.Results
         /// <param name="message">Additional optional status message.</param>
         public virtual void SendStatusUpdate(AlgorithmStatus status, string message = "")
         {
-            var statusPacket = new AlgorithmStatusPacket(_algorithmId, _projectId, status, message) { OptimizationId = _job.OptimizationId };
+            var statusPacket = new AlgorithmStatusPacket(_algorithmId, _projectId, status, message);
             MessagingHandler.Send(statusPacket);
         }
 

Engine/Results/BaseResultsHandler.cs

@@ -42,8 +42,6 @@ namespace QuantConnect.Lean.Engine.Results
         private static readonly TextWriter StandardOut = Console.Out;
         private static readonly TextWriter StandardError = Console.Error;
 
-        private string _hostName;
-
         /// <summary>
         /// The main loop update interval
         /// </summary>
@@ -234,7 +232,6 @@ namespace QuantConnect.Lean.Engine.Results
         protected virtual Dictionary<string, string> GetServerStatistics(DateTime utcNow)
         {
             var serverStatistics = OS.GetServerStatistics();
-            serverStatistics["Hostname"] = _hostName;
             var upTime = utcNow - StartTime;
             serverStatistics["Up Time"] = $"{upTime.Days}d {upTime:hh\\:mm\\:ss}";
             serverStatistics["Total RAM (MB)"] = RamAllocation;
@@ -308,7 +305,6 @@ namespace QuantConnect.Lean.Engine.Results
         /// <param name="transactionHandler">The transaction handler used to get the algorithms <see cref="Order"/> information</param>
         public virtual void Initialize(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, ITransactionHandler transactionHandler)
         {
-            _hostName = job.HostName ?? Environment.MachineName;
             MessagingHandler = messagingHandler;
             TransactionHandler = transactionHandler;
             CompileId = job.CompileId;

Indicators/AwesomeOscillator.cs

@@ -1,101 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using QuantConnect.Data.Market;
-using System;
-
-namespace QuantConnect.Indicators
-{
-    /// <summary>
-    /// The Awesome Oscillator Indicator tracks the price midpoint-movement of a security. Specifically,
-    /// <para>
-    /// AO = MAfast[(H+L)/2] - MAslow[(H+L)/2] 
-    /// </para>
-    /// where MAfast and MAslow denote simple moving averages wherein fast has a shorter period.
-    /// https://www.barchart.com/education/technical-indicators/awesome_oscillator
-    /// </summary>
-    public class AwesomeOscillator : BarIndicator, IIndicatorWarmUpPeriodProvider
-    {
-        /// <summary>
-        /// Gets the indicators slow period moving average.
-        /// </summary>
-        public IndicatorBase<IndicatorDataPoint> SlowAo { get; }
-
-        /// <summary>
-        /// Gets the indicators fast period moving average.
-        /// </summary>
-        public IndicatorBase<IndicatorDataPoint> FastAo { get; }
-
-        /// <summary>
-        /// Gets a flag indicating when this indicator is ready and fully initialized
-        /// </summary>
-        public override bool IsReady => SlowAo.IsReady && FastAo.IsReady;
-
-        /// <summary>
-        /// Required period, in data points, for the indicator to be ready and fully initialized.
-        /// </summary>
-        public int WarmUpPeriod { get; }
-
-        /// <summary>
-        /// Creates a new Awesome Oscillator from the specified periods.
-        /// </summary>
-        /// <param name="fastPeriod">The period of the fast moving average associated with the AO</param>
-        /// <param name="slowPeriod">The period of the slow moving average associated with the AO</param>
-        /// <param name="type">The type of moving average used when computing the fast and slow term. Defaults to simple moving average.</param>
-        public AwesomeOscillator(int fastPeriod, int slowPeriod, MovingAverageType type=MovingAverageType.Simple)
-            : this($"AO({fastPeriod},{slowPeriod},{type})", fastPeriod, slowPeriod, type)
-        {
-        }
-
-        /// <summary>
-        /// Creates a new Awesome Oscillator from the specified periods.
-        /// </summary>
-        /// <param name="name">The name of this indicator</param>
-        /// <param name="fastPeriod">The period of the fast moving average associated with the AO</param>
-        /// <param name="slowPeriod">The period of the slow moving average associated with the AO</param>
-        /// <param name="type">The type of moving average used when computing the fast and slow term. Defaults to simple moving average.</param>
-        public AwesomeOscillator(string name, int fastPeriod, int slowPeriod,  MovingAverageType type=MovingAverageType.Simple)
-            : base(name)
-        {
-            SlowAo = type.AsIndicator(slowPeriod);
-            FastAo = type.AsIndicator(fastPeriod);
-            WarmUpPeriod = Math.Max(slowPeriod, fastPeriod);
-        }
-
-        /// <summary>
-        /// Computes the next value of this indicator from the given state.
-        /// </summary>
-        /// <param name="input">The input given to the indicator</param>
-        /// <returns>A new value for this indicator</returns>
-        protected override decimal ComputeNextValue(IBaseDataBar input)
-        {
-            var presentValue = (input.High + input.Low) / 2;
-            SlowAo.Update(input.Time, presentValue);
-            FastAo.Update(input.Time, presentValue);
-
-            return IsReady ? FastAo - SlowAo : 0m;
-        }
-
-        /// <summary>
-        /// Resets this indicator 
-        /// </summary>
-        public override void Reset()
-        {
-            FastAo.Reset();
-            SlowAo.Reset();
-            base.Reset();
-        }
-    }
-}

Indicators/ChaikinMoneyFlow.cs

@@ -1,95 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using QuantConnect.Data.Market;
-
-namespace QuantConnect.Indicators
-{
-    /// <summary>
-    /// The Chaikin Money Flow Index (CMF) is a volume-weighted average of accumulation and distribution over
-    /// a specified period.
-    ///
-    /// CMF = n-day Sum of [(((C - L) - (H - C)) / (H - L)) x Vol] / n-day Sum of Vol
-    ///
-    /// Where:
-    /// n = number of periods, typically 21
-    /// H = high
-    /// L = low
-    /// C = close
-    /// Vol = volume
-    /// 
-    /// https://www.fidelity.com/learning-center/trading-investing/technical-analysis/technical-indicator-guide/cmf
-    /// </summary>
-    public class ChaikinMoneyFlow : TradeBarIndicator, IIndicatorWarmUpPeriodProvider
-    {
-        /// <summary>
-        /// Holds the point-wise flow-sum and volume terms. 
-        /// </summary>
-        private readonly Sum _flowRatioSum;
-
-        private readonly Sum _volumeSum;
-
-        /// <summary>
-        /// Gets a flag indicating when this indicator is ready and fully initialized
-        /// </summary>
-        public override bool IsReady => _flowRatioSum.IsReady;
-
-        /// <summary>
-        /// Required period, in data points, for the indicator to be ready and fully initialized.
-        /// </summary>
-        public int WarmUpPeriod { get; }
-
-        /// <summary>
-        /// Resets this indicator to its initial state
-        /// </summary>
-        public override void Reset()
-        {
-            _volumeSum.Reset();
-            _flowRatioSum.Reset();
-            base.Reset();
-        }
-
-        /// <summary>
-        /// Initializes a new instance of the ChaikinMoneyFlow class
-        /// </summary>
-        /// <param name="name">A name for the indicator</param>
-        /// <param name="period">The period over which to perform computation</param>
-        public ChaikinMoneyFlow(string name, int period)
-            : base($"CMF({name})")
-        {
-            WarmUpPeriod = period;
-            _flowRatioSum = new Sum(period);
-            _volumeSum = new Sum(period);
-        }
-
-        /// <summary>
-        /// Computes the next value for this indicator from the given state.
-        /// </summary>
-        /// <param name="input">The input value to this indicator on this time step</param>
-        /// <returns>A new value for this indicator</returns>
-        protected override decimal ComputeNextValue(TradeBar input)
-        {
-            var denominator = (input.High - input.Low);
-            var flowRatio = denominator > 0
-                ? input.Volume * (input.Close - input.Low - (input.High - input.Close)) / denominator
-                : 0m;
-
-            _flowRatioSum.Update(input.EndTime, flowRatio);
-            _volumeSum.Update(input.EndTime, input.Volume);
-
-            return !IsReady || _volumeSum == 0m ? 0m : _flowRatioSum / _volumeSum;
-        }
-    }
-}

Indicators/DeMarkerIndicator.cs

@@ -1,120 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-using System;
-using QuantConnect.Data.Market;
-
-namespace QuantConnect.Indicators
-{
-    /// <summary>
-    /// In the DeMarker strategy, for some period of size N, set:
-    /// <para>
-    /// DeMax = High - Previous High, and 
-    /// DeMin = Previous Low - Low
-    /// </para>
-    /// where, in the prior, if either term is less than zero (DeMax or DeMin), set it to zero.
-    /// We can now define the indicator itself, DEM, as:
-    ///<para>
-    /// DEM = MA(DeMax)/(MA(DeMax)+MA(DeMin))
-    ///</para>
-    /// where MA denotes a Moving Average of period N.
-    /// 
-    /// https://www.investopedia.com/terms/d/demarkerindicator.asp
-    /// </summary>
-    public class DeMarkerIndicator : BarIndicator, IIndicatorWarmUpPeriodProvider
-    {
-        private readonly IndicatorBase<IndicatorDataPoint> _maxMA;
-        private readonly IndicatorBase<IndicatorDataPoint> _minMA;
-        private decimal _lastHigh;
-        private decimal _lastLow;
-
-        /// <summary>
-        /// Initializes a new instance of the DeMarkerIndicator class with the specified period
-        /// </summary>
-        /// <param name="period">The period of the  DeMarker Indicator</param>
-        /// <param name="type">The type of moving average to use in calculations</param>
-        public DeMarkerIndicator(int period, MovingAverageType type = MovingAverageType.Simple)
-            : this($"DEM({period},{type})", period, type)
-        {
-        }
-
-        /// <summary>
-        /// Initializes a new instance of the DeMarkerIndicator class with the specified name and period
-        /// </summary>
-        /// <param name="name">The name of this indicator</param>
-        /// <param name="period">The period of the  DeMarker Indicator</param>
-        /// <param name="type">The type of moving average to use in calculations</param>
-        public DeMarkerIndicator(string name, int period, MovingAverageType type = MovingAverageType.Simple)
-            : base(name)
-        {
-            var _lastHigh = 0m;
-            var _lastLow = 0m;
-            WarmUpPeriod = period;
-            _maxMA = type.AsIndicator(period);
-            _minMA = type.AsIndicator(period);
-        }
-
-        /// <summary>
-        /// Gets a flag indicating when this indicator is ready and fully initialized
-        /// </summary>
-        public override bool IsReady => _maxMA.IsReady && _minMA.IsReady;
-
-        /// <summary>
-        /// Required period, in data points, for the indicator to be ready and fully initialized.
-        /// </summary>
-        public int WarmUpPeriod { get; }
-
-        /// <summary>
-        /// Resets this indicator to its initial state
-        /// </summary>
-        public override void Reset()
-        {
-            _maxMA.Reset();
-            _minMA.Reset();
-            base.Reset();
-        }
-
-
-        /// <summary>
-        /// Computes the next value of this indicator from the given state
-        /// </summary>
-        /// <param name="input">The input given to the indicator</param>
-        /// <returns>A new value for this indicator</returns>
-        protected override decimal ComputeNextValue(IBaseDataBar input)
-        {
-            var deMax = 0m;
-            var deMin = 0m;
-            if (Samples > 1)
-            {
-                // By default, DeMin and DeMax must be 0m initially
-                deMax = Math.Max(input.High - _lastHigh, 0);
-                deMin = Math.Max(_lastLow - input.Low, 0);
-            }
-
-            _maxMA.Update(input.Time, deMax);
-            _minMA.Update(input.Time, deMin);
-            _lastHigh = input.High;
-            _lastLow = input.Low;
-
-            if (!IsReady)
-            {
-                return 0m;
-            }
-
-            var currentValue = _maxMA + _minMA;
-            return currentValue > 0m ? _maxMA / currentValue : 0m;
-        }
-    }
-}

Indicators/IndicatorBase.cs

@@ -120,18 +120,7 @@ namespace QuantConnect.Indicators
                 return Update((T)(object)new IndicatorDataPoint(time, value));
             }
 
-            var suggestions = new List<string>
-            {
-                "Update(TradeBar)",
-                "Update(QuoteBar)"
-            };
-
-            if (typeof(T) == typeof(IBaseData))
-            {
-                suggestions.Add("Update(Tick)");
-            }
-
-            throw new NotSupportedException($"{GetType().Name} does not support the `Update(DateTime, decimal)` method. Use one of the following methods instead: {string.Join(", ", suggestions)}");
+            throw new NotSupportedException($"{GetType().Name} does not support Update(DateTime, decimal) method overload. Use Update({typeof(T).Name}) instead.");
         }
 
         /// <summary>
@@ -268,4 +257,4 @@ namespace QuantConnect.Indicators
             Updated?.Invoke(this, consolidated);
         }
     }
-}
+}

Indicators/QuantConnect.Indicators.csproj

@@ -108,7 +108,6 @@
     <Compile Include="ArnaudLegouxMovingAverage.cs" />
     <Compile Include="AverageDirectionalIndex.cs" />
     <Compile Include="AverageDirectionalMovementIndexRating.cs" />
-    <Compile Include="AwesomeOscillator.cs" />
     <Compile Include="BalanceOfPower.cs" />
     <Compile Include="CandlestickPatterns\UpDownGapThreeMethods.cs" />
     <Compile Include="CandlestickPatterns\UpsideGapTwoCrows.cs" />
@@ -174,10 +173,8 @@
     <Compile Include="CandlestickPatterns\ThreeWhiteSoldiers.cs" />
     <Compile Include="CandlestickPatterns\Doji.cs" />
     <Compile Include="CandlestickPatterns\TwoCrows.cs" />
-    <Compile Include="ChaikinMoneyFlow.cs" />
     <Compile Include="ChandeMomentumOscillator.cs" />
     <Compile Include="CoppockCurve.cs" />
-    <Compile Include="DeMarkerIndicator.cs" />
     <Compile Include="DetrendedPriceOscillator.cs" />
     <Compile Include="DonchianChannel.cs" />
     <Compile Include="DoubleExponentialMovingAverage.cs" />
@@ -186,7 +183,7 @@
     <Compile Include="SchaffTrendCycle.cs" />
     <Compile Include="WilderMovingAverage.cs" />
     <Compile Include="FractalAdaptiveMovingAverage.cs" />
-    <Compile Include="EaseOfMovementValue.cs" />
+    <Compile Include="EaseOfMovementValue.cs"/>
     <Compile Include="FilteredIdentity.cs" />
     <Compile Include="HullMovingAverage.cs" />
     <Compile Include="MassIndex.cs" />

Launcher/QuantConnect.Lean.Launcher.csproj

@@ -236,6 +236,9 @@
     <None Include="packages.config">
       <SubType>Designer</SubType>
     </None>
+    <None Include="Python.Runtime.dll.config">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
   </ItemGroup>
   <ItemGroup>
     <Analyzer Include="..\packages\Microsoft.CodeAnalysis.VersionCheckAnalyzer.2.9.3\analyzers\dotnet\Microsoft.CodeAnalysis.VersionCheckAnalyzer.dll" />

Launcher/app.config

@@ -27,6 +27,10 @@
         <assemblyIdentity name="System.IO.Compression" publicKeyToken="b77a5c561934e089" culture="neutral" />
         <bindingRedirect oldVersion="0.0.0.0-4.2.0.0" newVersion="4.2.0.0" />
       </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="System.Threading.Tasks.Extensions" publicKeyToken="cc7b13ffcd2ddd51" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-4.2.0.1" newVersion="4.2.0.1" />
+      </dependentAssembly>
     </assemblyBinding>
   </runtime>
 <startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.2" /></startup></configuration>

Logging/ConsoleErrorLogHandler.cs

@@ -1,39 +0,0 @@
-/*
- * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
- * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-namespace QuantConnect.Logging
-{
-    /// <summary>
-    /// Subclass of ConsoleLogHandler that only logs error messages
-    /// </summary>
-    public class ConsoleErrorLogHandler : ConsoleLogHandler
-    {
-        /// <summary>
-        /// Hide debug messages from log
-        /// </summary>
-        /// <param name="text">The debug text to log</param>
-        public override void Debug(string text)
-        {
-        }
-
-        /// <summary>
-        /// Hide trace messages from log
-        /// </summary>
-        /// <param name="text">The trace text to log</param>
-        public override void Trace(string text)
-        {
-        }
-    }
-}

Logging/ConsoleLogHandler.cs

@@ -54,7 +54,7 @@ namespace QuantConnect.Logging
         /// Write error message to log
         /// </summary>
         /// <param name="text">The error text to log</param>
-        public virtual void Error(string text)
+        public void Error(string text)
         {
 #if DEBUG
             Console.ForegroundColor = ConsoleColor.Red;
@@ -69,7 +69,7 @@ namespace QuantConnect.Logging
         /// Write debug message to log
         /// </summary>
         /// <param name="text">The debug text to log</param>
-        public virtual void Debug(string text)
+        public void Debug(string text)
         {
             _trace.WriteLine(DateTime.Now.ToString(_dateFormat, CultureInfo.InvariantCulture) + " DEBUG:: " + text);
         }
@@ -78,7 +78,7 @@ namespace QuantConnect.Logging
         /// Write debug message to log
         /// </summary>
         /// <param name="text">The trace text to log</param>
-        public virtual void Trace(string text)
+        public void Trace(string text)
         {
             _trace.WriteLine(DateTime.Now.ToString(_dateFormat, CultureInfo.InvariantCulture) + " Trace:: " + text);
         }

Logging/QuantConnect.Logging.csproj

@@ -109,7 +109,6 @@
     </Compile>
     <Compile Include="CompositeLogHandler.cs" />
     <Compile Include="CompositeNLogHandler.cs" />
-    <Compile Include="ConsoleErrorLogHandler.cs" />
     <Compile Include="NLogHandler.cs" />
     <Compile Include="QueueLogHandler.cs" />
     <Compile Include="FunctionalLogHandler.cs" />

Optimizer.Launcher/ConsoleLeanOptimizer.cs

@@ -18,7 +18,6 @@ using System;
 using System.Collections.Concurrent;
 using System.Diagnostics;
 using System.IO;
-using System.Linq;
 using QuantConnect.Optimizer.Parameters;
 using Log = QuantConnect.Logging.Log;
 
@@ -143,8 +142,7 @@ namespace QuantConnect.Optimizer.Launcher
             if (Status != OptimizationStatus.Ended && Status != OptimizationStatus.Aborted)
             {
                 var currentEstimate = GetCurrentEstimate();
-                var stats = GetRuntimeStatistics();
-                var message = $"ConsoleLeanOptimizer.SendUpdate(): {currentEstimate} {string.Join(", ", stats.Select(pair => $"{pair.Key}:{pair.Value}"))}";
+                var message = $"ConsoleLeanOptimizer.SendUpdate(): {currentEstimate}";
                 var currentBestBacktest = Strategy.Solution;
                 if (currentBestBacktest != null)
                 {

Optimizer/LeanOptimizer.cs

@@ -19,8 +19,6 @@ using QuantConnect.Util;
 using QuantConnect.Logging;
 using QuantConnect.Configuration;
 using System.Collections.Concurrent;
-using System.Collections.Generic;
-using System.Globalization;
 using QuantConnect.Optimizer.Objectives;
 using QuantConnect.Optimizer.Parameters;
 using QuantConnect.Optimizer.Strategies;
@@ -40,10 +38,6 @@ namespace QuantConnect.Optimizer
         private int _failedBacktest;
         private int _completedBacktest;
         private volatile bool _disposed;
-
-        /// <summary>
-        /// Lock to update optimization status
-        /// </summary>
         private object _statusLock = new object();
 
         /// <summary>
@@ -282,27 +276,19 @@ namespace QuantConnect.Optimizer
         /// <summary>
         /// Returns the current optimization status and strategy estimates
         /// </summary>
-        public int GetCurrentEstimate()
-        {
-            return Strategy.GetTotalBacktestEstimate();
-        }
-
-        /// <summary>
-        /// Get the current runtime statistics
-        /// </summary>
-        public Dictionary<string, string> GetRuntimeStatistics()
+        public OptimizationEstimate GetCurrentEstimate()
         {
             var completedCount = _completedBacktest;
-            var totalCount = completedCount + _failedBacktest;
             var runtime = DateTime.UtcNow - _startedAt;
-            return new Dictionary<string, string>
+            return new OptimizationEstimate
             {
-                { "Completed", $"{completedCount}"},
-                { "Failed", $"{_failedBacktest}"},
-                { "Running", $"{RunningParameterSetForBacktest.Count}"},
-                { "In Queue", $"{PendingParameterSet.Count}"},
-                { "Average Length", $"{(totalCount > 0 ? new TimeSpan(runtime.Ticks / totalCount) : TimeSpan.Zero).ToString(@"hh\:mm\:ss", CultureInfo.InvariantCulture)}"},
-                { "Total Runtime", $"{runtime.ToString(@"hh\:mm\:ss", CultureInfo.InvariantCulture)}" }
+                TotalBacktest = Strategy.GetTotalBacktestEstimate(),
+                CompletedBacktest = completedCount,
+                FailedBacktest = _failedBacktest,
+                RunningBacktest = RunningParameterSetForBacktest.Count,
+                InQueueBacktest = PendingParameterSet.Count,
+                AverageBacktest = completedCount > 0 ? new TimeSpan(runtime.Ticks / completedCount) : TimeSpan.Zero,
+                TotalRuntime = runtime
             };
         }
 
@@ -315,7 +301,7 @@ namespace QuantConnect.Optimizer
             {
                 return $"OID {NodePacket.OptimizationId}";
             }
-            return $"UI {NodePacket.UserId} PID {NodePacket.ProjectId} OID {NodePacket.OptimizationId} S {Status}";
+            return $"UI {NodePacket.UserId} PID {NodePacket.ProjectId} OID {NodePacket.OptimizationId}";
         }
 
         /// <summary>
@@ -333,7 +319,7 @@ namespace QuantConnect.Optimizer
         /// Sets the current optimization status
         /// </summary>
         /// <param name="optimizationStatus">The new optimization status</param>
-        protected virtual void SetOptimizationStatus(OptimizationStatus optimizationStatus)
+        protected void SetOptimizationStatus(OptimizationStatus optimizationStatus)
         {
             lock (_statusLock)
             {

Optimizer/OptimizationEstimate.cs

@@ -0,0 +1,74 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using Newtonsoft.Json;
+
+namespace QuantConnect.Optimizer
+{
+    public class OptimizationEstimate
+    {
+        /// <summary>
+        /// Total number of backtests, approximately
+        /// </summary>
+        [JsonProperty("totalBacktest")]
+        public int TotalBacktest { get; set; }
+
+        /// <summary>
+        /// Number of completed backtests
+        /// </summary>
+        [JsonProperty("completedBacktest")]
+        public int CompletedBacktest { get; set; }
+
+        /// <summary>
+        /// Number of failed backtests
+        /// </summary>
+        [JsonProperty("failedBacktest")]
+        public int FailedBacktest { get; set; }
+
+        /// <summary>
+        /// Number of running backtests
+        /// </summary>
+        [JsonProperty("runningBacktest")]
+        public int RunningBacktest { get; set; }
+
+        /// <summary>
+        /// Number of backtests in queue
+        /// </summary>
+        [JsonProperty("inQueueBacktest")]
+        public int InQueueBacktest { get; set; }
+
+        /// <summary>
+        /// Indicates backtest average duration; (start - now) / CompletedBacktest
+        /// </summary>
+        [JsonProperty("averageBacktest")]
+        public TimeSpan AverageBacktest { get; set; }
+
+        /// <summary>
+        /// The run time of this optimization
+        /// </summary>
+        [JsonProperty(PropertyName = "totalRuntime")]
+        public TimeSpan TotalRuntime { get; set; }
+
+        /// <summary>
+        /// Pretty representation of an optimization estimate
+        /// </summary>
+        public override string ToString()
+        {
+            return $"TotalBacktest: {TotalBacktest}. CompletedBacktest: {CompletedBacktest}. FailedBacktest: {FailedBacktest}." +
+                $" RunningBacktest: {RunningBacktest}. InQueueBacktest: {InQueueBacktest}. TotalRuntime {TotalRuntime}. AverageBacktest: {AverageBacktest}";
+        }
+    }
+}

Optimizer/QuantConnect.Optimizer.csproj

@@ -63,6 +63,7 @@
     <Compile Include="Strategies\OptimizationStrategySettings.cs" />
     <Compile Include="Strategies\StepBaseOptimizationStrategy.cs" />
     <Compile Include="Objectives\Objective.cs" />
+    <Compile Include="OptimizationEstimate.cs" />
     <Compile Include="Objectives\Target.cs" />
     <Compile Include="Objectives\Constraint.cs" />
     <Compile Include="LeanOptimizer.cs" />

QuantConnect.Lean.sln

@@ -230,6 +230,14 @@ Global
 		{D46D2A8D-340C-4B40-8EE6-6BAA7B1198AB}.Release|Any CPU.Build.0 = Release|Any CPU
 		{D46D2A8D-340C-4B40-8EE6-6BAA7B1198AB}.Release|x64.ActiveCfg = Release|Any CPU
 		{D46D2A8D-340C-4B40-8EE6-6BAA7B1198AB}.Release|x64.Build.0 = Release|Any CPU
+		{B3F3B1AC-3912-4020-945C-1DA8814C0A3B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B3F3B1AC-3912-4020-945C-1DA8814C0A3B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B3F3B1AC-3912-4020-945C-1DA8814C0A3B}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B3F3B1AC-3912-4020-945C-1DA8814C0A3B}.Debug|x64.Build.0 = Debug|Any CPU
+		{B3F3B1AC-3912-4020-945C-1DA8814C0A3B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B3F3B1AC-3912-4020-945C-1DA8814C0A3B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B3F3B1AC-3912-4020-945C-1DA8814C0A3B}.Release|x64.ActiveCfg = Release|Any CPU
+		{B3F3B1AC-3912-4020-945C-1DA8814C0A3B}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

Report/App.config

@@ -1,10 +1,14 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <configuration>
     <startup> 
-        <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.2"/>
+        <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.2" />
     </startup>
   <runtime>
     <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
+      <dependentAssembly>
+        <assemblyIdentity name="System.Threading.Tasks.Extensions" publicKeyToken="cc7b13ffcd2ddd51" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-4.1.1.0" newVersion="4.1.1.0" />
+      </dependentAssembly>
     </assemblyBinding>
   </runtime>
 </configuration>

Report/PortfolioLooper/PortfolioLooper.cs

@@ -128,9 +128,6 @@ namespace QuantConnect.Report
             // More initialization, this time with Algorithm and other misc. classes
             _resultHandler.Initialize(job, new Messaging.Messaging(), new Api.Api(), transactions);
             _resultHandler.SetAlgorithm(Algorithm, Algorithm.Portfolio.TotalPortfolioValue);
-
-            Algorithm.Transactions.SetOrderProcessor(transactions);
-
             transactions.Initialize(Algorithm, new BacktestingBrokerage(Algorithm), _resultHandler);
             feed.Initialize(Algorithm, job, _resultHandler, null, null, null, _dataManager, null, null);
 

Research/BasicCSharpQuantBookTemplate.ipynb

@@ -27,7 +27,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "#load \"../QuantConnect.csx\"\n",
+    "#load \"QuantConnect.csx\"\n",
     "var qb = new QuantBook();\n",
     "\n",
     "// Selecting asset data\n",

Research/QuantBook.cs

@@ -324,22 +324,7 @@ namespace QuantConnect.Research
             // Load a canonical option Symbol if the user provides us with an underlying Symbol
             if (symbol.SecurityType != SecurityType.Option && symbol.SecurityType != SecurityType.FutureOption)
             {
-                var option = AddOption(symbol, resolution, symbol.ID.Market);
-
-                // Allow 20 strikes from the money for futures. No expiry filter is applied
-                // so that any future contract provided will have data returned.
-                if (symbol.SecurityType == SecurityType.Future && symbol.IsCanonical())
-                {
-                    throw new ArgumentException("The Future Symbol provided is a canonical Symbol (i.e. a Symbol representing all Futures), which is not supported at this time. " +
-                        "If you are using the Symbol accessible from `AddFuture(...)`, use the Symbol from `AddFutureContract(...)` instead. " +
-                        "You can use `qb.FutureOptionChainProvider(canonicalFuture, datetime)` to get a list of futures contracts for a given date, and add them to your algorithm with `AddFutureContract(symbol, Resolution)`.");
-                }
-                if (symbol.SecurityType == SecurityType.Future && !symbol.IsCanonical())
-                {
-                    option.SetFilter(universe => universe.Strikes(-10, +10));
-                }
-
-                symbol = option.Symbol;
+                symbol = AddOption(symbol, resolution, symbol.ID.Market).Symbol;
             }
 
             IEnumerable<Symbol> symbols;
@@ -352,19 +337,8 @@ namespace QuantConnect.Research
                                                        .GetHighestResolution();
                 if (!Securities.ContainsKey(symbol.Underlying))
                 {
-                    if (symbol.Underlying.SecurityType == SecurityType.Equity)
-                    {
-                        // only add underlying if not present
-                        AddEquity(symbol.Underlying.Value, resolutionToUseForUnderlying);
-                    }
-                    if (symbol.Underlying.SecurityType == SecurityType.Future && symbol.Underlying.IsCanonical())
-                    {
-                        AddFuture(symbol.Underlying.ID.Symbol, resolutionToUseForUnderlying);
-                    }
-                    else if (symbol.Underlying.SecurityType == SecurityType.Future)
-                    {
-                        AddFutureContract(symbol.Underlying, resolutionToUseForUnderlying);
-                    }
+                    // only add underlying if not present
+                    AddEquity(symbol.Underlying.Value, resolutionToUseForUnderlying);
                 }
                 var allSymbols = new List<Symbol>();
                 for (var date = start; date < end; date = date.AddDays(1))

Tests/Algorithm/Framework/QCAlgorithmFrameworkTests.cs

@@ -71,47 +71,6 @@ namespace QuantConnect.Tests.Algorithm.Framework
             Assert.IsTrue(construction.Insights.All(insight => insight.CloseTimeUtc != default(DateTime)));
         }
 
-        [TestCase(true, 0)]
-        [TestCase(false, 2)]
-        public void DelistedSecuritiesInsightsTest(bool isDelisted, int expectedCount)
-        {
-            var algorithm = new QCAlgorithm();
-            algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(algorithm));
-            algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor());
-            algorithm.SetStartDate(2007, 5, 16);
-            algorithm.SetUniverseSelection(new ManualUniverseSelectionModel());
-            algorithm.SetFinishedWarmingUp();
-
-            var alpha = new FakeAlpha();
-            algorithm.SetAlpha(alpha);
-
-            var construction = new FakePortfolioConstruction();
-            algorithm.SetPortfolioConstruction(construction);
-
-            var actualInsights = new List<Insight>();
-            algorithm.InsightsGenerated += (s, e) => actualInsights.AddRange(e.Insights);
-
-            var security = algorithm.AddEquity("SPY", Resolution.Daily);
-            var tick = new Tick
-            {
-                Symbol = security.Symbol,
-                Value = 1,
-                Quantity = 2
-            };
-
-            security.SetMarketPrice(tick);
-            security.IsDelisted = isDelisted;
-
-            // Trigger Alpha to emit insight
-            algorithm.OnFrameworkData(new Slice(new DateTime(2000, 01, 01), new List<BaseData>() { tick }));
-
-            // Manually emit insight
-            algorithm.EmitInsights(Insight.Price(Symbols.SPY, TimeSpan.FromDays(1), InsightDirection.Up, .5, .75));
-
-            // Should be zero because security is delisted
-            Assert.AreEqual(expectedCount, actualInsights.Count);
-        }
-
         class FakeAlpha : AlphaModel
         {
             public override IEnumerable<Insight> Update(QCAlgorithm algorithm, Slice data)

Tests/AlgorithmRunner.cs

@@ -86,14 +86,12 @@ namespace QuantConnect.Tests
                         ? "../../../Algorithm.Python/" + algorithm + ".py"
                         : "QuantConnect.Algorithm." + language + ".dll");
 
-                // Store initial log variables
-                var initialLogHandler = Log.LogHandler;
-                var initialDebugEnabled = Log.DebuggingEnabled;
 
-                // Log handlers specific to this test function
-                var newLogHandlers = new ILogHandler[] { new ConsoleErrorLogHandler(), new FileLogHandler(logFile, false) };
+                var debugEnabled = Log.DebuggingEnabled;
 
-                using (Log.LogHandler = new CompositeLogHandler(newLogHandlers))
+
+                var logHandlers = new ILogHandler[] {new ConsoleLogHandler(), new FileLogHandler(logFile, false)};
+                using (Log.LogHandler = new CompositeLogHandler(logHandlers))
                 using (var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance))
                 using (var systemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance))
                 using (var workerThread  = new TestWorkerThread())
@@ -105,6 +103,7 @@ namespace QuantConnect.Tests
                     Log.Trace("");
 
                     // run the algorithm in its own thread
+
                     var engine = new Lean.Engine.Engine(systemHandlers, algorithmHandlers, false);
                     Task.Factory.StartNew(() =>
                     {
@@ -138,11 +137,9 @@ namespace QuantConnect.Tests
 
                     var defaultAlphaHandler = (DefaultAlphaHandler) algorithmHandlers.Alphas;
                     alphaStatistics = defaultAlphaHandler.RuntimeStatistics;
-                }
 
-                // Reset settings to initial values
-                Log.LogHandler = initialLogHandler;
-                Log.DebuggingEnabled = initialDebugEnabled;
+                    Log.DebuggingEnabled = debugEnabled;
+                }
             }
             catch (Exception ex)
             {

Tests/Api/ApiTests.cs

@@ -353,8 +353,7 @@ namespace QuantConnect.Tests.API
             Assert.IsTrue(backtestRead.Success);
             Assert.IsTrue(backtestRead.Progress == 1);
             Assert.IsTrue(backtestRead.Name == backtestName);
-            Assert.IsTrue(backtestRead.Statistics["Total Trades"] == "1");
-            Assert.IsTrue(backtestRead.Charts["Benchmark"].Series.Count > 0);
+            Assert.IsTrue(backtestRead.Result.Statistics["Total Trades"] == "1");
 
             // Verify we have the backtest in our project
             var listBacktests = _api.ListBacktests(project.Projects.First().ProjectId);

Tests/AssemblyInitialize.cs

@@ -21,27 +21,16 @@ using QuantConnect;
 using QuantConnect.Configuration;
 using QuantConnect.Logging;
 using QuantConnect.Python;
-using QuantConnect.Util;
 
 [SetUpFixture]
 public class AssemblyInitialize
 {
     [OneTimeSetUp]
-    public void InitializeTestEnvironment()
+    public void SetLogHandler()
     {
         AdjustCurrentDirectory();
-        
-        if (TestContext.Parameters.Exists("log-handler"))
-        {
-            var logHandler = TestContext.Parameters["log-handler"];
-            Log.Trace($"QuantConnect.Tests.AssemblyInitialize(): Log handler test parameter loaded {logHandler}");
-
-            Log.LogHandler = Composer.Instance.GetExportedValueByTypeName<ILogHandler>(logHandler);
-        }
-        else
-        {
-            Log.LogHandler = new ConsoleLogHandler();
-        }
+        // save output to file as well
+        Log.LogHandler = new ConsoleLogHandler();
     }
 
     public static void AdjustCurrentDirectory()

Tests/Brokerages/Alpaca/AlpacaBrokerageHistoryProviderTests.cs

@@ -62,6 +62,8 @@ namespace QuantConnect.Tests.Brokerages.Alpaca
         [Test, TestCaseSource(nameof(TestParameters))]
         public void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan period, bool shouldBeEmpty)
         {
+            Log.LogHandler = new ConsoleLogHandler();
+
             var keyId = Config.Get("alpaca-key-id");
             var secretKey = Config.Get("alpaca-secret-key");
             var tradingMode = Config.Get("alpaca-trading-mode");
@@ -99,14 +101,14 @@ namespace QuantConnect.Tests.Brokerages.Alpaca
                     {
                         foreach (var tick in slice.Ticks[symbol])
                         {
-                            Log.Trace($"{tick.Time}: {tick.Symbol} - P={tick.Price}, Q={tick.Quantity}");
+                            Console.WriteLine($"{tick.Time}: {tick.Symbol} - P={tick.Price}, Q={tick.Quantity}");
                         }
                     }
                     else
                     {
                         var bar = slice.Bars[symbol];
 
-                        Log.Trace($"{bar.Time}: {bar.Symbol} - O={bar.Open}, H={bar.High}, L={bar.Low}, C={bar.Close}, V={bar.Volume}");
+                        Console.WriteLine($"{bar.Time}: {bar.Symbol} - O={bar.Open}, H={bar.High}, L={bar.Low}, C={bar.Close}, V={bar.Volume}");
                     }
                 }
 

Tests/Brokerages/Alpaca/AlpacaBrokerageTests.cs

@@ -25,7 +25,6 @@ using NUnit.Framework;
 using QuantConnect.Brokerages.Alpaca;
 using QuantConnect.Configuration;
 using QuantConnect.Interfaces;
-using QuantConnect.Logging;
 using QuantConnect.Orders;
 using QuantConnect.Securities;
 
@@ -163,12 +162,12 @@ namespace QuantConnect.Tests.Brokerages.Alpaca
             {
                 var order = new MarketOrder(symbol, 10, DateTime.UtcNow);
                 OrderProvider.Add(order);
-                Log.Trace("Buy Order");
+                Console.WriteLine("Buy Order");
                 alpaca.PlaceOrder(order);
 
                 var orderr = new MarketOrder(symbol, -10, DateTime.UtcNow);
                 OrderProvider.Add(orderr);
-                Log.Trace("Sell Order");
+                Console.WriteLine("Sell Order");
                 alpaca.PlaceOrder(orderr);
             }
 
@@ -292,9 +291,9 @@ namespace QuantConnect.Tests.Brokerages.Alpaca
 
             var tenMinutes = TimeSpan.FromMinutes(10);
 
-            Log.Trace("------");
-            Log.Trace("Waiting for internet disconnection ");
-            Log.Trace("------");
+            Console.WriteLine("------");
+            Console.WriteLine("Waiting for internet disconnection ");
+            Console.WriteLine("------");
 
             // spin while we manually disconnect the internet
             while (brokerage.IsConnected)
@@ -305,9 +304,9 @@ namespace QuantConnect.Tests.Brokerages.Alpaca
 
             var stopwatch = Stopwatch.StartNew();
 
-            Log.Trace("------");
-            Log.Trace("Trying to reconnect ");
-            Log.Trace("------");
+            Console.WriteLine("------");
+            Console.WriteLine("Trying to reconnect ");
+            Console.WriteLine("------");
 
             // spin until we're reconnected
             while (!brokerage.IsConnected && stopwatch.Elapsed < tenMinutes)

Tests/Brokerages/BrokerageTests.cs

@@ -461,7 +461,7 @@ namespace QuantConnect.Tests.Brokerages
                 lock (sync)
                 {
                     remaining -= orderEvent.FillQuantity;
-                    Log.Trace("Remaining: " + remaining + " FillQuantity: " + orderEvent.FillQuantity);
+                    Console.WriteLine("Remaining: " + remaining + " FillQuantity: " + orderEvent.FillQuantity);
                     if (orderEvent.Status == OrderStatus.Filled)
                     {
                         manualResetEvent.Set();
@@ -480,7 +480,7 @@ namespace QuantConnect.Tests.Brokerages
             manualResetEvent.WaitOne(2500);
             manualResetEvent.WaitOne(2500);
 
-            Log.Trace("Remaining: " + remaining);
+            Console.WriteLine("Remaining: " + remaining);
             Assert.AreEqual(0, remaining);
         }
 
@@ -636,7 +636,7 @@ namespace QuantConnect.Tests.Brokerages
                 }
                 catch (Exception err)
                 {
-                    Log.Error(err.Message);
+                    Console.WriteLine(err.Message);
                 }
             }, cancellationToken.Token);
         }

Tests/Brokerages/Fxcm/FxcmBrokerageTests.cs

@@ -22,7 +22,6 @@ using QuantConnect.Brokerages.Fxcm;
 using QuantConnect.Configuration;
 using QuantConnect.Interfaces;
 using QuantConnect.Lean.Engine.DataFeeds;
-using QuantConnect.Logging;
 using QuantConnect.Orders;
 using QuantConnect.Securities;
 
@@ -115,9 +114,9 @@ namespace QuantConnect.Tests.Brokerages.Fxcm
 
             var tenMinutes = TimeSpan.FromMinutes(10);
 
-            Log.Trace("------");
-            Log.Trace("Waiting for internet disconnection ");
-            Log.Trace("------");
+            Console.WriteLine("------");
+            Console.WriteLine("Waiting for internet disconnection ");
+            Console.WriteLine("------");
 
             // spin while we manually disconnect the internet
             while (brokerage.IsConnected)
@@ -128,9 +127,9 @@ namespace QuantConnect.Tests.Brokerages.Fxcm
 
             var stopwatch = Stopwatch.StartNew();
 
-            Log.Trace("------");
-            Log.Trace("Trying to reconnect ");
-            Log.Trace("------");
+            Console.WriteLine("------");
+            Console.WriteLine("Trying to reconnect ");
+            Console.WriteLine("------");
 
             // spin until we're reconnected
             while (!brokerage.IsConnected && stopwatch.Elapsed < tenMinutes)

Tests/Brokerages/InteractiveBrokers/InteractiveBrokersBrokerageAdditionalTests.cs

@@ -27,7 +27,6 @@ using QuantConnect.Brokerages.InteractiveBrokers;
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
 using QuantConnect.Lean.Engine.DataFeeds;
-using QuantConnect.Logging;
 using QuantConnect.Securities;
 using QuantConnect.Util;
 using Order = QuantConnect.Orders.Order;
@@ -69,7 +68,7 @@ namespace QuantConnect.Tests.Brokerages.InteractiveBrokers
                     var stopwatch = Stopwatch.StartNew();
                     var value = (ContractDetails)method.Invoke(brokerage, parameters);
                     stopwatch.Stop();
-                    Log.Trace($"{DateTime.UtcNow:O} Response time: {stopwatch.Elapsed}");
+                    Console.WriteLine($"{DateTime.UtcNow:O} Response time: {stopwatch.Elapsed}");
                 });
                 while (!result.IsCompleted) Thread.Sleep(1000);
             }

Tests/Brokerages/InteractiveBrokers/InteractiveBrokersBrokerageDataQueueHandlerTest.cs

@@ -137,7 +137,7 @@ namespace QuantConnect.Tests.Brokerages.InteractiveBrokers
                 }
                 catch (Exception err)
                 {
-                    QuantConnect.Logging.Log.Error(err.Message);
+                    Console.WriteLine(err.Message);
                 }
             });
         }
@@ -146,7 +146,7 @@ namespace QuantConnect.Tests.Brokerages.InteractiveBrokers
         {
             if (tick != null)
             {
-                QuantConnect.Logging.Log.Trace("{0}: {1} - {2} @ {3}", tick.Time, tick.Symbol, tick.Price, ((Tick)tick).Quantity);
+                Console.WriteLine("{0}: {1} - {2} @ {3}", tick.Time, tick.Symbol, tick.Price, ((Tick)tick).Quantity);
             }
         }
     }

Tests/Brokerages/InteractiveBrokers/InteractiveBrokersBrokerageTests.cs

@@ -447,7 +447,7 @@ namespace QuantConnect.Tests.Brokerages.InteractiveBrokers
 
             foreach (var holding in previousHoldings)
             {
-                Log.Trace(holding.Value.ToString());
+                Console.WriteLine(holding.Value);
             }
 
             var hasSymbol = previousHoldings.ContainsKey(Symbols.USDJPY);
@@ -497,7 +497,7 @@ namespace QuantConnect.Tests.Brokerages.InteractiveBrokers
             Assert.IsTrue(cashBalance.Any(x => x.Currency == Currencies.USD));
             foreach (var cash in cashBalance)
             {
-                Log.Trace(cash.ToString());
+                Console.WriteLine(cash);
                 if (cash.Currency == Currencies.USD)
                 {
                     Assert.AreNotEqual(0m, cashBalance.Single(x => x.Currency == Currencies.USD));
@@ -590,7 +590,7 @@ namespace QuantConnect.Tests.Brokerages.InteractiveBrokers
             var executions = ib.GetExecutions(null, null, null, DateTime.UtcNow.AddDays(-1), null);
 
             stopwatch.Stop();
-            Log.Trace("Total executions fetched: {0}, elapsed time: {1} ms", executions.Count, stopwatch.ElapsedMilliseconds);
+            Console.WriteLine("Total executions fetched: {0}, elapsed time: {1} ms", executions.Count, stopwatch.ElapsedMilliseconds);
 
             Assert.IsTrue(executions.Any(x => order.BrokerId.Any(id => executions.Any(e => e.Execution.OrderId == Parse.Int(id)))));
         }
@@ -630,9 +630,9 @@ namespace QuantConnect.Tests.Brokerages.InteractiveBrokers
 
             var tenMinutes = TimeSpan.FromMinutes(10);
 
-            Log.Trace("------");
-            Log.Trace("Waiting for internet disconnection ");
-            Log.Trace("------");
+            Console.WriteLine("------");
+            Console.WriteLine("Waiting for internet disconnection ");
+            Console.WriteLine("------");
 
             // spin while we manually disconnect the internet
             while (ib.IsConnected)
@@ -643,9 +643,9 @@ namespace QuantConnect.Tests.Brokerages.InteractiveBrokers
 
             var stopwatch = Stopwatch.StartNew();
 
-            Log.Trace("------");
-            Log.Trace("Trying to reconnect ");
-            Log.Trace("------");
+            Console.WriteLine("------");
+            Console.WriteLine("Trying to reconnect ");
+            Console.WriteLine("------");
 
             // spin until we're reconnected
             while (!ib.IsConnected && stopwatch.Elapsed < tenMinutes)

Tests/Brokerages/InteractiveBrokers/InteractiveBrokersFuturesTests.cs

@@ -31,6 +31,8 @@ namespace QuantConnect.Tests.Brokerages.InteractiveBrokers
         [Test]
         public void CreatesExpectedFuturesContracts()
         {
+            Log.LogHandler = new ConsoleLogHandler();
+
             var symbolMapper = new InteractiveBrokersSymbolMapper();
 
             using (var ib = new InteractiveBrokersBrokerage(new QCAlgorithm(), new OrderProvider(), new SecurityProvider(), new AggregationManager()))

Tests/Brokerages/Oanda/OandaBrokerageDataQueueHandlerTests.cs

@@ -94,7 +94,7 @@ namespace QuantConnect.Tests.Brokerages.Oanda
                 brokerage.Subscribe(config, (s, e) => { });
             }
             stopwatch.Stop();
-            Log.Trace("Subscribe: Elapsed time: " + stopwatch.Elapsed);
+            Console.WriteLine("Subscribe: Elapsed time: " + stopwatch.Elapsed);
 
             Thread.Sleep(10000);
 
@@ -103,7 +103,7 @@ namespace QuantConnect.Tests.Brokerages.Oanda
             {
                 brokerage.Unsubscribe(config);
             }
-            Log.Trace("Unsubscribe: Elapsed time: " + stopwatch.Elapsed);
+            Console.WriteLine("Unsubscribe: Elapsed time: " + stopwatch.Elapsed);
 
             Thread.Sleep(5000);
         }