Add extension RemoveFromStart

Common/Extensions.cs

@@ -2713,7 +2713,7 @@ namespace QuantConnect
         /// <returns></returns>
         public static string RemoveFromEnd(this string s, string ending)
         {
-            if (s.EndsWith(ending))
+            if (s.EndsWith(ending, StringComparison.InvariantCulture))
             {
                 return s.Substring(0, s.Length - ending.Length);
             }
@@ -2723,6 +2723,24 @@ namespace QuantConnect
             }
         }
 
+        /// <summary>
+        /// Returns a new string in which specified start in the current instance is removed.
+        /// </summary>
+        /// <param name="s">original string value</param>
+        /// <param name="start">the string to be removed</param>
+        /// <returns>Substring with start removed</returns>
+        public static string RemoveFromStart(this string s, string start)
+        {
+            if (!s.IsNullOrEmpty() && !start.IsNullOrEmpty() && s.StartsWith(start, StringComparison.InvariantCulture))
+            {
+                return s.Substring(start.Length);
+            }
+            else
+            {
+                return s;
+            }
+        }
+
         /// <summary>
         /// Normalizes the specified price based on the DataNormalizationMode
         /// </summary>

Common/Packets/AlgorithmNodePacket.cs

@@ -117,6 +117,12 @@ namespace QuantConnect.Packets
         [JsonProperty(PropertyName = "sVersion")]
         public string Version;
 
+        /// <summary>
+        /// Virtual environment directory to use for PythonNet libraries
+        /// </summary>
+        [JsonProperty(PropertyName = "sPythonVirtualEnvironment")]
+        public string PythonVirtualEnvironment;
+
         /// <summary>
         /// An algorithm packet which has already been run and is being redelivered on this node.
         /// In this event we don't want to relaunch the task as it may result in unexpected behaviour for user.

Common/Packets/Packet.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -165,6 +165,9 @@ namespace QuantConnect.Packets
         OptimizationStatus,
 
         /// Optimization work result
-        OptimizationResult
+        OptimizationResult,
+
+        /// Python Environment Packet
+        PythonEnvironment
     }
 }

Common/Python/PythonInitializer.cs

@@ -19,6 +19,8 @@ using System.Linq;
 using Python.Runtime;
 using QuantConnect.Logging;
 using System.Collections.Generic;
+using System.IO;
+using System.Runtime.InteropServices;
 using QuantConnect.Util;
 
 namespace QuantConnect.Python
@@ -59,29 +61,75 @@ namespace QuantConnect.Python
         /// </summary>
         public static void AddPythonPaths(IEnumerable<string> paths)
         {
-            if (paths == null)
+            // Filter out any paths that are already on our Python path
+            if (paths.IsNullOrEmpty())
             {
                 return;
             }
 
+            // Add these paths to our pending additions
+            _pendingPathAdditions.AddRange(paths);
+
             if (_isInitialized)
             {
                 using (Py.GIL())
                 {
-                    _pendingPathAdditions.AddRange(paths);
+                    dynamic sys = Py.Import("sys");
+                    var locals = new PyDict();
+                    locals.SetItem("sys", sys);
 
-                    // Generate the python code to add these to our path and execute
-                    var code = string.Join(";", _pendingPathAdditions.Select(s => $"sys.path.append('{s}')"))
-                        .Replace('\\', '/');
+                    // Filter out any already paths that already exist on our current PythonPath
+                    var currentPath = PythonEngine.Eval("sys.path", null, locals.Handle).As<List<string>>();
+                    _pendingPathAdditions = _pendingPathAdditions.Where(x => !currentPath.Contains(x.Replace('\\', '/'))).ToList();
 
-                    PythonEngine.Exec($"import sys;{code}");
-                    _pendingPathAdditions.Clear();
+                    // Insert any pending path additions
+                    if (!_pendingPathAdditions.IsNullOrEmpty())
+                    {
+                        var code = string.Join(";", _pendingPathAdditions
+                            .Select(s => $"sys.path.insert(0, '{s}')")).Replace('\\', '/');
+                        PythonEngine.Exec(code, null, locals.Handle);
+
+                        _pendingPathAdditions.Clear();
+                    }
+
+                    locals.Dispose();
                 }
             }
-            else
+        }
+
+        /// <summary>
+        /// "Activate" a virtual Python environment by prepending its library storage to Pythons
+        /// path. This allows the libraries in this venv to be selected prior to our base install.
+        /// Requires PYTHONNET_PYDLL to be set to base install.
+        /// </summary>
+        /// <remarks>If a module is already loaded, Python will use its cached version first
+        /// these modules must be reloaded by reload() from importlib library</remarks>
+        public static void ActivatePythonVirtualEnvironment(string pathToVirtualEnv)
+        {
+            if (pathToVirtualEnv != null)
             {
-                // Add these paths to our pending additions list
-                _pendingPathAdditions.AddRange(paths);
+                pathToVirtualEnv = pathToVirtualEnv.TrimEnd('/').TrimEnd('\\');
+                var pathsToPrepend = new List<string>();
+                if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+                {
+                    // For linux we need to know the python version to determine the lib folder containing our packages
+                    // Compare our PyDLL to the directory names under the lib directory and get a match
+                    var pyDll = Environment.GetEnvironmentVariable("PYTHONNET_PYDLL");
+                    var version = Path.GetFileNameWithoutExtension(pyDll);
+                    var libDir = Directory.GetDirectories($"{pathToVirtualEnv}/lib")
+                        .Select(d => new DirectoryInfo(d).Name)
+                        .First(x => version.Contains(x, StringComparison.InvariantCulture));
+
+                    pathsToPrepend.Add($"{pathToVirtualEnv}/lib/{libDir}");
+                    pathsToPrepend.Add($"{pathToVirtualEnv}/lib/{libDir}/site-packages");
+                }
+                else
+                {
+                    pathsToPrepend.Add($"{pathToVirtualEnv}\\Lib");
+                    pathsToPrepend.Add($"{pathToVirtualEnv}\\Lib\\site-packages");
+                }
+
+                AddPythonPaths(pathsToPrepend);
             }
         }
     }

DockerfileLeanFoundation

@@ -43,6 +43,11 @@ RUN wget https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-p
     rm packages-microsoft-prod.deb && \
     apt-get clean && apt-get autoclean && apt-get autoremove --purge -y && rm -rf /var/lib/apt/lists/*
 
+# Install libffi7 for rpy2 package
+RUN wget http://es.archive.ubuntu.com/ubuntu/pool/main/libf/libffi/libffi7_3.3-4_amd64.deb && \
+    dpkg -i libffi7_3.3-4_amd64.deb && \
+    rm libffi7_3.3-4_amd64.deb
+
 # Set PythonDLL variable for PythonNet
 ENV PYTHONNET_PYDLL="/opt/miniconda3/lib/libpython3.6m.so"
 
@@ -80,9 +85,7 @@ RUN pip install --no-cache-dir      \
     numba==0.46                     \
     setuptools-git==1.2             \
     xarray==0.15.1                  \
-    plotly==4.7.1                   \
     jupyterlab==2.1.2               \
-    tensorflow==1.15.2              \
     docutils==0.14                  \
     cvxopt==1.2.0                   \
     gensim==3.8.0                   \
@@ -92,10 +95,8 @@ RUN pip install --no-cache-dir      \
     nltk==3.4.5                     \
     pomegranate==0.11.1             \
     graphviz==0.8.4                 \
-    cmdstanpy==0.4                  \
     copulae==0.3.1                  \
     featuretools==0.14.0            \
-    fbprophet==0.6                  \
     PuLP==1.6.8                     \
     pymc3==3.8                      \
     rauth==0.7.3                    \
@@ -109,12 +110,10 @@ RUN pip install --no-cache-dir      \
     PyWavelets==1.1.1               \
     umap-learn==0.4.3               \
     nvidia-ml-py3==7.352.0          \
-    fastai==1.0.61                  \
     arch==4.14                      \
     copulalib==1.1.0                \
     copulas==0.3.3                  \
     creme==0.5.1                    \
-    cufflinks==0.17.3               \
     gym==0.17.2                     \
     ipywidgets==7.5.1               \
     deap==1.3.1                     \
@@ -122,18 +121,13 @@ RUN pip install --no-cache-dir      \
     pykalman==0.9.5                 \
     pyportfolioopt==1.2.2           \
     pyramid-arima==0.9.0            \
-    pyro-ppl==1.3.1                 \
     riskparityportfolio==0.2        \
     sklearn-json==0.1.0             \
     stable-baselines==2.10.0        \
     statistics==1.0.3.5             \
     statsmodels==0.11.1             \
-    tensorforce==0.5.5              \
     QuantLib-Python==1.18           \
-    xgboost==1.1.0                  \
     dtw-python==1.0.5               \
-    cntk==2.7                       \
-    mxnet==1.6                      \
     gluonts==0.4.3                  \
     gplearn==0.4.1                  \
     jax==0.1.68                     \
@@ -143,8 +137,6 @@ RUN pip install --no-cache-dir      \
     neural-tangents==0.2.1          \
     mplfinance==0.12.4a0            \
     hmmlearn==0.2.3                 \
-    catboost==0.23.2                \
-    fastai2==0.0.17                 \
     ppscore==0.0.2                  \
     scikit-tda==0.0.3               \
     ta==0.5.25                      \
@@ -153,7 +145,6 @@ RUN pip install --no-cache-dir      \
     optuna==2.3.0                   \
     findiff==0.8.5                  \
     sktime==0.3.0                   \
-    sktime-dl==0.1.0                \
     hyperopt==0.2.5                 \
     bayesian-optimization==1.2.0    \
     rpy2==3.3.6                     \
@@ -165,8 +156,7 @@ RUN pip install --no-cache-dir      \
     dgl==0.6.1                      \
     ruptures==1.1.3                 \
     simpy==4.0.1                    \
-    scikit-learn-extra==0.2.0       \
-    ray==1.5.1
+    scikit-learn-extra==0.2.0
 
 # feature_selector has overly strict dependency version ranges
 # We already installed close-enough versions of all of its dependencies above
@@ -189,21 +179,8 @@ RUN conda install -y -c conda-forge \
 # Install non-math packages
 RUN conda install -y                \
     blaze=0.11.3                    \
-    tensorflow-base=1.15.0          \
     && conda clean -y --all
 
-# Install math/ML from pytorch
-RUN conda install -y -c pytorch     \
-    pytorch=1.5.0                   \
-    torchvision=0.6.0               \
-    && conda clean -y --all
-
-# Install PyTorch Geometric
-RUN TORCH=$(python -c "import torch; print(torch.__version__)") && \
-    CUDA=$(python -c "import torch; print('cu' + torch.version.cuda.replace('.', ''))") && \
-    pip install --no-cache-dir -f https://pytorch-geometric.com/whl/torch-${TORCH}+${CUDA}.html \
-    torch-scatter==2.0.5 torch-sparse==0.6.7 torch-cluster==1.5.7 torch-spline-conv==1.2.0 torch-geometric==1.7.0
-
 RUN python -m nltk.downloader -d /usr/share/nltk_data punkt && \
     python -m nltk.downloader -d /usr/share/nltk_data vader_lexicon && \
     python -m nltk.downloader -d /usr/share/nltk_data stopwords
@@ -257,14 +234,15 @@ RUN wget https://cdn.quantconnect.com/tigramite/tigramite-4.1.zip && \
     unzip -q tigramite-4.1.zip && cd tigramite-4.1 && \
     python setup.py install && cd .. && rm -rf tigramite-4.1 && rm tigramite-4.1.zip
 
-# Install H2O: https://www.h2o.ai/download/
-RUN wget https://cdn.quantconnect.com/h2o/h2o-3.30.0.3.zip && \
-    unzip -q h2o-3.30.0.3.zip && \
-    pip install h2o-3.30.0.3/python/h2o-3.30.0.3-py2.py3-none-any.whl && \
-    rm -rf h2o-3.30.0.3 && rm h2o-3.30.0.3.zip
 
 # Remove black-listed packages
 RUN pip uninstall -y s3transfer
 
+# Install a shell Python Virtual Environment
+# Selected in Lean with directory via JSON JobPacket "sPythonVirtualEnvironment" or config "python-venv" value
+RUN python -m venv /venv --system-site-packages && . /venv/bin/activate \
+    && python -m ipykernel install --name=venv \
+    && deactivate 
+
 # List all packages
 RUN conda list

DockerfileLeanFoundationARM

@@ -186,5 +186,11 @@ RUN wget https://cdn.quantconnect.com/h2o/h2o-3.30.0.3.zip && \
 # Remove black-listed packages
 RUN pip uninstall -y s3transfer
 
+# Install a shell Python Virtual Environment
+# Selected in Lean with directory via JSON JobPacket "sPythonVirtualEnvironment" or config "python-venv" value
+RUN python -m venv /venv --system-site-packages && . /venv/bin/activate \
+    && python -m ipykernel install --name=venv \
+    && deactivate 
+
 # List all packages
 RUN conda list

Launcher/Program.cs

@@ -22,6 +22,7 @@ using QuantConnect.Configuration;
 using QuantConnect.Lean.Engine;
 using QuantConnect.Logging;
 using QuantConnect.Packets;
+using QuantConnect.Python;
 using QuantConnect.Util;
 
 namespace QuantConnect.Lean.Launcher
@@ -69,6 +70,12 @@ namespace QuantConnect.Lean.Launcher
             string assemblyPath;
             job = leanEngineSystemHandlers.JobQueue.NextJob(out assemblyPath);
 
+            // Activate our PythonVirtualEnvironment if provided
+            if (!string.IsNullOrEmpty(job.PythonVirtualEnvironment))
+            {
+                PythonInitializer.ActivatePythonVirtualEnvironment(job.PythonVirtualEnvironment);
+            }
+
             leanEngineAlgorithmHandlers = Initializer.GetAlgorithmHandlers();
 
             if (job == null)

Launcher/config.json

@@ -28,6 +28,9 @@
   "debugging": false,
   "debugging-method": "LocalCmdline",
 
+  // location of a python virtual env to use libraries from
+  //"python-venv": "/venv",
+
   // handlers
   "log-handler": "QuantConnect.Logging.CompositeLogHandler",
   "messaging-handler": "QuantConnect.Messaging.Messaging",

Queues/JobQueue.cs

@@ -115,7 +115,8 @@ namespace QuantConnect.Queues
                     DeployId = algorithmId,
                     Parameters = parameters,
                     Language = Language,
-                    Controls = controls
+                    Controls = controls,
+                    PythonVirtualEnvironment = Config.Get("python-venv")
                 };
 
                 try
@@ -147,7 +148,8 @@ namespace QuantConnect.Queues
                 BacktestId = algorithmId,
                 Language = Language,
                 Parameters = parameters,
-                Controls = controls
+                Controls = controls,
+                PythonVirtualEnvironment = Config.Get("python-venv")
             };
 
             return backtestJob;

Tests/AssemblyInitialize.cs

@@ -50,6 +50,14 @@ namespace QuantConnect.Tests
             Directory.SetCurrentDirectory(dir);
             Config.Reset();
             Globals.Reset();
+
+            // Activate virtual environment if defined
+            var pythonVirtualEnvironment = Config.Get("python-venv");
+            if(!string.IsNullOrEmpty(pythonVirtualEnvironment)){
+                PythonInitializer.ActivatePythonVirtualEnvironment(pythonVirtualEnvironment);
+            }
+
+            // Initialize and add our Paths
             PythonInitializer.Initialize();
             PythonInitializer.AddPythonPaths(
                 new[]