Set ENV to UNKNOWN by default

Common/Extensions.cs

@@ -2782,7 +2782,7 @@ namespace QuantConnect
         /// <returns></returns>
         public static string RemoveFromEnd(this string s, string ending)
         {
-            if (s.EndsWith(ending))
+            if (s.EndsWith(ending, StringComparison.InvariantCulture))
             {
                 return s.Substring(0, s.Length - ending.Length);
             }
@@ -2792,6 +2792,24 @@ namespace QuantConnect
             }
         }
 
+        /// <summary>
+        /// Returns a new string in which specified start in the current instance is removed.
+        /// </summary>
+        /// <param name="s">original string value</param>
+        /// <param name="start">the string to be removed</param>
+        /// <returns>Substring with start removed</returns>
+        public static string RemoveFromStart(this string s, string start)
+        {
+            if (!s.IsNullOrEmpty() && !start.IsNullOrEmpty() && s.StartsWith(start, StringComparison.InvariantCulture))
+            {
+                return s.Substring(start.Length);
+            }
+            else
+            {
+                return s;
+            }
+        }
+
         /// <summary>
         /// Normalizes the specified price based on the DataNormalizationMode
         /// </summary>

Common/Packets/AlgorithmNodePacket.cs

@@ -117,6 +117,12 @@ namespace QuantConnect.Packets
         [JsonProperty(PropertyName = "sVersion")]
         public string Version;
 
+        /// <summary>
+        /// Virtual environment directory to use for PythonNet libraries
+        /// </summary>
+        [JsonProperty(PropertyName = "sPythonVirtualEnvironment")]
+        public string PythonVirtualEnvironment;
+
         /// <summary>
         /// An algorithm packet which has already been run and is being redelivered on this node.
         /// In this event we don't want to relaunch the task as it may result in unexpected behaviour for user.

Common/Packets/Packet.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -165,6 +165,9 @@ namespace QuantConnect.Packets
         OptimizationStatus,
 
         /// Optimization work result
-        OptimizationResult
+        OptimizationResult,
+
+        /// Python Environment Packet
+        PythonEnvironment
     }
 }

Common/Python/PythonInitializer.cs

@@ -19,6 +19,8 @@ using System.Linq;
 using Python.Runtime;
 using QuantConnect.Logging;
 using System.Collections.Generic;
+using System.IO;
+using System.Runtime.InteropServices;
 using QuantConnect.Util;
 
 namespace QuantConnect.Python
@@ -59,29 +61,75 @@ namespace QuantConnect.Python
         /// </summary>
         public static void AddPythonPaths(IEnumerable<string> paths)
         {
-            if (paths == null)
+            // Filter out any paths that are already on our Python path
+            if (paths.IsNullOrEmpty())
             {
                 return;
             }
 
+            // Add these paths to our pending additions
+            _pendingPathAdditions.AddRange(paths);
+
             if (_isInitialized)
             {
                 using (Py.GIL())
                 {
-                    _pendingPathAdditions.AddRange(paths);
+                    dynamic sys = Py.Import("sys");
+                    var locals = new PyDict();
+                    locals.SetItem("sys", sys);
 
-                    // Generate the python code to add these to our path and execute
-                    var code = string.Join(";", _pendingPathAdditions.Select(s => $"sys.path.append('{s}')"))
-                        .Replace('\\', '/');
+                    // Filter out any already paths that already exist on our current PythonPath
+                    var currentPath = PythonEngine.Eval("sys.path", null, locals.Handle).As<List<string>>();
+                    _pendingPathAdditions = _pendingPathAdditions.Where(x => !currentPath.Contains(x.Replace('\\', '/'))).ToList();
 
-                    PythonEngine.Exec($"import sys;{code}");
-                    _pendingPathAdditions.Clear();
+                    // Insert any pending path additions
+                    if (!_pendingPathAdditions.IsNullOrEmpty())
+                    {
+                        var code = string.Join(";", _pendingPathAdditions
+                            .Select(s => $"sys.path.insert(0, '{s}')")).Replace('\\', '/');
+                        PythonEngine.Exec(code, null, locals.Handle);
+
+                        _pendingPathAdditions.Clear();
+                    }
+
+                    locals.Dispose();
                 }
             }
-            else
+        }
+
+        /// <summary>
+        /// "Activate" a virtual Python environment by prepending its library storage to Pythons
+        /// path. This allows the libraries in this venv to be selected prior to our base install.
+        /// Requires PYTHONNET_PYDLL to be set to base install.
+        /// </summary>
+        /// <remarks>If a module is already loaded, Python will use its cached version first
+        /// these modules must be reloaded by reload() from importlib library</remarks>
+        public static void ActivatePythonVirtualEnvironment(string pathToVirtualEnv)
+        {
+            if (pathToVirtualEnv != null)
             {
-                // Add these paths to our pending additions list
-                _pendingPathAdditions.AddRange(paths);
+                pathToVirtualEnv = pathToVirtualEnv.TrimEnd('/').TrimEnd('\\');
+                var pathsToPrepend = new List<string>();
+                if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
+                {
+                    // For linux we need to know the python version to determine the lib folder containing our packages
+                    // Compare our PyDLL to the directory names under the lib directory and get a match
+                    var pyDll = Environment.GetEnvironmentVariable("PYTHONNET_PYDLL");
+                    var version = Path.GetFileNameWithoutExtension(pyDll);
+                    var libDir = Directory.GetDirectories($"{pathToVirtualEnv}/lib")
+                        .Select(d => new DirectoryInfo(d).Name)
+                        .First(x => version.Contains(x, StringComparison.InvariantCulture));
+
+                    pathsToPrepend.Add($"{pathToVirtualEnv}/lib/{libDir}");
+                    pathsToPrepend.Add($"{pathToVirtualEnv}/lib/{libDir}/site-packages");
+                }
+                else
+                {
+                    pathsToPrepend.Add($"{pathToVirtualEnv}\\Lib");
+                    pathsToPrepend.Add($"{pathToVirtualEnv}\\Lib\\site-packages");
+                }
+
+                AddPythonPaths(pathsToPrepend);
             }
         }
     }

Dockerfile

@@ -8,6 +8,13 @@ FROM quantconnect/lean:foundation
 
 MAINTAINER QuantConnect <contact@quantconnect.com>
 
+#Save foundation image and lean version to environment variable
+ARG LEAN_FOUNDATION_VERSION=UNKNOWN
+ENV LEAN_FOUNDATION_VERSION=$LEAN_FOUNDATION_VERSION
+
+ARG LEAN_TAG=UNKNOWN
+ENV LEAN_VERSION=$LEAN_TAG
+
 #Install Python Tool for Visual Studio Debugger for remote python debugging
 RUN pip install ptvsd
 

DockerfileJupyter

@@ -56,6 +56,11 @@ RUN echo "if [ ! -d \"${WORK}Notebooks\" ]; then mkdir ${WORK}Notebooks; fi && \
     jupyter lab --ip='0.0.0.0' --port=8888 --no-browser --allow-root --notebook-dir=\"Notebooks\" --LabApp.token='' " \
     > start.sh
 
+#Save foundation image and lean version to environment variable
+ARG LEAN_FOUNDATION_VERSION=UNKNOWN
+ENV LEAN_FOUNDATION_VERSION=$LEAN_FOUNDATION_VERSION
+ENV LEAN_VERSION=$LEAN_TAG
+
 RUN chmod -R 777 ${WORK}
 EXPOSE 8888
 WORKDIR $WORK

DockerfileLeanFoundation

@@ -44,6 +44,11 @@ RUN wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-p
     rm packages-microsoft-prod.deb && \
     apt-get clean && apt-get autoclean && apt-get autoremove --purge -y && rm -rf /var/lib/apt/lists/*
 
+# Install libffi7 for rpy2 package
+RUN wget http://es.archive.ubuntu.com/ubuntu/pool/main/libf/libffi/libffi7_3.3-4_amd64.deb && \
+    dpkg -i libffi7_3.3-4_amd64.deb && \
+    rm libffi7_3.3-4_amd64.deb
+
 # Set PythonDLL variable for PythonNet
 ENV PYTHONNET_PYDLL="/opt/miniconda3/lib/libpython3.6m.so"
 
@@ -82,9 +87,7 @@ RUN pip install --no-cache-dir      \
     numba==0.46                     \
     setuptools-git==1.2             \
     xarray==0.15.1                  \
-    plotly==4.7.1                   \
     jupyterlab==2.1.2               \
-    tensorflow==1.15.2              \
     docutils==0.14                  \
     cvxopt==1.2.0                   \
     gensim==3.8.0                   \
@@ -94,7 +97,6 @@ RUN pip install --no-cache-dir      \
     nltk==3.4.5                     \
     pomegranate==0.11.1             \
     graphviz==0.8.4                 \
-    cmdstanpy==0.4                  \
     copulae==0.3.1                  \
     featuretools==0.14.0            \
     PuLP==1.6.8                     \
@@ -110,12 +112,10 @@ RUN pip install --no-cache-dir      \
     PyWavelets==1.1.1               \
     umap-learn==0.4.3               \
     nvidia-ml-py3==7.352.0          \
-    fastai==1.0.61                  \
     arch==4.14                      \
     copulalib==1.1.0                \
     copulas==0.3.3                  \
     creme==0.5.1                    \
-    cufflinks==0.17.3               \
     gym==0.17.2                     \
     ipywidgets==7.5.1               \
     deap==1.3.1                     \
@@ -123,18 +123,13 @@ RUN pip install --no-cache-dir      \
     pykalman==0.9.5                 \
     pyportfolioopt==1.2.2           \
     pyramid-arima==0.9.0            \
-    pyro-ppl==1.3.1                 \
     riskparityportfolio==0.2        \
     sklearn-json==0.1.0             \
     stable-baselines==2.10.0        \
     statistics==1.0.3.5             \
     statsmodels==0.11.1             \
-    tensorforce==0.5.5              \
     QuantLib-Python==1.18           \
-    xgboost==1.1.0                  \
     dtw-python==1.0.5               \
-    cntk==2.7                       \
-    mxnet==1.6                      \
     gluonts==0.4.3                  \
     gplearn==0.4.1                  \
     jax==0.1.68                     \
@@ -144,8 +139,6 @@ RUN pip install --no-cache-dir      \
     neural-tangents==0.2.1          \
     mplfinance==0.12.4a0            \
     hmmlearn==0.2.3                 \
-    catboost==0.23.2                \
-    fastai2==0.0.17                 \
     ppscore==0.0.2                  \
     scikit-tda==0.0.3               \
     ta==0.5.25                      \
@@ -154,7 +147,6 @@ RUN pip install --no-cache-dir      \
     optuna==2.3.0                   \
     findiff==0.8.5                  \
     sktime==0.3.0                   \
-    sktime-dl==0.1.0                \
     hyperopt==0.2.5                 \
     bayesian-optimization==1.2.0    \
     rpy2==3.3.6                     \
@@ -166,8 +158,7 @@ RUN pip install --no-cache-dir      \
     dgl==0.6.1                      \
     ruptures==1.1.3                 \
     simpy==4.0.1                    \
-    scikit-learn-extra==0.2.0       \
-    ray==1.5.1
+    scikit-learn-extra==0.2.0
 
 # feature_selector has overly strict dependency version ranges
 # We already installed close-enough versions of all of its dependencies above
@@ -190,21 +181,8 @@ RUN conda install -y -c conda-forge \
 # Install non-math packages
 RUN conda install -y                \
     blaze=0.11.3                    \
-    tensorflow-base=1.15.0          \
     && conda clean -y --all
 
-# Install math/ML from pytorch
-RUN conda install -y -c pytorch     \
-    pytorch=1.5.0                   \
-    torchvision=0.6.0               \
-    && conda clean -y --all
-
-# Install PyTorch Geometric
-RUN TORCH=$(python -c "import torch; print(torch.__version__)") && \
-    CUDA=$(python -c "import torch; print('cu' + torch.version.cuda.replace('.', ''))") && \
-    pip install --no-cache-dir -f https://pytorch-geometric.com/whl/torch-${TORCH}+${CUDA}.html \
-    torch-scatter==2.0.5 torch-sparse==0.6.7 torch-cluster==1.5.7 torch-spline-conv==1.2.0 torch-geometric==1.7.0
-
 RUN python -m nltk.downloader -d /usr/share/nltk_data punkt && \
     python -m nltk.downloader -d /usr/share/nltk_data vader_lexicon && \
     python -m nltk.downloader -d /usr/share/nltk_data stopwords
@@ -258,14 +236,15 @@ RUN wget https://cdn.quantconnect.com/tigramite/tigramite-4.1.zip && \
     unzip -q tigramite-4.1.zip && cd tigramite-4.1 && \
     python setup.py install && cd .. && rm -rf tigramite-4.1 && rm tigramite-4.1.zip
 
-# Install H2O: https://www.h2o.ai/download/
-RUN wget https://cdn.quantconnect.com/h2o/h2o-3.30.0.3.zip && \
-    unzip -q h2o-3.30.0.3.zip && \
-    pip install h2o-3.30.0.3/python/h2o-3.30.0.3-py2.py3-none-any.whl && \
-    rm -rf h2o-3.30.0.3 && rm h2o-3.30.0.3.zip
 
 # Remove black-listed packages
 RUN pip uninstall -y s3transfer
 
+# Install a shell Python Virtual Environment
+# Selected in Lean with directory via JSON JobPacket "sPythonVirtualEnvironment" or config "python-venv" value
+RUN python -m venv /venv --system-site-packages && . /venv/bin/activate \
+    && python -m ipykernel install --name=venv \
+    && deactivate 
+
 # List all packages
 RUN conda list

DockerfileLeanFoundationARM

@@ -83,7 +83,6 @@ RUN pip install --no-cache-dir      \
     msgpack==1.0.0                  \
     setuptools-git==1.2             \
     xarray==0.15.1                  \
-    plotly==4.7.1                   \
     docutils==0.14                  \
     lightgbm==2.3.0                 \
     nltk==3.4.5                     \
@@ -100,16 +99,13 @@ RUN pip install --no-cache-dir      \
     nvidia-ml-py3==7.352.0          \
     copulalib==1.1.0                \
     copulas==0.3.3                  \
-    cufflinks==0.17.3               \
     gym==0.17.2                     \
     ipywidgets==7.5.1               \
     deap==1.3.1                     \
     pykalman==0.9.5                 \
-    pyro-ppl==1.3.1                 \
     sklearn-json==0.1.0             \
     statistics==1.0.3.5             \
     dtw-python==1.0.5               \
-    mxnet==1.6                      \
     gluonts==0.4.3                  \
     gplearn==0.4.1                  \
     jax==0.1.68                     \
@@ -178,14 +174,14 @@ RUN wget https://cdn.quantconnect.com/tigramite/tigramite-4.1.zip && \
     unzip -q tigramite-4.1.zip && cd tigramite-4.1 && \
     python setup.py install && cd .. && rm -rf tigramite-4.1 && rm tigramite-4.1.zip
 
-# Install H2O: https://www.h2o.ai/download/
-RUN wget https://cdn.quantconnect.com/h2o/h2o-3.30.0.3.zip && \
-    unzip -q h2o-3.30.0.3.zip && \
-    pip install h2o-3.30.0.3/python/h2o-3.30.0.3-py2.py3-none-any.whl && \
-    rm -rf h2o-3.30.0.3 && rm h2o-3.30.0.3.zip
-
 # Remove black-listed packages
 RUN pip uninstall -y s3transfer
 
+# Install a shell Python Virtual Environment
+# Selected in Lean with directory via JSON JobPacket "sPythonVirtualEnvironment" or config "python-venv" value
+RUN python -m venv /venv --system-site-packages && . /venv/bin/activate \
+    && python -m ipykernel install --name=venv \
+    && deactivate 
+
 # List all packages
 RUN conda list

Launcher/Program.cs

@@ -22,6 +22,7 @@ using QuantConnect.Configuration;
 using QuantConnect.Lean.Engine;
 using QuantConnect.Logging;
 using QuantConnect.Packets;
+using QuantConnect.Python;
 using QuantConnect.Util;
 
 namespace QuantConnect.Lean.Launcher
@@ -69,6 +70,12 @@ namespace QuantConnect.Lean.Launcher
             string assemblyPath;
             job = leanEngineSystemHandlers.JobQueue.NextJob(out assemblyPath);
 
+            // Activate our PythonVirtualEnvironment if provided
+            if (!string.IsNullOrEmpty(job.PythonVirtualEnvironment))
+            {
+                PythonInitializer.ActivatePythonVirtualEnvironment(job.PythonVirtualEnvironment);
+            }
+
             leanEngineAlgorithmHandlers = Initializer.GetAlgorithmHandlers();
 
             if (job == null)

Launcher/config.json

@@ -28,6 +28,9 @@
   "debugging": false,
   "debugging-method": "LocalCmdline",
 
+  // location of a python virtual env to use libraries from
+  //"python-venv": "/venv",
+
   // handlers
   "log-handler": "QuantConnect.Logging.CompositeLogHandler",
   "messaging-handler": "QuantConnect.Messaging.Messaging",

Queues/JobQueue.cs

@@ -115,7 +115,8 @@ namespace QuantConnect.Queues
                     DeployId = algorithmId,
                     Parameters = parameters,
                     Language = Language,
-                    Controls = controls
+                    Controls = controls,
+                    PythonVirtualEnvironment = Config.Get("python-venv")
                 };
 
                 try
@@ -147,7 +148,8 @@ namespace QuantConnect.Queues
                 BacktestId = algorithmId,
                 Language = Language,
                 Parameters = parameters,
-                Controls = controls
+                Controls = controls,
+                PythonVirtualEnvironment = Config.Get("python-venv")
             };
 
             return backtestJob;

Tests/AssemblyInitialize.cs

@@ -50,6 +50,14 @@ namespace QuantConnect.Tests
             Directory.SetCurrentDirectory(dir);
             Config.Reset();
             Globals.Reset();
+
+            // Activate virtual environment if defined
+            var pythonVirtualEnvironment = Config.Get("python-venv");
+            if(!string.IsNullOrEmpty(pythonVirtualEnvironment)){
+                PythonInitializer.ActivatePythonVirtualEnvironment(pythonVirtualEnvironment);
+            }
+
+            // Initialize and add our Paths
             PythonInitializer.Initialize();
             PythonInitializer.AddPythonPaths(
                 new[]