Minor fix for option contract universe (#8337)

* Generic command support

- Adding generic algorithm command support. Adding regression algorithms
- Allow PythonWrapper to validate classes too

* Minor improvements

Algorithm.CSharp/CallbackCommandRegressionAlgorithm.cs

@@ -0,0 +1,147 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Commands;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Regression algorithm asserting the behavior of different callback commands call
+    /// </summary>
+    public class CallbackCommandRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        /// <summary>
+        /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
+        /// </summary>
+        public override void Initialize()
+        {
+            SetStartDate(2013, 10, 07);
+            SetEndDate(2013, 10, 11);
+
+            AddEquity("SPY");
+            AddEquity("BAC");
+            AddEquity("IBM");
+            AddCommand<BoolCommand>();
+            AddCommand<VoidCommand>();
+        }
+
+        /// <summary>
+        /// Handle generic command callback
+        /// </summary>
+        public override bool? OnCommand(dynamic data)
+        {
+            Buy(data.Symbol, data.parameters["quantity"]);
+            return true;
+        }
+
+        private class VoidCommand : Command
+        {
+            public DateTime TargetTime { get; set; }
+            public string[] Target { get; set; }
+            public decimal Quantity { get; set; }
+            public Dictionary<string, string> Parameters { get; set; }
+            public override bool? Run(IAlgorithm algorithm)
+            {
+                if (TargetTime != algorithm.Time)
+                {
+                    return null;
+                }
+
+                ((QCAlgorithm)algorithm).Order(Target[0], Quantity, tag: Parameters["tag"]);
+                return null;
+            }
+        }
+        private class BoolCommand : Command
+        {
+            public bool? Result { get; set; }
+            public override bool? Run(IAlgorithm algorithm)
+            {
+                var shouldTrade = MyCustomMethod();
+                if (shouldTrade.HasValue && shouldTrade.Value)
+                {
+                    ((QCAlgorithm)algorithm).Buy("IBM", 1);
+                }
+                return shouldTrade;
+            }
+
+            private bool? MyCustomMethod()
+            {
+                return Result;
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public List<Language> Languages { get; } = new() { Language.CSharp, Language.Python };
+
+        /// <summary>
+        /// Data Points count of all timeslices of algorithm
+        /// </summary>
+        public long DataPoints => 3943;
+
+        /// <summary>
+        /// Data Points count of the algorithm history
+        /// </summary>
+        public int AlgorithmHistoryDataPoints => 0;
+
+        /// <summary>
+        /// Final status of the algorithm
+        /// </summary>
+        public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Orders", "1"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "271.453%"},
+            {"Drawdown", "2.200%"},
+            {"Expectancy", "0"},
+            {"Start Equity", "100000"},
+            {"End Equity", "101691.92"},
+            {"Net Profit", "1.692%"},
+            {"Sharpe Ratio", "8.854"},
+            {"Sortino Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "67.609%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "-0.005"},
+            {"Beta", "0.996"},
+            {"Annual Standard Deviation", "0.222"},
+            {"Annual Variance", "0.049"},
+            {"Information Ratio", "-14.565"},
+            {"Tracking Error", "0.001"},
+            {"Treynor Ratio", "1.97"},
+            {"Total Fees", "$3.44"},
+            {"Estimated Strategy Capacity", "$56000000.00"},
+            {"Lowest Capacity Asset", "SPY R735QTJ8XC9X"},
+            {"Portfolio Turnover", "19.93%"},
+            {"OrderListHash", "3da9fa60bf95b9ed148b95e02e0cfc9e"}
+        };
+    }
+}

Algorithm.Python/CallbackCommandRegressionAlgorithm.py

@@ -0,0 +1,73 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+class InvalidCommand():
+    variable = 10
+
+class VoidCommand():
+    quantity = 0
+    target = []
+    parameters = {}
+    targettime = None
+
+    def run(self, algo: QCAlgorithm) -> bool | None:
+        if not self.targettime or self.targettime != algo.time:
+            return
+        tag = self.parameters["tag"]
+        algo.order(self.target[0], self.get_quantity(), tag=tag)
+
+    def get_quantity(self):
+        return self.quantity
+
+class BoolCommand(Command):
+    result = False
+
+    def run(self, algo: QCAlgorithm) -> bool | None:
+        trade_ibm = self.my_custom_method()
+        if trade_ibm:
+            algo.buy("IBM", 1)
+        return trade_ibm
+
+    def my_custom_method(self):
+        return self.result
+
+### <summary>
+### Regression algorithm asserting the behavior of different callback commands call
+### </summary>
+class CallbackCommandRegressionAlgorithm(QCAlgorithm):
+    def initialize(self):
+        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
+
+        self.set_start_date(2013, 10, 7)
+        self.set_end_date(2013, 10, 11)
+
+        self.add_equity("SPY")
+        self.add_equity("IBM")
+        self.add_equity("BAC")
+
+        self.add_command(VoidCommand)
+        self.add_command(BoolCommand)
+
+        threw_exception = False
+        try:
+            self.add_command(InvalidCommand)
+        except:
+            threw_exception = True
+        if not threw_exception:
+            raise ValueError('InvalidCommand did not throw!')
+
+    def on_command(self, data):
+        self.buy(data.symbol, data.parameters["quantity"])
+        return True # False, None

Algorithm/QCAlgorithm.Python.cs

@@ -31,6 +31,7 @@ using QuantConnect.Scheduling;
 using QuantConnect.Util;
 using QuantConnect.Interfaces;
 using QuantConnect.Orders;
+using QuantConnect.Commands;
 
 namespace QuantConnect.Algorithm
 {
@@ -1620,6 +1621,23 @@ namespace QuantConnect.Algorithm
             return Liquidate(symbols.ConvertToSymbolEnumerable(), asynchronous, tag, orderProperties);
         }
 
+        /// <summary>
+        /// Register a command type to be used
+        /// </summary>
+        /// <param name="type">The command type</param>
+        public void AddCommand(PyObject type)
+        {
+            // create a test instance to validate interface is implemented accurate
+            var testInstance = new CommandPythonWrapper(type);
+
+            var wrappedType = Extensions.CreateType(type);
+            _registeredCommands[wrappedType.Name] = (CallbackCommand command) =>
+            {
+                var commandWrapper = new CommandPythonWrapper(type, command.Payload);
+                return commandWrapper.Run(this);
+            };
+        }
+
         /// <summary>
         /// Gets indicator base type
         /// </summary>

Algorithm/QCAlgorithm.cs

@@ -54,6 +54,8 @@ using QuantConnect.Securities.CryptoFuture;
 using QuantConnect.Algorithm.Framework.Alphas.Analysis;
 using QuantConnect.Algorithm.Framework.Portfolio.SignalExports;
 using Python.Runtime;
+using QuantConnect.Commands;
+using Newtonsoft.Json;
 
 namespace QuantConnect.Algorithm
 {
@@ -115,6 +117,9 @@ namespace QuantConnect.Algorithm
         private IStatisticsService _statisticsService;
         private IBrokerageModel _brokerageModel;
 
+        private readonly HashSet<string> _oneTimeCommandErrors = new();
+        private readonly Dictionary<string, Func<CallbackCommand, bool?>> _registeredCommands = new(StringComparer.InvariantCultureIgnoreCase);
+
         //Error tracking to avoid message flooding:
         private string _previousDebugMessage = "";
         private string _previousErrorMessage = "";
@@ -3385,6 +3390,62 @@ namespace QuantConnect.Algorithm
             return new DataHistory<OptionUniverse>(optionChain, new Lazy<PyObject>(() => PandasConverter.GetDataFrame(optionChain)));
         }
 
+        /// <summary>
+        /// Register a command type to be used
+        /// </summary>
+        /// <typeparam name="T">The command type</typeparam>
+        public void AddCommand<T>() where T : Command
+        {
+            _registeredCommands[typeof(T).Name] = (CallbackCommand command) =>
+            {
+                var commandInstance = JsonConvert.DeserializeObject<T>(command.Payload);
+                return commandInstance.Run(this);
+            };
+        }
+
+        /// <summary>
+        /// Run a callback command instance
+        /// </summary>
+        /// <param name="command">The callback command instance</param>
+        /// <returns>The command result</returns>
+        public CommandResultPacket RunCommand(CallbackCommand command)
+        {
+            bool? result = null;
+            if (_registeredCommands.TryGetValue(command.Type, out var target))
+            {
+                try
+                {
+                    result = target.Invoke(command);
+                }
+                catch (Exception ex)
+                {
+                    QuantConnect.Logging.Log.Error(ex);
+                    if (_oneTimeCommandErrors.Add(command.Type))
+                    {
+                        Log($"Unexpected error running command '{command.Type}' error: '{ex.Message}'");
+                    }
+                }
+            }
+            else
+            {
+                if (_oneTimeCommandErrors.Add(command.Type))
+                {
+                    Log($"Detected unregistered command type '{command.Type}', will be ignored");
+                }
+            }
+            return new CommandResultPacket(command, result) { CommandName = command.Type };
+        }
+
+        /// <summary>
+        /// Generic untyped command call handler
+        /// </summary>
+        /// <param name="data">The associated data</param>
+        /// <returns>True if success, false otherwise. Returning null will disable command feedback</returns>
+        public virtual bool? OnCommand(dynamic data)
+        {
+            return true;
+        }
+
         private static Symbol GetCanonicalOptionSymbol(Symbol symbol)
         {
             // We got the underlying

Algorithm/Selection/OptionContractUniverse.cs

@@ -35,7 +35,7 @@ namespace QuantConnect.Algorithm.Selection
         /// <param name="configuration">The universe configuration to use</param>
         /// <param name="universeSettings">The universe settings to use</param>
         public OptionContractUniverse(SubscriptionDataConfig configuration, UniverseSettings universeSettings)
-            : base(configuration, universeSettings, Time.EndOfTimeTimeSpan,
+            : base(AdjustUniverseConfiguration(configuration), universeSettings, Time.EndOfTimeTimeSpan,
                 // Argument isn't used since we override 'SelectSymbols'
                 Enumerable.Empty<Symbol>())
         {
@@ -95,5 +95,13 @@ namespace QuantConnect.Algorithm.Selection
 
             return new Symbol(sid, ticker);
         }
+
+        /// <summary>
+        /// Make sure the configuration of the universe is what we want
+        /// </summary>
+        private static SubscriptionDataConfig AdjustUniverseConfiguration(SubscriptionDataConfig input)
+        {
+            return new SubscriptionDataConfig(input, fillForward: false);
+        }
     }
 }

AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs

@@ -37,6 +37,7 @@ using QuantConnect.Storage;
 using QuantConnect.Statistics;
 using QuantConnect.Data.Market;
 using QuantConnect.Algorithm.Framework.Alphas.Analysis;
+using QuantConnect.Commands;
 
 namespace QuantConnect.AlgorithmFactory.Python.Wrappers
 {
@@ -62,6 +63,7 @@ namespace QuantConnect.AlgorithmFactory.Python.Wrappers
         private dynamic _onEndOfDay;
         private dynamic _onMarginCallWarning;
         private dynamic _onOrderEvent;
+        private dynamic _onCommand;
         private dynamic _onAssignmentOrderEvent;
         private dynamic _onSecuritiesChanged;
         private dynamic _onFrameworkSecuritiesChanged;
@@ -153,6 +155,7 @@ namespace QuantConnect.AlgorithmFactory.Python.Wrappers
                             _onDelistings = _algorithm.GetMethod("OnDelistings");
                             _onSymbolChangedEvents = _algorithm.GetMethod("OnSymbolChangedEvents");
                             _onEndOfDay = _algorithm.GetMethod("OnEndOfDay");
+                            _onCommand = _algorithm.GetMethod("OnCommand");
                             _onMarginCallWarning = _algorithm.GetMethod("OnMarginCallWarning");
                             _onOrderEvent = _algorithm.GetMethod("OnOrderEvent");
                             _onAssignmentOrderEvent = _algorithm.GetMethod("OnAssignmentOrderEvent");
@@ -921,6 +924,16 @@ namespace QuantConnect.AlgorithmFactory.Python.Wrappers
             _onOrderEvent(newEvent);
         }
 
+        /// <summary>
+        /// Generic untyped command call handler
+        /// </summary>
+        /// <param name="data">The associated data</param>
+        /// <returns>True if success, false otherwise. Returning null will disable command feedback</returns>
+        public bool? OnCommand(dynamic data)
+        {
+            return _onCommand(data);
+        }
+
         /// <summary>
         /// Will submit an order request to the algorithm
         /// </summary>
@@ -1242,5 +1255,13 @@ namespace QuantConnect.AlgorithmFactory.Python.Wrappers
         {
             _baseAlgorithm.SetTags(tags);
         }
+
+        /// <summary>
+        /// Run a callback command instance
+        /// </summary>
+        /// <param name="command">The callback command instance</param>
+        /// <returns>The command result</returns>
+        public CommandResultPacket RunCommand(CallbackCommand command) => _baseAlgorithm.RunCommand(command);
+
     }
 }

Api/Api.cs

@@ -1023,7 +1023,6 @@ namespace QuantConnect.Api
         /// </summary>
         /// <param name="projectId">Project for the live instance we want to stop</param>
         /// <returns><see cref="RestResponse"/></returns>
-
         public RestResponse StopLiveAlgorithm(int projectId)
         {
             var request = new RestRequest("live/update/stop", Method.POST)
@@ -1040,6 +1039,29 @@ namespace QuantConnect.Api
             return result;
         }
 
+        /// <summary>
+        /// Create a live command
+        /// </summary>
+        /// <param name="projectId">Project for the live instance we want to run the command against</param>
+        /// <param name="command">The command to run</param>
+        /// <returns><see cref="RestResponse"/></returns>
+        public RestResponse CreateLiveCommand(int projectId, object command)
+        {
+            var request = new RestRequest("live/commands/create", Method.POST)
+            {
+                RequestFormat = DataFormat.Json
+            };
+
+            request.AddParameter("application/json", JsonConvert.SerializeObject(new
+            {
+                projectId,
+                command
+            }), ParameterType.RequestBody);
+
+            ApiConnection.TryRequest(request, out RestResponse result);
+            return result;
+        }
+
         /// <summary>
         /// Gets the logs of a specific live algorithm
         /// </summary>

Common/AlgorithmImports.py

@@ -39,6 +39,7 @@ from QuantConnect.Orders import *
 from QuantConnect.Python import *
 from QuantConnect.Storage import *
 from QuantConnect.Research import *
+from QuantConnect.Commands import *
 from QuantConnect.Algorithm import *
 from QuantConnect.Statistics import *
 from QuantConnect.Parameters import *

Common/Commands/BaseCommandHandler.cs

@@ -15,8 +15,10 @@
 
 using System;
 using System.Linq;
+using Newtonsoft.Json;
 using QuantConnect.Logging;
 using QuantConnect.Packets;
+using Newtonsoft.Json.Linq;
 using QuantConnect.Interfaces;
 using System.Collections.Generic;
 
@@ -27,6 +29,8 @@ namespace QuantConnect.Commands
     /// </summary>
     public abstract class BaseCommandHandler : ICommandHandler
     {
+        protected static readonly JsonSerializerSettings Settings = new() { TypeNameHandling = TypeNameHandling.All };
+
         /// <summary>
         /// The algorithm instance
         /// </summary>
@@ -104,5 +108,41 @@ namespace QuantConnect.Commands
         {
             // nop
         }
+
+        /// <summary>
+        /// Helper method to create a callback command
+        /// </summary>
+        protected ICommand TryGetCallbackCommand(string payload)
+        {
+            Dictionary<string, JToken> deserialized = new(StringComparer.InvariantCultureIgnoreCase);
+            try
+            {
+                if (!string.IsNullOrEmpty(payload))
+                {
+                    var jobject = JObject.Parse(payload);
+                    foreach (var kv in jobject)
+                    {
+                        deserialized[kv.Key] = kv.Value;
+                    }
+                }
+            }
+            catch (Exception err)
+            {
+                Log.Error(err, $"Payload: '{payload}'");
+                return null;
+            }
+
+            if (!deserialized.TryGetValue("id", out var id) || id == null)
+            {
+                id = string.Empty;
+            }
+
+            if (!deserialized.TryGetValue("$type", out var type) || type == null)
+            {
+                type = string.Empty;
+            }
+
+            return new CallbackCommand { Id = id.ToString(), Type = type.ToString(), Payload = payload };
+        }
     }
 }

Common/Commands/CallbackCommand.cs

@@ -0,0 +1,63 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using Newtonsoft.Json;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Commands
+{
+    /// <summary>
+    /// Algorithm callback command type
+    /// </summary>
+    public class CallbackCommand : BaseCommand
+    {
+        /// <summary>
+        /// The target command type to run, if empty or null will be the generic untyped command handler
+        /// </summary>
+        public string Type { get; set; }
+
+        /// <summary>
+        /// The command payload
+        /// </summary>
+        public string Payload { get; set; }
+
+        /// <summary>
+        /// Runs this command against the specified algorithm instance
+        /// </summary>
+        /// <param name="algorithm">The algorithm to run this command against</param>
+        public override CommandResultPacket Run(IAlgorithm algorithm)
+        {
+            if (string.IsNullOrEmpty(Type))
+            {
+                // target is the untyped algorithm handler
+                var result = algorithm.OnCommand(string.IsNullOrEmpty(Payload) ? null : JsonConvert.DeserializeObject<Command>(Payload));
+                return new CommandResultPacket(this, result);
+            }
+            return algorithm.RunCommand(this);
+        }
+
+        /// <summary>
+        /// The command string representation
+        /// </summary>
+        public override string ToString()
+        {
+            if (!string.IsNullOrEmpty(Type))
+            {
+                return Type;
+            }
+            return "OnCommand";
+        }
+    }
+}

Common/Commands/Command.cs

@@ -0,0 +1,91 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Dynamic;
+using QuantConnect.Data;
+using System.Reflection;
+using System.Linq.Expressions;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using Newtonsoft.Json.Linq;
+
+namespace QuantConnect.Commands
+{
+    /// <summary>
+    /// Base generic dynamic command class
+    /// </summary>
+    public class Command : DynamicObject
+    {
+        private static readonly MethodInfo SetPropertyMethodInfo = typeof(Command).GetMethod("SetProperty");
+        private static readonly MethodInfo GetPropertyMethodInfo = typeof(Command).GetMethod("GetProperty");
+
+        private readonly Dictionary<string, object> _storage = new(StringComparer.InvariantCultureIgnoreCase);
+
+        /// <summary>
+        /// Get the metaObject required for Dynamism.
+        /// </summary>
+        public sealed override DynamicMetaObject GetMetaObject(Expression parameter)
+        {
+            return new GetSetPropertyDynamicMetaObject(parameter, this, SetPropertyMethodInfo, GetPropertyMethodInfo);
+        }
+
+        /// <summary>
+        /// Sets the property with the specified name to the value. This is a case-insensitve search.
+        /// </summary>
+        /// <param name="name">The property name to set</param>
+        /// <param name="value">The new property value</param>
+        /// <returns>Returns the input value back to the caller</returns>
+        public object SetProperty(string name, object value)
+        {
+            if (value is JArray jArray)
+            {
+                return _storage[name] = jArray.ToObject<List<object>>();
+            }
+            else if (value is JObject jobject)
+            {
+                return _storage[name] = jobject.ToObject<Dictionary<string, object>>();
+            }
+            else
+            {
+                return _storage[name] = value;
+            }
+        }
+
+        /// <summary>
+        /// Gets the property's value with the specified name. This is a case-insensitve search.
+        /// </summary>
+        /// <param name="name">The property name to access</param>
+        /// <returns>object value of BaseData</returns>
+        public object GetProperty(string name)
+        {
+            if (!_storage.TryGetValue(name, out var value))
+            {
+                throw new KeyNotFoundException($"Property with name \'{name}\' does not exist. Properties: {string.Join(", ", _storage.Keys)}");
+            }
+            return value;
+        }
+
+        /// <summary>
+        /// Run this command using the target algorithm
+        /// </summary>
+        /// <param name="algorithm">The algorithm instance</param>
+        /// <returns>True if success, false otherwise. Returning null will disable command feedback</returns>
+        public virtual bool? Run(IAlgorithm algorithm)
+        {
+            throw new NotImplementedException($"Please implement the 'def run(algorithm) -> bool | None:' method");
+        }
+    }
+}

Common/Commands/CommandResultsPacket.cs

@@ -31,12 +31,12 @@ namespace QuantConnect.Commands
         /// <summary>
         /// Gets or sets whether or not the
         /// </summary>
-        public bool Success { get; set; }
+        public bool? Success { get; set; }
 
         /// <summary>
         /// Initializes a new instance of the <see cref="CommandResultPacket"/> class
         /// </summary>
-        public CommandResultPacket(ICommand command, bool success)
+        public CommandResultPacket(ICommand command, bool? success)
             : base(PacketType.CommandResult)
         {
             Success = success;

Common/Commands/FileCommandHandler.cs

@@ -19,6 +19,7 @@ using Newtonsoft.Json;
 using QuantConnect.Logging;
 using System.Collections.Generic;
 using System.Linq;
+using Newtonsoft.Json.Linq;
 
 namespace QuantConnect.Commands
 {
@@ -90,7 +91,9 @@ namespace QuantConnect.Commands
         private void ReadCommandFile(string commandFilePath)
         {
             Log.Trace($"FileCommandHandler.ReadCommandFile(): {Messages.FileCommandHandler.ReadingCommandFile(commandFilePath)}");
-            object deserialized;
+            string contents = null;
+            Exception exception = null;
+            object deserialized = null;
             try
             {
                 if (!File.Exists(commandFilePath))
@@ -98,13 +101,12 @@ namespace QuantConnect.Commands
                     Log.Error($"FileCommandHandler.ReadCommandFile(): {Messages.FileCommandHandler.CommandFileDoesNotExist(commandFilePath)}");
                     return;
                 }
-                var contents = File.ReadAllText(commandFilePath);
-                deserialized = JsonConvert.DeserializeObject(contents, new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.All });
+                contents = File.ReadAllText(commandFilePath);
+                deserialized = JsonConvert.DeserializeObject(contents, Settings);
             }
             catch (Exception err)
             {
-                Log.Error(err);
-                deserialized = null;
+                exception = err;
             }
 
             // remove the file when we're done reading it
@@ -126,6 +128,20 @@ namespace QuantConnect.Commands
             if (item != null)
             {
                 _commands.Enqueue(item);
+                return;
+            }
+
+            var callbackCommand = TryGetCallbackCommand(contents);
+            if (callbackCommand != null)
+            {
+                _commands.Enqueue(callbackCommand);
+                return;
+            }
+
+            if (exception != null)
+            {
+                // if we are here we failed
+                Log.Error(exception);
             }
         }
     }

Common/Data/DataQueueHandlerSubscriptionManager.cs

@@ -102,6 +102,21 @@ namespace QuantConnect.Data
                 .Distinct();
         }
 
+        /// <summary>
+        /// Retrieves the list of unique <see cref="Symbol"/> instances that are currently subscribed for a specific <see cref="TickType"/>.
+        /// </summary>
+        /// <param name="tickType">The type of tick data to filter subscriptions by.</param>
+        /// <returns>A collection of unique <see cref="Symbol"/> objects that match the specified <paramref name="tickType"/>.</returns>
+        public IEnumerable<Symbol> GetSubscribedSymbols(TickType tickType)
+        {
+            var channelName = ChannelNameFromTickType(tickType);
+#pragma warning disable CA1309
+            return SubscribersByChannel.Keys.Where(x => x.Name.Equals(channelName, StringComparison.InvariantCultureIgnoreCase))
+#pragma warning restore CA1309
+                .Select(c => c.Symbol)
+                .Distinct();
+        }
+
         /// <summary>
         /// Checks if there is existing subscriber for current channel
         /// </summary>

Common/Data/Market/Tick.cs

@@ -231,6 +231,21 @@ namespace QuantConnect.Data.Market
             AskPrice = ask;
         }
 
+        /// <summary>
+        /// Initializes a new instance of the <see cref="Tick"/> class to <see cref="TickType.OpenInterest"/>.
+        /// </summary>
+        /// <param name="time">The time at which the open interest tick occurred.</param>
+        /// <param name="symbol">The symbol associated with the open interest tick.</param>
+        /// <param name="openInterest">The value of the open interest for the specified symbol.</param>
+        public Tick(DateTime time, Symbol symbol, decimal openInterest)
+        {
+            Time = time;
+            Symbol = symbol;
+            Value = openInterest;
+            DataType = MarketDataType.Tick;
+            TickType = TickType.OpenInterest;
+        }
+
         /// <summary>
         /// Initializer for a last-trade equity tick with bid or ask prices.
         /// </summary>

Common/Interfaces/IAlgorithm.cs

@@ -32,6 +32,7 @@ using QuantConnect.Securities.Option;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Algorithm.Framework.Alphas;
 using QuantConnect.Algorithm.Framework.Alphas.Analysis;
+using QuantConnect.Commands;
 
 namespace QuantConnect.Interfaces
 {
@@ -608,6 +609,13 @@ namespace QuantConnect.Interfaces
         /// <param name="newEvent">Event information</param>
         void OnOrderEvent(OrderEvent newEvent);
 
+        /// <summary>
+        /// Generic untyped command call handler
+        /// </summary>
+        /// <param name="data">The associated data</param>
+        /// <returns>True if success, false otherwise. Returning null will disable command feedback</returns>
+        bool? OnCommand(dynamic data);
+
         /// <summary>
         /// Will submit an order request to the algorithm
         /// </summary>
@@ -919,5 +927,12 @@ namespace QuantConnect.Interfaces
         /// </summary>
         /// <param name="tags">The tags</param>
         void SetTags(HashSet<string> tags);
+
+        /// <summary>
+        /// Run a callback command instance
+        /// </summary>
+        /// <param name="command">The callback command instance</param>
+        /// <returns>The command result</returns>
+        CommandResultPacket RunCommand(CallbackCommand command);
     }
 }

Common/Isolator.cs

@@ -18,7 +18,6 @@ using System.Threading;
 using System.Threading.Tasks;
 using QuantConnect.Logging;
 using QuantConnect.Util;
-using static QuantConnect.StringExtensions;
 
 namespace QuantConnect
 {
@@ -36,22 +35,6 @@ namespace QuantConnect
             get; private set;
         }
 
-        /// <summary>
-        /// Algo cancellation controls - cancellation token for algorithm thread.
-        /// </summary>
-        public CancellationToken CancellationToken
-        {
-            get { return CancellationTokenSource.Token; }
-        }
-
-        /// <summary>
-        /// Check if this task isolator is cancelled, and exit the analysis
-        /// </summary>
-        public bool IsCancellationRequested
-        {
-            get { return CancellationTokenSource.IsCancellationRequested; }
-        }
-
         /// <summary>
         /// Initializes a new instance of the <see cref="Isolator"/> class
         /// </summary>
@@ -117,7 +100,7 @@ namespace QuantConnect
             memoryCap *= 1024 * 1024;
             var spikeLimit = memoryCap*2;
 
-            while (!task.IsCompleted && utcNow < end)
+            while (!task.IsCompleted && !CancellationTokenSource.IsCancellationRequested && utcNow < end)
             {
                 // if over 80% allocation force GC then sample
                 var sample = Convert.ToDouble(GC.GetTotalMemory(memoryUsed > memoryCap * 0.8));
@@ -166,15 +149,26 @@ namespace QuantConnect
                 utcNow = DateTime.UtcNow;
             }
 
-            if (task.IsCompleted == false && string.IsNullOrEmpty(message))
+            if (task.IsCompleted == false)
             {
-                message = Messages.Isolator.MemoryUsageMonitorTaskTimedOut(timeSpan);
-                Log.Trace($"Isolator.ExecuteWithTimeLimit(): {message}");
+                if (CancellationTokenSource.IsCancellationRequested)
+                {
+                    Log.Trace($"Isolator.ExecuteWithTimeLimit(): Operation was canceled");
+                    throw new OperationCanceledException("Operation was canceled");
+                }
+                else if (string.IsNullOrEmpty(message))
+                {
+                    message = Messages.Isolator.MemoryUsageMonitorTaskTimedOut(timeSpan);
+                    Log.Trace($"Isolator.ExecuteWithTimeLimit(): {message}");
+                }
             }
 
             if (!string.IsNullOrEmpty(message))
             {
-                CancellationTokenSource.Cancel();
+                if (!CancellationTokenSource.IsCancellationRequested)
+                {
+                    CancellationTokenSource.Cancel();
+                }
                 Log.Error($"Security.ExecuteWithTimeLimit(): {message}");
                 throw new TimeoutException(message);
             }

Common/Python/CommandPythonWrapper.cs

@@ -0,0 +1,74 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using Python.Runtime;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using QuantConnect.Commands;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+
+namespace QuantConnect.Python
+{
+    /// <summary>
+    /// Python wrapper for a python defined command type
+    /// </summary>
+    public class CommandPythonWrapper : BasePythonWrapper<Command>
+    {
+        /// <summary>
+        /// Constructor for initialising the <see cref="CommandPythonWrapper"/> class with wrapped <see cref="PyObject"/> object
+        /// </summary>
+        /// <param name="type">Python command type</param>
+        /// <param name="data">Command data</param>
+        public CommandPythonWrapper(PyObject type, string data = null)
+            : base()
+        {
+            using var _ = Py.GIL();
+
+            var instance = type.Invoke();
+
+            SetPythonInstance(instance);
+            if (data != null)
+            {
+                foreach (var kvp in JsonConvert.DeserializeObject<Dictionary<string, object>>(data))
+                {
+                    if (kvp.Value is JArray jArray)
+                    {
+                        SetProperty(kvp.Key, jArray.ToObject<List<object>>());
+                    }
+                    else if (kvp.Value is JObject jobject)
+                    {
+                        SetProperty(kvp.Key, jobject.ToObject<Dictionary<string, object>>());
+                    }
+                    else
+                    {
+                        SetProperty(kvp.Key, kvp.Value);
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Run this command using the target algorithm
+        /// </summary>
+        /// <param name="algorithm">The algorithm instance</param>
+        /// <returns>True if success, false otherwise. Returning null will disable command feedback</returns>
+        public bool? Run(IAlgorithm algorithm)
+        {
+            var result = InvokeMethod(nameof(Run), algorithm);
+            return result.GetAndDispose<bool?>();
+        }
+    }
+}

Common/Python/PythonWrapper.cs

@@ -34,21 +34,28 @@ namespace QuantConnect.Python
         /// <param name="model">The model implementing the interface type</param>
         public static PyObject ValidateImplementationOf<TInterface>(this PyObject model)
         {
-            if (!typeof(TInterface).IsInterface)
-            {
-                throw new ArgumentException(
-                    $"{nameof(PythonWrapper)}.{nameof(ValidateImplementationOf)}(): {Messages.PythonWrapper.ExpectedInterfaceTypeParameter}");
-            }
-
+            var notInterface = !typeof(TInterface).IsInterface;
             var missingMembers = new List<string>();
             var members = typeof(TInterface).GetMembers(BindingFlags.Public | BindingFlags.Instance);
             using (Py.GIL())
             {
                 foreach (var member in members)
                 {
-                    if ((member is not MethodInfo method || !method.IsSpecialName) &&
+                    var method = member as MethodInfo;
+                    if ((method == null || !method.IsSpecialName) &&
                         !model.HasAttr(member.Name) && !model.HasAttr(member.Name.ToSnakeCase()))
                     {
+                        if (notInterface)
+                        {
+                            if (method != null && !method.IsAbstract && (method.IsFinal || !method.IsVirtual || method.DeclaringType != typeof(TInterface)))
+                            {
+                                continue;
+                            }
+                            else if (member is ConstructorInfo)
+                            {
+                                continue;
+                            }
+                        }
                         missingMembers.Add(member.Name);
                     }
                 }

Engine/AlgorithmManager.cs

@@ -48,6 +48,8 @@ namespace QuantConnect.Lean.Engine
         private IAlgorithm _algorithm;
         private readonly object _lock;
         private readonly bool _liveMode;
+        private bool _cancelRequested;
+        private CancellationTokenSource _cancellationTokenSource;
 
         /// <summary>
         /// Publicly accessible algorithm status
@@ -111,14 +113,17 @@ namespace QuantConnect.Lean.Engine
         /// <param name="results">Result handler object</param>
         /// <param name="realtime">Realtime processing object</param>
         /// <param name="leanManager">ILeanManager implementation that is updated periodically with the IAlgorithm instance</param>
-        /// <param name="token">Cancellation token</param>
+        /// <param name="cancellationTokenSource">Cancellation token source to monitor</param>
         /// <remarks>Modify with caution</remarks>
-        public void Run(AlgorithmNodePacket job, IAlgorithm algorithm, ISynchronizer synchronizer, ITransactionHandler transactions, IResultHandler results, IRealTimeHandler realtime, ILeanManager leanManager, CancellationToken token)
+        public void Run(AlgorithmNodePacket job, IAlgorithm algorithm, ISynchronizer synchronizer, ITransactionHandler transactions, IResultHandler results, IRealTimeHandler realtime, ILeanManager leanManager, CancellationTokenSource cancellationTokenSource)
         {
             //Initialize:
             DataPoints = 0;
             _algorithm = algorithm;
 
+            var token = cancellationTokenSource.Token;
+            _cancellationTokenSource = cancellationTokenSource;
+
             var backtestMode = (job.Type == PacketType.BacktestNode);
             var methodInvokers = new Dictionary<Type, MethodInvoker>();
             var marginCallFrequency = TimeSpan.FromMinutes(5);
@@ -607,6 +612,22 @@ namespace QuantConnect.Lean.Engine
                 {
                     _algorithm.SetStatus(state);
                 }
+
+                if (_cancellationTokenSource != null && !_cancellationTokenSource.IsCancellationRequested && !_cancelRequested)
+                {
+                    if (state == AlgorithmStatus.Deleted)
+                    {
+                        _cancelRequested = true;
+                        // if the algorithm was deleted, let's give the algorithm a few seconds to shutdown and cancel it out
+                        _cancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(5));
+                    }
+                    else if (state == AlgorithmStatus.Stopped)
+                    {
+                        _cancelRequested = true;
+                        // if the algorithm was stopped, let's give the algorithm a few seconds to shutdown and cancel it out
+                        _cancellationTokenSource.CancelAfter(TimeSpan.FromMinutes(1));
+                    }
+                }
             }
         }
 

Engine/DataFeeds/FileSystemDataFeed.cs

@@ -196,11 +196,6 @@ namespace QuantConnect.Lean.Engine.DataFeeds
                 factory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse,
                     _marketHoursDatabase,
                     _timeProvider);
-
-                if (request.Universe is UserDefinedUniverse)
-                {
-                    return factory.CreateEnumerator(request, _dataProvider);
-                }
             }
             else if (request.Configuration.Type == typeof(FundamentalUniverse))
             {

Engine/Engine.cs

@@ -334,7 +334,7 @@ namespace QuantConnect.Lean.Engine
                                 // -> Using this Data Feed,
                                 // -> Send Orders to this TransactionHandler,
                                 // -> Send Results to ResultHandler.
-                                algorithmManager.Run(job, algorithm, synchronizer, AlgorithmHandlers.Transactions, AlgorithmHandlers.Results, AlgorithmHandlers.RealTime, SystemHandlers.LeanManager, isolator.CancellationToken);
+                                algorithmManager.Run(job, algorithm, synchronizer, AlgorithmHandlers.Transactions, AlgorithmHandlers.Results, AlgorithmHandlers.RealTime, SystemHandlers.LeanManager, isolator.CancellationTokenSource);
                             }
                             catch (Exception err)
                             {

Engine/RealTime/LiveTradingRealTimeHandler.cs

@@ -209,7 +209,7 @@ namespace QuantConnect.Lean.Engine.RealTime
         /// </summary>
         public override void Exit()
         {
-            _realTimeThread.StopSafely(TimeSpan.FromMinutes(5), _cancellationTokenSource);
+            _realTimeThread.StopSafely(TimeSpan.FromMinutes(1), _cancellationTokenSource);
             _cancellationTokenSource.DisposeSafely();
             base.Exit();
         }

Engine/Server/LocalLeanManager.cs

@@ -91,8 +91,7 @@ namespace QuantConnect.Lean.Engine.Server
         {
             if (Algorithm.LiveMode)
             {
-                _commandHandler = new FileCommandHandler();
-                _commandHandler.Initialize(_job, Algorithm);
+                SetCommandHandler();
             }
         }
 
@@ -119,5 +118,14 @@ namespace QuantConnect.Lean.Engine.Server
         {
             _commandHandler.DisposeSafely();
         }
+
+        /// <summary>
+        /// Set the command handler to use, protected for testing purposes
+        /// </summary>
+        protected virtual void SetCommandHandler()
+        {
+            _commandHandler = new FileCommandHandler();
+            _commandHandler.Initialize(_job, Algorithm);
+        }
     }
 }

Tests/Api/ApiTestBase.cs

@@ -86,7 +86,7 @@ namespace QuantConnect.Tests.API
                 return;
             }
             Log.Debug("ApiTestBase.Setup(): Waiting for test compile to complete");
-            compile = WaitForCompilerResponse(TestProject.ProjectId, compile.CompileId);
+            compile = WaitForCompilerResponse(ApiClient, TestProject.ProjectId, compile.CompileId);
             if (!compile.Success)
             {
                 Assert.Warn("Could not create compile for the test project, tests using it will fail.");
@@ -134,14 +134,14 @@ namespace QuantConnect.Tests.API
         /// <param name="projectId">Id of the project</param>
         /// <param name="compileId">Id of the compilation of the project</param>
         /// <returns></returns>
-        protected Compile WaitForCompilerResponse(int projectId, string compileId)
+        protected static Compile WaitForCompilerResponse(Api.Api apiClient, int projectId, string compileId, int seconds = 60)
         {
-            Compile compile;
-            var finish = DateTime.UtcNow.AddSeconds(60);
+            var compile = new Compile();
+            var finish = DateTime.UtcNow.AddSeconds(seconds);
             do
             {
-                Thread.Sleep(1000);
-                compile = ApiClient.ReadCompile(projectId, compileId);
+                Thread.Sleep(100);
+                compile = apiClient.ReadCompile(projectId, compileId);
             } while (compile.State != CompileState.BuildSuccess && DateTime.UtcNow < finish);
 
             return compile;
@@ -169,7 +169,7 @@ namespace QuantConnect.Tests.API
         /// <summary>
         /// Reload configuration, making sure environment variables are loaded into the config
         /// </summary>
-        private static void ReloadConfiguration()
+        internal static void ReloadConfiguration()
         {
             // nunit 3 sets the current folder to a temp folder we need it to be the test bin output folder
             var dir = TestContext.CurrentContext.TestDirectory;

Tests/Api/CommandTests.cs

@@ -0,0 +1,120 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using NUnit.Framework;
+using QuantConnect.Api;
+using System.Threading;
+using System.Collections.Generic;
+
+namespace QuantConnect.Tests.API
+{
+    [TestFixture, Explicit("Requires configured api access, a live node to run on, and brokerage configurations.")]
+    public class CommandTests
+    {
+        private Api.Api _apiClient;
+
+        [OneTimeSetUp]
+        public void Setup()
+        {
+            ApiTestBase.ReloadConfiguration();
+
+            _apiClient = new Api.Api();
+            _apiClient.Initialize(Globals.UserId, Globals.UserToken, Globals.DataFolder);
+        }
+
+        [TestCase("MyCommand")]
+        [TestCase("MyCommand2")]
+        [TestCase("MyCommand3")]
+        [TestCase("")]
+        public void LiveCommand(string commandType)
+        {
+            var command = new Dictionary<string, object>
+            {
+                { "quantity", 0.1 },
+                { "target", "BTCUSD" },
+                { "$type", commandType }
+            };
+
+            var projectId = RunLiveAlgorithm();
+            try
+            {
+                // allow algo to be deployed and prices to be set so we can trade
+                Thread.Sleep(TimeSpan.FromSeconds(10));
+                var result = _apiClient.CreateLiveCommand(projectId, command);
+                Assert.IsTrue(result.Success);
+            }
+            finally
+            {
+                _apiClient.StopLiveAlgorithm(projectId);
+                _apiClient.DeleteProject(projectId);
+            }
+        }
+
+        private int RunLiveAlgorithm()
+        {
+            var settings = new Dictionary<string, object>()
+            {
+                { "id", "QuantConnectBrokerage" },
+                { "environment", "paper" },
+                { "user", "" },
+                { "password", "" },
+                { "account", "" }
+            };
+
+            var file = new ProjectFile
+            {
+                Name = "Main.cs",
+                Code = @"from AlgorithmImports import *
+
+class MyCommand():
+    quantity = 0
+    target = ''
+    def run(self, algo: QCAlgorithm) -> bool | None:
+        self.execute_order(algo)
+    def execute_order(self, algo):
+        algo.order(self.target, self.quantity)
+
+class MyCommand2():
+    quantity = 0
+    target = ''
+    def run(self, algo: QCAlgorithm) -> bool | None:
+        algo.order(self.target, self.quantity)
+        return True
+
+class MyCommand3():
+    quantity = 0
+    target = ''
+    def run(self, algo: QCAlgorithm) -> bool | None:
+        algo.order(self.target, self.quantity)
+        return False
+
+class DeterminedSkyBlueGorilla(QCAlgorithm):
+    def initialize(self):
+        self.set_start_date(2023, 3, 17)
+        self.add_crypto(""BTCUSD"", Resolution.SECOND)
+        self.add_command(MyCommand)
+        self.add_command(MyCommand2)
+        self.add_command(MyCommand3)
+
+    def on_command(self, data):
+        self.order(data.target, data.quantity)"
+            };
+
+            // Run the live algorithm
+            return LiveTradingTests.RunLiveAlgorithm(_apiClient, settings, file, stopLiveAlgos: false, language: Language.Python);
+        }
+    }
+}

Tests/Api/LiveTradingTests.cs

@@ -721,42 +721,48 @@ namespace QuantConnect.Tests.API
         /// <param name="dataProviders">Dictionary with the data providers and their corresponding credentials</param>
         /// <returns>The id of the project created with the algorithm in</returns>
         private int RunLiveAlgorithm(Dictionary<string, object> settings, ProjectFile file, bool stopLiveAlgos, Dictionary<string, object> dataProviders = null)
+        {
+            return RunLiveAlgorithm(ApiClient, settings, file, stopLiveAlgos, dataProviders);
+        }
+
+        internal static int RunLiveAlgorithm(Api.Api apiClient, Dictionary<string, object> settings, ProjectFile file, bool stopLiveAlgos,
+            Dictionary<string, object> dataProviders = null, Language language = Language.CSharp)
         {
             // Create a new project
-            var project = ApiClient.CreateProject($"Test project - {DateTime.Now.ToStringInvariant()}", Language.CSharp, TestOrganization);
+            var project = apiClient.CreateProject($"Test project - {DateTime.Now.ToStringInvariant()}", language, Globals.OrganizationID);
             var projectId = project.Projects.First().ProjectId;
 
             // Update Project Files
-            var updateProjectFileContent = ApiClient.UpdateProjectFileContent(projectId, "Main.cs", file.Code);
+            var updateProjectFileContent = apiClient.UpdateProjectFileContent(projectId, language == Language.CSharp ? "Main.cs" : "main.py", file.Code);
             Assert.IsTrue(updateProjectFileContent.Success);
 
             // Create compile
-            var compile = ApiClient.CreateCompile(projectId);
+            var compile = apiClient.CreateCompile(projectId);
             Assert.IsTrue(compile.Success);
 
             // Wait at max 30 seconds for project to compile
-            var compileCheck = WaitForCompilerResponse(projectId, compile.CompileId, 30);
+            var compileCheck = WaitForCompilerResponse(apiClient, projectId, compile.CompileId, 30);
             Assert.IsTrue(compileCheck.Success);
             Assert.IsTrue(compileCheck.State == CompileState.BuildSuccess);
 
             // Get a live node to launch the algorithm on
-            var nodesResponse = ApiClient.ReadProjectNodes(projectId);
+            var nodesResponse = apiClient.ReadProjectNodes(projectId);
             Assert.IsTrue(nodesResponse.Success);
             var freeNode = nodesResponse.Nodes.LiveNodes.Where(x => x.Busy == false);
             Assert.IsNotEmpty(freeNode, "No free Live Nodes found");
 
             // Create live default algorithm
-            var createLiveAlgorithm = ApiClient.CreateLiveAlgorithm(projectId, compile.CompileId, freeNode.FirstOrDefault().Id, settings, dataProviders: dataProviders);
+            var createLiveAlgorithm = apiClient.CreateLiveAlgorithm(projectId, compile.CompileId, freeNode.FirstOrDefault().Id, settings, dataProviders: dataProviders);
             Assert.IsTrue(createLiveAlgorithm.Success);
 
             if (stopLiveAlgos)
             {
                 // Liquidate live algorithm; will also stop algorithm
-                var liquidateLive = ApiClient.LiquidateLiveAlgorithm(projectId);
+                var liquidateLive = apiClient.LiquidateLiveAlgorithm(projectId);
                 Assert.IsTrue(liquidateLive.Success);
 
                 // Delete the project
-                var deleteProject = ApiClient.DeleteProject(projectId);
+                var deleteProject = apiClient.DeleteProject(projectId);
                 Assert.IsTrue(deleteProject.Success);
             }
 
@@ -820,7 +826,7 @@ namespace QuantConnect.Tests.API
             Assert.IsTrue(compile.Success);
 
             // Wait at max 30 seconds for project to compile
-            var compileCheck = WaitForCompilerResponse(projectId, compile.CompileId, 30);
+            var compileCheck = WaitForCompilerResponse(ApiClient, projectId, compile.CompileId, 30);
             Assert.IsTrue(compileCheck.Success);
             Assert.IsTrue(compileCheck.State == CompileState.BuildSuccess);
 
@@ -863,26 +869,6 @@ def CreateLiveAlgorithmFromPython(apiClient, projectId, compileId, nodeId):
             }
         }
 
-        /// <summary>
-        /// Wait for the compiler to respond to a specified compile request
-        /// </summary>
-        /// <param name="projectId">Id of the project</param>
-        /// <param name="compileId">Id of the compilation of the project</param>
-        /// <param name="seconds">Seconds to allow for compile time</param>
-        /// <returns></returns>
-        private Compile WaitForCompilerResponse(int projectId, string compileId, int seconds)
-        {
-            var compile = new Compile();
-            var finish = DateTime.Now.AddSeconds(seconds);
-            while (DateTime.Now < finish)
-            {
-                Thread.Sleep(1000);
-                compile = ApiClient.ReadCompile(projectId, compileId);
-                if (compile.State == CompileState.BuildSuccess) break;
-            }
-            return compile;
-        }
-
         /// <summary>
         /// Wait to receive at least one order
         /// </summary>

Tests/Api/OptimizationTests.cs

@@ -220,7 +220,7 @@ namespace QuantConnect.Tests.API
             Assert.IsTrue(compile.Success);
 
             // Wait at max 30 seconds for project to compile
-            var compileCheck = WaitForCompilerResponse(projectId, compile.CompileId);
+            var compileCheck = WaitForCompilerResponse(ApiClient, projectId, compile.CompileId);
             Assert.IsTrue(compileCheck.Success);
             Assert.IsTrue(compileCheck.State == CompileState.BuildSuccess);
 

Tests/Api/ProjectTests.cs

@@ -257,7 +257,7 @@ namespace QuantConnect.Tests.API
             Assert.AreEqual(CompileState.InQueue, compileCreate.State);
 
             // Read out the compile
-            var compileSuccess = WaitForCompilerResponse(project.Projects.First().ProjectId, compileCreate.CompileId);
+            var compileSuccess = WaitForCompilerResponse(ApiClient, project.Projects.First().ProjectId, compileCreate.CompileId);
             Assert.IsTrue(compileSuccess.Success);
             Assert.AreEqual(CompileState.BuildSuccess, compileSuccess.State);
 
@@ -265,7 +265,7 @@ namespace QuantConnect.Tests.API
             file.Code += "[Jibberish at end of the file to cause a build error]";
             ApiClient.UpdateProjectFileContent(project.Projects.First().ProjectId, file.Name, file.Code);
             var compileError = ApiClient.CreateCompile(project.Projects.First().ProjectId);
-            compileError = WaitForCompilerResponse(project.Projects.First().ProjectId, compileError.CompileId);
+            compileError = WaitForCompilerResponse(ApiClient, project.Projects.First().ProjectId, compileError.CompileId);
             Assert.IsTrue(compileError.Success); // Successfully processed rest request.
             Assert.AreEqual(CompileState.BuildError, compileError.State); //Resulting in build fail.
 
@@ -336,7 +336,7 @@ namespace QuantConnect.Tests.API
                 $"Error updating project file:\n    {string.Join("\n    ", updateProjectFileContent.Errors)}");
 
             var compileCreate = ApiClient.CreateCompile(project.ProjectId);
-            var compileSuccess = WaitForCompilerResponse(project.ProjectId, compileCreate.CompileId);
+            var compileSuccess = WaitForCompilerResponse(ApiClient, project.ProjectId, compileCreate.CompileId);
             Assert.IsTrue(compileSuccess.Success, $"Error compiling project:\n    {string.Join("\n    ", compileSuccess.Errors)}");
 
             var backtestName = $"ReadBacktestOrders Backtest {GetTimestamp()}";
@@ -559,7 +559,7 @@ namespace QuantConnect.Tests.API
             Assert.IsTrue(compile.Success);
 
             // Wait at max 30 seconds for project to compile
-            var compileCheck = WaitForCompilerResponse(projectId, compile.CompileId);
+            var compileCheck = WaitForCompilerResponse(ApiClient, projectId, compile.CompileId);
             Assert.IsTrue(compileCheck.Success);
             Assert.IsTrue(compileCheck.State == CompileState.BuildSuccess);
 
@@ -642,7 +642,7 @@ namespace QuantConnect.Tests.API
             Assert.IsTrue(compile.Success);
 
             // Wait at max 30 seconds for project to compile
-            var compileCheck = WaitForCompilerResponse(projectId, compile.CompileId);
+            var compileCheck = WaitForCompilerResponse(ApiClient, projectId, compile.CompileId);
             Assert.IsTrue(compileCheck.Success);
             Assert.IsTrue(compileCheck.State == CompileState.BuildSuccess);
 
@@ -726,7 +726,7 @@ namespace QuantConnect.Tests.API
             compileId = compile.CompileId;
 
             // Wait at max 30 seconds for project to compile
-            var compileCheck = WaitForCompilerResponse(projectId, compile.CompileId);
+            var compileCheck = WaitForCompilerResponse(ApiClient, projectId, compile.CompileId);
             Assert.IsTrue(compileCheck.Success);
             Assert.IsTrue(compileCheck.State == CompileState.BuildSuccess);
         }

Tests/Brokerages/Paper/PaperBrokerageTests.cs

@@ -130,6 +130,7 @@ namespace QuantConnect.Tests.Brokerages.Paper
             var realTime = new BacktestingRealTimeHandler();
             using var nullLeanManager = new AlgorithmManagerTests.NullLeanManager();
 
+            using var tokenSource = new CancellationTokenSource();
             // run algorithm manager
             manager.Run(job,
                 algorithm,
@@ -138,7 +139,7 @@ namespace QuantConnect.Tests.Brokerages.Paper
                 results,
                 realTime,
                 nullLeanManager,
-                new CancellationToken()
+                tokenSource
             );
 
             var postDividendCash = algorithm.Portfolio.CashBook[Currencies.USD].Amount;

Tests/Common/Commands/CallbackCommandTests.cs

@@ -0,0 +1,129 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.IO;
+using NUnit.Framework;
+using Newtonsoft.Json;
+using QuantConnect.Statistics;
+using QuantConnect.Configuration;
+using System.Collections.Generic;
+using QuantConnect.Algorithm.CSharp;
+using QuantConnect.Lean.Engine.Server;
+using System;
+
+namespace QuantConnect.Tests.Common.Commands
+{
+    [TestFixture]
+    public class CallbackCommandTests
+    {
+        [TestCase(Language.CSharp)]
+        [TestCase(Language.Python)]
+        public void CommanCallback(Language language)
+        {
+            var parameter = new RegressionTests.AlgorithmStatisticsTestParameters(typeof(CallbackCommandRegressionAlgorithm).Name,
+                new Dictionary<string, string> {
+                    {PerformanceMetrics.TotalOrders, "3"},
+                    {"Average Win", "0%"},
+                    {"Average Loss", "0%"},
+                    {"Compounding Annual Return", "0.212%"},
+                    {"Drawdown", "0.000%"},
+                    {"Expectancy", "0"},
+                    {"Net Profit", "0.003%"},
+                    {"Sharpe Ratio", "-5.552"},
+                    {"Sortino Ratio", "0"},
+                    {"Probabilistic Sharpe Ratio", "66.765%"},
+                    {"Loss Rate", "0%"},
+                    {"Win Rate", "0%"},
+                    {"Profit-Loss Ratio", "0"},
+                    {"Alpha", "-0.01"},
+                    {"Beta", "0.003"},
+                    {"Annual Standard Deviation", "0.001"},
+                    {"Annual Variance", "0"},
+                    {"Information Ratio", "-8.919"},
+                    {"Tracking Error", "0.222"},
+                    {"Treynor Ratio", "-1.292"},
+                    {"Total Fees", "$3.00"},
+                    {"Estimated Strategy Capacity", "$670000000.00"},
+                    {"Lowest Capacity Asset", "IBM R735QTJ8XC9X"},
+                    {"Portfolio Turnover", "0.06%"}
+                },
+                language,
+                AlgorithmStatus.Completed);
+
+            Config.Set("lean-manager-type", typeof(TestLocalLeanManager).Name);
+
+            var result = AlgorithmRunner.RunLocalBacktest(parameter.Algorithm,
+                parameter.Statistics,
+                parameter.Language,
+                parameter.ExpectedFinalStatus);
+        }
+
+        internal class TestLocalLeanManager : LocalLeanManager
+        {
+            private bool _sentCommands;
+            public override void Update()
+            {
+                if (!_sentCommands && Algorithm.Time.TimeOfDay > TimeSpan.FromHours(9.50))
+                {
+                    _sentCommands = true;
+                    var commands = new List<Dictionary<string, object>>
+                    {
+                        new()
+                        {
+                            { "$type", "" },
+                            { "id", 1 },
+                            { "Symbol", "SPY" },
+                            { "Parameters", new Dictionary<string, decimal> { {  "quantity", 1 } } },
+                            { "unused", 99 }
+                        },
+                        new()
+                        {
+                            { "$type", "VoidCommand" },
+                            { "id", null },
+                            { "Quantity", 1 },
+                            { "targettime", Algorithm.Time },
+                            { "target", new [] { "BAC" } },
+                            { "Parameters", new Dictionary<string, string> { {  "tag", "a tag" }, { "something", "else" } } },
+                        },
+                        new()
+                        {
+                            { "id", "2" },
+                            { "$type", "BoolCommand" },
+                            { "Result", true },
+                            { "unused", new [] { 99 } }
+                        },
+                        new()
+                        {
+                            { "$type", "BoolCommand" },
+                            { "Result", null },
+                        }
+                    };
+
+                    for (var i = 1; i <= commands.Count; i++)
+                    {
+                        var command = commands[i - 1];
+                        command["id"] = i;
+                        File.WriteAllText($"command-{i}.json", JsonConvert.SerializeObject(command));
+                    }
+                    base.Update();
+                }
+            }
+            public override void OnAlgorithmStart()
+            {
+                SetCommandHandler();
+            }
+        }
+    }
+}

Tests/Common/Data/DataQueueHandlerSubscriptionManagerTests.cs

@@ -113,6 +113,33 @@ namespace QuantConnect.Tests.Common.Data
             Assert.IsFalse(subscriptionManager.IsSubscribed(Symbols.AAPL, TickType.Quote));
         }
 
+        [TestCase(TickType.Trade, MarketDataType.TradeBar, 1)]
+        [TestCase(TickType.Trade, MarketDataType.QuoteBar, 0)]
+        [TestCase(TickType.Quote, MarketDataType.QuoteBar, 1)]
+        [TestCase(TickType.OpenInterest, MarketDataType.Tick, 1)]
+        [TestCase(TickType.OpenInterest, MarketDataType.TradeBar, 0)]
+        public void GetSubscribeSymbolsBySpecificTickType(TickType tickType, MarketDataType dataType, int expectedCount)
+        {
+            using var fakeDataQueueHandler = new FakeDataQueuehandlerSubscriptionManager((tickType) => tickType!.ToString());
+
+            switch (dataType)
+            {
+                case MarketDataType.TradeBar:
+                    fakeDataQueueHandler.Subscribe(GetSubscriptionDataConfig<TradeBar>(Symbols.AAPL, Resolution.Minute));
+                    break;
+                case MarketDataType.QuoteBar:
+                    fakeDataQueueHandler.Subscribe(GetSubscriptionDataConfig<QuoteBar>(Symbols.AAPL, Resolution.Minute));
+                    break;
+                case MarketDataType.Tick:
+                    fakeDataQueueHandler.Subscribe(GetSubscriptionDataConfig<OpenInterest>(Symbols.AAPL, Resolution.Minute));
+                    break;
+            }
+
+            var subscribeSymbols = fakeDataQueueHandler.GetSubscribedSymbols(tickType).ToList();
+
+            Assert.That(subscribeSymbols.Count, Is.EqualTo(expectedCount));
+        }
+
         #region helper
 
         private SubscriptionDataConfig GetSubscriptionDataConfig(Type T, Symbol symbol, Resolution resolution, TickType? tickType = null)

Tests/Common/IsolatorTests.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -21,7 +21,7 @@ using QuantConnect.Util;
 
 namespace QuantConnect.Tests.Common
 {
-    [TestFixture]
+    [TestFixture, Parallelizable(ParallelScope.All)]
     public class IsolatorTests
     {
         [Test]
@@ -45,6 +45,38 @@ namespace QuantConnect.Tests.Common
             }
         }
 
+        [Test]
+        public void Cancellation()
+        {
+            var isolator = new Isolator();
+            var executed = false;
+            var ended = false;
+            var canceled = false;
+            var result = false;
+            isolator.CancellationTokenSource.CancelAfter(TimeSpan.FromMilliseconds(100));
+            try
+            {
+                result = isolator.ExecuteWithTimeLimit(
+                    TimeSpan.FromSeconds(5),
+                    () => {
+                        executed = true;
+                        Thread.Sleep(5000);
+                        ended = true;
+                    },
+                    5000,
+                    sleepIntervalMillis: 10
+                );
+            }
+            catch (OperationCanceledException)
+            {
+                canceled = true;
+            }
+            Assert.IsTrue(canceled);
+            Assert.IsFalse(result);
+            Assert.IsTrue(executed);
+            Assert.IsFalse(ended);
+        }
+
         [TestCase(Language.Python, true)]
         [TestCase(Language.Python, false)]
         [TestCase(Language.CSharp, true)]
@@ -98,4 +130,4 @@ namespace QuantConnect.Tests.Common
 
         }
     }
-}
+}

Tests/Engine/AlgorithmManagerTests.cs

@@ -120,7 +120,6 @@ namespace QuantConnect.Tests.Engine
             var results = new BacktestingResultHandler();
             var realtime = new BacktestingRealTimeHandler();
             using var leanManager = new NullLeanManager();
-            var token = new CancellationToken();
             var nullSynchronizer = new NullSynchronizer(algorithm);
 
             algorithm.Initialize();
@@ -136,7 +135,8 @@ namespace QuantConnect.Tests.Engine
 
             Log.Trace("Starting algorithm manager loop to process " + nullSynchronizer.Count + " time slices");
             var sw = Stopwatch.StartNew();
-            algorithmManager.Run(job, algorithm, nullSynchronizer, transactions, results, realtime, leanManager, token);
+            using var tokenSource = new CancellationTokenSource();
+            algorithmManager.Run(job, algorithm, nullSynchronizer, transactions, results, realtime, leanManager, tokenSource);
             sw.Stop();
 
             realtime.Exit();