Minor improvements

- Adding generic algorithm command support. Adding regression algorithms
- Allow PythonWrapper to validate classes too

Algorithm.CSharp/CallbackCommandRegressionAlgorithm.cs

@@ -0,0 +1,147 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using QuantConnect.Commands;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Regression algorithm asserting the behavior of different callback commands call
+    /// </summary>
+    public class CallbackCommandRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        /// <summary>
+        /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
+        /// </summary>
+        public override void Initialize()
+        {
+            SetStartDate(2013, 10, 07);
+            SetEndDate(2013, 10, 11);
+
+            AddEquity("SPY");
+            AddEquity("BAC");
+            AddEquity("IBM");
+            AddCommand<BoolCommand>();
+            AddCommand<VoidCommand>();
+        }
+
+        /// <summary>
+        /// Handle generic command callback
+        /// </summary>
+        public override bool? OnCommand(dynamic data)
+        {
+            Buy(data.Symbol, data.parameters["quantity"]);
+            return true;
+        }
+
+        private class VoidCommand : Command
+        {
+            public DateTime TargetTime { get; set; }
+            public string[] Target { get; set; }
+            public decimal Quantity { get; set; }
+            public Dictionary<string, string> Parameters { get; set; }
+            public override bool? Run(IAlgorithm algorithm)
+            {
+                if (TargetTime != algorithm.Time)
+                {
+                    return null;
+                }
+
+                ((QCAlgorithm)algorithm).Order(Target[0], Quantity, tag: Parameters["tag"]);
+                return null;
+            }
+        }
+        private class BoolCommand : Command
+        {
+            public bool? Result { get; set; }
+            public override bool? Run(IAlgorithm algorithm)
+            {
+                var shouldTrade = MyCustomMethod();
+                if (shouldTrade.HasValue && shouldTrade.Value)
+                {
+                    ((QCAlgorithm)algorithm).Buy("IBM", 1);
+                }
+                return shouldTrade;
+            }
+
+            private bool? MyCustomMethod()
+            {
+                return Result;
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public List<Language> Languages { get; } = new() { Language.CSharp, Language.Python };
+
+        /// <summary>
+        /// Data Points count of all timeslices of algorithm
+        /// </summary>
+        public long DataPoints => 3943;
+
+        /// <summary>
+        /// Data Points count of the algorithm history
+        /// </summary>
+        public int AlgorithmHistoryDataPoints => 0;
+
+        /// <summary>
+        /// Final status of the algorithm
+        /// </summary>
+        public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Orders", "1"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "271.453%"},
+            {"Drawdown", "2.200%"},
+            {"Expectancy", "0"},
+            {"Start Equity", "100000"},
+            {"End Equity", "101691.92"},
+            {"Net Profit", "1.692%"},
+            {"Sharpe Ratio", "8.854"},
+            {"Sortino Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "67.609%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "-0.005"},
+            {"Beta", "0.996"},
+            {"Annual Standard Deviation", "0.222"},
+            {"Annual Variance", "0.049"},
+            {"Information Ratio", "-14.565"},
+            {"Tracking Error", "0.001"},
+            {"Treynor Ratio", "1.97"},
+            {"Total Fees", "$3.44"},
+            {"Estimated Strategy Capacity", "$56000000.00"},
+            {"Lowest Capacity Asset", "SPY R735QTJ8XC9X"},
+            {"Portfolio Turnover", "19.93%"},
+            {"OrderListHash", "3da9fa60bf95b9ed148b95e02e0cfc9e"}
+        };
+    }
+}

Algorithm.Python/CallbackCommandRegressionAlgorithm.py

@@ -0,0 +1,73 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+class InvalidCommand():
+    variable = 10
+
+class VoidCommand():
+    quantity = 0
+    target = []
+    parameters = {}
+    targettime = None
+
+    def run(self, algo: QCAlgorithm) -> bool | None:
+        if not self.targettime or self.targettime != algo.time:
+            return
+        tag = self.parameters["tag"]
+        algo.order(self.target[0], self.get_quantity(), tag=tag)
+
+    def get_quantity(self):
+        return self.quantity
+
+class BoolCommand(Command):
+    result = False
+
+    def run(self, algo: QCAlgorithm) -> bool | None:
+        trade_ibm = self.my_custom_method()
+        if trade_ibm:
+            algo.buy("IBM", 1)
+        return trade_ibm
+
+    def my_custom_method(self):
+        return self.result
+
+### <summary>
+### Regression algorithm asserting the behavior of different callback commands call
+### </summary>
+class CallbackCommandRegressionAlgorithm(QCAlgorithm):
+    def initialize(self):
+        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
+
+        self.set_start_date(2013, 10, 7)
+        self.set_end_date(2013, 10, 11)
+
+        self.add_equity("SPY")
+        self.add_equity("IBM")
+        self.add_equity("BAC")
+
+        self.add_command(VoidCommand)
+        self.add_command(BoolCommand)
+
+        threw_exception = False
+        try:
+            self.add_command(InvalidCommand)
+        except:
+            threw_exception = True
+        if not threw_exception:
+            raise ValueError('InvalidCommand did not throw!')
+
+    def on_command(self, data):
+        self.buy(data.symbol, data.parameters["quantity"])
+        return True # False, None

Algorithm/QCAlgorithm.Python.cs

@@ -31,6 +31,7 @@ using QuantConnect.Scheduling;
 using QuantConnect.Util;
 using QuantConnect.Interfaces;
 using QuantConnect.Orders;
+using QuantConnect.Commands;
 
 namespace QuantConnect.Algorithm
 {
@@ -1620,6 +1621,23 @@ namespace QuantConnect.Algorithm
             return Liquidate(symbols.ConvertToSymbolEnumerable(), asynchronous, tag, orderProperties);
         }
 
+        /// <summary>
+        /// Register a command type to be used
+        /// </summary>
+        /// <param name="type">The command type</param>
+        public void AddCommand(PyObject type)
+        {
+            // create a test instance to validate interface is implemented accurate
+            var testInstance = new CommandPythonWrapper(type);
+
+            var wrappedType = Extensions.CreateType(type);
+            _registeredCommands[wrappedType.Name] = (CallbackCommand command) =>
+            {
+                var commandWrapper = new CommandPythonWrapper(type, command.Payload);
+                return commandWrapper.Run(this);
+            };
+        }
+
         /// <summary>
         /// Gets indicator base type
         /// </summary>

Algorithm/QCAlgorithm.cs

@@ -54,6 +54,8 @@ using QuantConnect.Securities.CryptoFuture;
 using QuantConnect.Algorithm.Framework.Alphas.Analysis;
 using QuantConnect.Algorithm.Framework.Portfolio.SignalExports;
 using Python.Runtime;
+using QuantConnect.Commands;
+using Newtonsoft.Json;
 
 namespace QuantConnect.Algorithm
 {
@@ -115,6 +117,9 @@ namespace QuantConnect.Algorithm
         private IStatisticsService _statisticsService;
         private IBrokerageModel _brokerageModel;
 
+        private readonly HashSet<string> _oneTimeCommandErrors = new();
+        private readonly Dictionary<string, Func<CallbackCommand, bool?>> _registeredCommands = new(StringComparer.InvariantCultureIgnoreCase);
+
         //Error tracking to avoid message flooding:
         private string _previousDebugMessage = "";
         private string _previousErrorMessage = "";
@@ -3385,6 +3390,62 @@ namespace QuantConnect.Algorithm
             return new DataHistory<OptionUniverse>(optionChain, new Lazy<PyObject>(() => PandasConverter.GetDataFrame(optionChain)));
         }
 
+        /// <summary>
+        /// Register a command type to be used
+        /// </summary>
+        /// <typeparam name="T">The command type</typeparam>
+        public void AddCommand<T>() where T : Command
+        {
+            _registeredCommands[typeof(T).Name] = (CallbackCommand command) =>
+            {
+                var commandInstance = JsonConvert.DeserializeObject<T>(command.Payload);
+                return commandInstance.Run(this);
+            };
+        }
+
+        /// <summary>
+        /// Run a callback command instance
+        /// </summary>
+        /// <param name="command">The callback command instance</param>
+        /// <returns>The command result</returns>
+        public CommandResultPacket RunCommand(CallbackCommand command)
+        {
+            bool? result = null;
+            if (_registeredCommands.TryGetValue(command.Type, out var target))
+            {
+                try
+                {
+                    result = target.Invoke(command);
+                }
+                catch (Exception ex)
+                {
+                    QuantConnect.Logging.Log.Error(ex);
+                    if (_oneTimeCommandErrors.Add(command.Type))
+                    {
+                        Log($"Unexpected error running command {command.Type}, {ex.Message}");
+                    }
+                }
+            }
+            else
+            {
+                if (_oneTimeCommandErrors.Add(command.Type))
+                {
+                    Log($"Detected unregistered command type {command.Type}, will be ignored");
+                }
+            }
+            return new CommandResultPacket(command, result) { CommandName = command.Type };
+        }
+
+        /// <summary>
+        /// Generic untyped command call handler
+        /// </summary>
+        /// <param name="data">The associated data</param>
+        /// <returns>True if success, false otherwise. Returning null will disable command feedback</returns>
+        public virtual bool? OnCommand(dynamic data)
+        {
+            return true;
+        }
+
         private static Symbol GetCanonicalOptionSymbol(Symbol symbol)
         {
             // We got the underlying

AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs

@@ -37,6 +37,7 @@ using QuantConnect.Storage;
 using QuantConnect.Statistics;
 using QuantConnect.Data.Market;
 using QuantConnect.Algorithm.Framework.Alphas.Analysis;
+using QuantConnect.Commands;
 
 namespace QuantConnect.AlgorithmFactory.Python.Wrappers
 {
@@ -62,6 +63,7 @@ namespace QuantConnect.AlgorithmFactory.Python.Wrappers
         private dynamic _onEndOfDay;
         private dynamic _onMarginCallWarning;
         private dynamic _onOrderEvent;
+        private dynamic _onCommand;
         private dynamic _onAssignmentOrderEvent;
         private dynamic _onSecuritiesChanged;
         private dynamic _onFrameworkSecuritiesChanged;
@@ -153,6 +155,7 @@ namespace QuantConnect.AlgorithmFactory.Python.Wrappers
                             _onDelistings = _algorithm.GetMethod("OnDelistings");
                             _onSymbolChangedEvents = _algorithm.GetMethod("OnSymbolChangedEvents");
                             _onEndOfDay = _algorithm.GetMethod("OnEndOfDay");
+                            _onCommand = _algorithm.GetMethod("OnCommand");
                             _onMarginCallWarning = _algorithm.GetMethod("OnMarginCallWarning");
                             _onOrderEvent = _algorithm.GetMethod("OnOrderEvent");
                             _onAssignmentOrderEvent = _algorithm.GetMethod("OnAssignmentOrderEvent");
@@ -921,6 +924,16 @@ namespace QuantConnect.AlgorithmFactory.Python.Wrappers
             _onOrderEvent(newEvent);
         }
 
+        /// <summary>
+        /// Generic untyped command call handler
+        /// </summary>
+        /// <param name="data">The associated data</param>
+        /// <returns>True if success, false otherwise. Returning null will disable command feedback</returns>
+        public bool? OnCommand(dynamic data)
+        {
+            return _onCommand(data);
+        }
+
         /// <summary>
         /// Will submit an order request to the algorithm
         /// </summary>
@@ -1242,5 +1255,13 @@ namespace QuantConnect.AlgorithmFactory.Python.Wrappers
         {
             _baseAlgorithm.SetTags(tags);
         }
+
+        /// <summary>
+        /// Run a callback command instance
+        /// </summary>
+        /// <param name="command">The callback command instance</param>
+        /// <returns>The command result</returns>
+        public CommandResultPacket RunCommand(CallbackCommand command) => _baseAlgorithm.RunCommand(command);
+
     }
 }

Api/Api.cs

@@ -1023,7 +1023,6 @@ namespace QuantConnect.Api
         /// </summary>
         /// <param name="projectId">Project for the live instance we want to stop</param>
         /// <returns><see cref="RestResponse"/></returns>
-
         public RestResponse StopLiveAlgorithm(int projectId)
         {
             var request = new RestRequest("live/update/stop", Method.POST)
@@ -1040,6 +1039,29 @@ namespace QuantConnect.Api
             return result;
         }
 
+        /// <summary>
+        /// Create a live command
+        /// </summary>
+        /// <param name="projectId">Project for the live instance we want to run the command against</param>
+        /// <param name="command">The serialized command</param>
+        /// <returns><see cref="RestResponse"/></returns>
+        public RestResponse CreateLiveCommand(int projectId, string command)
+        {
+            var request = new RestRequest("live/commands/create", Method.POST)
+            {
+                RequestFormat = DataFormat.Json
+            };
+
+            request.AddParameter("application/json", JsonConvert.SerializeObject(new
+            {
+                projectId,
+                command
+            }), ParameterType.RequestBody);
+
+            ApiConnection.TryRequest(request, out RestResponse result);
+            return result;
+        }
+
         /// <summary>
         /// Gets the logs of a specific live algorithm
         /// </summary>

Common/AlgorithmImports.py

@@ -39,6 +39,7 @@ from QuantConnect.Orders import *
 from QuantConnect.Python import *
 from QuantConnect.Storage import *
 from QuantConnect.Research import *
+from QuantConnect.Commands import *
 from QuantConnect.Algorithm import *
 from QuantConnect.Statistics import *
 from QuantConnect.Parameters import *

Common/Commands/BaseCommandHandler.cs

@@ -15,8 +15,10 @@
 
 using System;
 using System.Linq;
+using Newtonsoft.Json;
 using QuantConnect.Logging;
 using QuantConnect.Packets;
+using Newtonsoft.Json.Linq;
 using QuantConnect.Interfaces;
 using System.Collections.Generic;
 
@@ -27,6 +29,8 @@ namespace QuantConnect.Commands
     /// </summary>
     public abstract class BaseCommandHandler : ICommandHandler
     {
+        protected static readonly JsonSerializerSettings Settings = new() { TypeNameHandling = TypeNameHandling.All };
+
         /// <summary>
         /// The algorithm instance
         /// </summary>
@@ -104,5 +108,41 @@ namespace QuantConnect.Commands
         {
             // nop
         }
+
+        /// <summary>
+        /// Helper method to create a callback command
+        /// </summary>
+        protected ICommand TryGetCallbackCommand(string payload)
+        {
+            Dictionary<string, JToken> deserialized = new(StringComparer.InvariantCultureIgnoreCase);
+            try
+            {
+                if (!string.IsNullOrEmpty(payload))
+                {
+                    var jobject = JObject.Parse(payload);
+                    foreach (var kv in jobject)
+                    {
+                        deserialized[kv.Key] = kv.Value;
+                    }
+                }
+            }
+            catch (Exception err)
+            {
+                Log.Error(err, $"Payload: '{payload}'");
+                return null;
+            }
+
+            if (!deserialized.TryGetValue("id", out var id) || id == null)
+            {
+                id = string.Empty;
+            }
+
+            if (!deserialized.TryGetValue("$type", out var type) || type == null)
+            {
+                type = string.Empty;
+            }
+
+            return new CallbackCommand { Id = id.ToString(), Type = type.ToString(), Payload = payload };
+        }
     }
 }

Common/Commands/CallbackCommand.cs

@@ -0,0 +1,51 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using Newtonsoft.Json;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Commands
+{
+    /// <summary>
+    /// Algorithm callback command type
+    /// </summary>
+    public class CallbackCommand : BaseCommand
+    {
+        /// <summary>
+        /// The target command type to run, if empty or null will be the generic untyped command handler
+        /// </summary>
+        public string Type { get; set; }
+
+        /// <summary>
+        /// The command payload
+        /// </summary>
+        public string Payload { get; set; }
+
+        /// <summary>
+        /// Runs this command against the specified algorithm instance
+        /// </summary>
+        /// <param name="algorithm">The algorithm to run this command against</param>
+        public override CommandResultPacket Run(IAlgorithm algorithm)
+        {
+            if (string.IsNullOrEmpty(Type))
+            {
+                // target is the untyped algorithm handler
+                var result = algorithm.OnCommand(string.IsNullOrEmpty(Payload) ? null : JsonConvert.DeserializeObject<Command>(Payload));
+                return new CommandResultPacket(this, result);
+            }
+            return algorithm.RunCommand(this);
+        }
+    }
+}

Common/Commands/Command.cs

@@ -0,0 +1,91 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Dynamic;
+using QuantConnect.Data;
+using System.Reflection;
+using System.Linq.Expressions;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using Newtonsoft.Json.Linq;
+
+namespace QuantConnect.Commands
+{
+    /// <summary>
+    /// Base generic dynamic command class
+    /// </summary>
+    public class Command : DynamicObject
+    {
+        private static readonly MethodInfo SetPropertyMethodInfo = typeof(Command).GetMethod("SetProperty");
+        private static readonly MethodInfo GetPropertyMethodInfo = typeof(Command).GetMethod("GetProperty");
+
+        private readonly Dictionary<string, object> _storage = new(StringComparer.InvariantCultureIgnoreCase);
+
+        /// <summary>
+        /// Get the metaObject required for Dynamism.
+        /// </summary>
+        public sealed override DynamicMetaObject GetMetaObject(Expression parameter)
+        {
+            return new GetSetPropertyDynamicMetaObject(parameter, this, SetPropertyMethodInfo, GetPropertyMethodInfo);
+        }
+
+        /// <summary>
+        /// Sets the property with the specified name to the value. This is a case-insensitve search.
+        /// </summary>
+        /// <param name="name">The property name to set</param>
+        /// <param name="value">The new property value</param>
+        /// <returns>Returns the input value back to the caller</returns>
+        public object SetProperty(string name, object value)
+        {
+            if (value is JArray jArray)
+            {
+                return _storage[name] = jArray.ToObject<List<object>>();
+            }
+            else if (value is JObject jobject)
+            {
+                return _storage[name] = jobject.ToObject<Dictionary<string, object>>();
+            }
+            else
+            {
+                return _storage[name] = value;
+            }
+        }
+
+        /// <summary>
+        /// Gets the property's value with the specified name. This is a case-insensitve search.
+        /// </summary>
+        /// <param name="name">The property name to access</param>
+        /// <returns>object value of BaseData</returns>
+        public object GetProperty(string name)
+        {
+            if (!_storage.TryGetValue(name, out var value))
+            {
+                throw new KeyNotFoundException($"Property with name \'{name}\' does not exist. Properties: {string.Join(", ", _storage.Keys)}");
+            }
+            return value;
+        }
+
+        /// <summary>
+        /// Run this command using the target algorithm
+        /// </summary>
+        /// <param name="algorithm">The algorithm instance</param>
+        /// <returns>True if success, false otherwise. Returning null will disable command feedback</returns>
+        public virtual bool? Run(IAlgorithm algorithm)
+        {
+            throw new NotImplementedException($"Please implement the 'def run(algorithm) -> bool | None:' method");
+        }
+    }
+}

Common/Commands/CommandResultsPacket.cs

@@ -31,12 +31,12 @@ namespace QuantConnect.Commands
         /// <summary>
         /// Gets or sets whether or not the
         /// </summary>
-        public bool Success { get; set; }
+        public bool? Success { get; set; }
 
         /// <summary>
         /// Initializes a new instance of the <see cref="CommandResultPacket"/> class
         /// </summary>
-        public CommandResultPacket(ICommand command, bool success)
+        public CommandResultPacket(ICommand command, bool? success)
             : base(PacketType.CommandResult)
         {
             Success = success;

Common/Commands/FileCommandHandler.cs

@@ -19,6 +19,7 @@ using Newtonsoft.Json;
 using QuantConnect.Logging;
 using System.Collections.Generic;
 using System.Linq;
+using Newtonsoft.Json.Linq;
 
 namespace QuantConnect.Commands
 {
@@ -90,7 +91,9 @@ namespace QuantConnect.Commands
         private void ReadCommandFile(string commandFilePath)
         {
             Log.Trace($"FileCommandHandler.ReadCommandFile(): {Messages.FileCommandHandler.ReadingCommandFile(commandFilePath)}");
-            object deserialized;
+            string contents = null;
+            Exception exception = null;
+            object deserialized = null;
             try
             {
                 if (!File.Exists(commandFilePath))
@@ -98,13 +101,12 @@ namespace QuantConnect.Commands
                     Log.Error($"FileCommandHandler.ReadCommandFile(): {Messages.FileCommandHandler.CommandFileDoesNotExist(commandFilePath)}");
                     return;
                 }
-                var contents = File.ReadAllText(commandFilePath);
-                deserialized = JsonConvert.DeserializeObject(contents, new JsonSerializerSettings { TypeNameHandling = TypeNameHandling.All });
+                contents = File.ReadAllText(commandFilePath);
+                deserialized = JsonConvert.DeserializeObject(contents, Settings);
             }
             catch (Exception err)
             {
-                Log.Error(err);
-                deserialized = null;
+                exception = err;
             }
 
             // remove the file when we're done reading it
@@ -126,6 +128,20 @@ namespace QuantConnect.Commands
             if (item != null)
             {
                 _commands.Enqueue(item);
+                return;
+            }
+
+            var callbackCommand = TryGetCallbackCommand(contents);
+            if (callbackCommand != null)
+            {
+                _commands.Enqueue(callbackCommand);
+                return;
+            }
+
+            if (exception != null)
+            {
+                // if we are here we failed
+                Log.Error(exception);
             }
         }
     }

Common/Data/DataQueueHandlerSubscriptionManager.cs

@@ -102,6 +102,21 @@ namespace QuantConnect.Data
                 .Distinct();
         }
 
+        /// <summary>
+        /// Retrieves the list of unique <see cref="Symbol"/> instances that are currently subscribed for a specific <see cref="TickType"/>.
+        /// </summary>
+        /// <param name="tickType">The type of tick data to filter subscriptions by.</param>
+        /// <returns>A collection of unique <see cref="Symbol"/> objects that match the specified <paramref name="tickType"/>.</returns>
+        public IEnumerable<Symbol> GetSubscribedSymbols(TickType tickType)
+        {
+            var channelName = ChannelNameFromTickType(tickType);
+#pragma warning disable CA1309
+            return SubscribersByChannel.Keys.Where(x => x.Name.Equals(channelName, StringComparison.InvariantCultureIgnoreCase))
+#pragma warning restore CA1309
+                .Select(c => c.Symbol)
+                .Distinct();
+        }
+
         /// <summary>
         /// Checks if there is existing subscriber for current channel
         /// </summary>

Common/Data/Market/Tick.cs

@@ -231,6 +231,21 @@ namespace QuantConnect.Data.Market
             AskPrice = ask;
         }
 
+        /// <summary>
+        /// Initializes a new instance of the <see cref="Tick"/> class to <see cref="TickType.OpenInterest"/>.
+        /// </summary>
+        /// <param name="time">The time at which the open interest tick occurred.</param>
+        /// <param name="symbol">The symbol associated with the open interest tick.</param>
+        /// <param name="openInterest">The value of the open interest for the specified symbol.</param>
+        public Tick(DateTime time, Symbol symbol, decimal openInterest)
+        {
+            Time = time;
+            Symbol = symbol;
+            Value = openInterest;
+            DataType = MarketDataType.Tick;
+            TickType = TickType.OpenInterest;
+        }
+
         /// <summary>
         /// Initializer for a last-trade equity tick with bid or ask prices.
         /// </summary>

Common/Interfaces/IAlgorithm.cs

@@ -32,6 +32,7 @@ using QuantConnect.Securities.Option;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Algorithm.Framework.Alphas;
 using QuantConnect.Algorithm.Framework.Alphas.Analysis;
+using QuantConnect.Commands;
 
 namespace QuantConnect.Interfaces
 {
@@ -608,6 +609,13 @@ namespace QuantConnect.Interfaces
         /// <param name="newEvent">Event information</param>
         void OnOrderEvent(OrderEvent newEvent);
 
+        /// <summary>
+        /// Generic untyped command call handler
+        /// </summary>
+        /// <param name="data">The associated data</param>
+        /// <returns>True if success, false otherwise. Returning null will disable command feedback</returns>
+        bool? OnCommand(dynamic data);
+
         /// <summary>
         /// Will submit an order request to the algorithm
         /// </summary>
@@ -919,5 +927,12 @@ namespace QuantConnect.Interfaces
         /// </summary>
         /// <param name="tags">The tags</param>
         void SetTags(HashSet<string> tags);
+
+        /// <summary>
+        /// Run a callback command instance
+        /// </summary>
+        /// <param name="command">The callback command instance</param>
+        /// <returns>The command result</returns>
+        CommandResultPacket RunCommand(CallbackCommand command);
     }
 }

Common/Python/CommandPythonWrapper.cs

@@ -0,0 +1,74 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using Python.Runtime;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using QuantConnect.Commands;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+
+namespace QuantConnect.Python
+{
+    /// <summary>
+    /// Python wrapper for a python defined command type
+    /// </summary>
+    public class CommandPythonWrapper : BasePythonWrapper<Command>
+    {
+        /// <summary>
+        /// Constructor for initialising the <see cref="CommandPythonWrapper"/> class with wrapped <see cref="PyObject"/> object
+        /// </summary>
+        /// <param name="type">Python command type</param>
+        /// <param name="data">Command data</param>
+        public CommandPythonWrapper(PyObject type, string data = null)
+            : base()
+        {
+            using var _ = Py.GIL();
+
+            var instance = type.Invoke();
+
+            SetPythonInstance(instance);
+            if (data != null)
+            {
+                foreach (var kvp in JsonConvert.DeserializeObject<Dictionary<string, object>>(data))
+                {
+                    if (kvp.Value is JArray jArray)
+                    {
+                        SetProperty(kvp.Key, jArray.ToObject<List<object>>());
+                    }
+                    else if (kvp.Value is JObject jobject)
+                    {
+                        SetProperty(kvp.Key, jobject.ToObject<Dictionary<string, object>>());
+                    }
+                    else
+                    {
+                        SetProperty(kvp.Key, kvp.Value);
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Run this command using the target algorithm
+        /// </summary>
+        /// <param name="algorithm">The algorithm instance</param>
+        /// <returns>True if success, false otherwise. Returning null will disable command feedback</returns>
+        public bool? Run(IAlgorithm algorithm)
+        {
+            var result = InvokeMethod(nameof(Run), algorithm);
+            return result.GetAndDispose<bool?>();
+        }
+    }
+}

Common/Python/PythonWrapper.cs

@@ -34,21 +34,28 @@ namespace QuantConnect.Python
         /// <param name="model">The model implementing the interface type</param>
         public static PyObject ValidateImplementationOf<TInterface>(this PyObject model)
         {
-            if (!typeof(TInterface).IsInterface)
-            {
-                throw new ArgumentException(
-                    $"{nameof(PythonWrapper)}.{nameof(ValidateImplementationOf)}(): {Messages.PythonWrapper.ExpectedInterfaceTypeParameter}");
-            }
-
+            var notInterface = !typeof(TInterface).IsInterface;
             var missingMembers = new List<string>();
             var members = typeof(TInterface).GetMembers(BindingFlags.Public | BindingFlags.Instance);
             using (Py.GIL())
             {
                 foreach (var member in members)
                 {
-                    if ((member is not MethodInfo method || !method.IsSpecialName) &&
+                    var method = member as MethodInfo;
+                    if ((method == null || !method.IsSpecialName) &&
                         !model.HasAttr(member.Name) && !model.HasAttr(member.Name.ToSnakeCase()))
                     {
+                        if (notInterface)
+                        {
+                            if (method != null && !method.IsAbstract && (method.IsFinal || !method.IsVirtual || method.DeclaringType != typeof(TInterface)))
+                            {
+                                continue;
+                            }
+                            else if (member is ConstructorInfo)
+                            {
+                                continue;
+                            }
+                        }
                         missingMembers.Add(member.Name);
                     }
                 }

Engine/Server/LocalLeanManager.cs

@@ -91,8 +91,7 @@ namespace QuantConnect.Lean.Engine.Server
         {
             if (Algorithm.LiveMode)
             {
-                _commandHandler = new FileCommandHandler();
-                _commandHandler.Initialize(_job, Algorithm);
+                SetCommandHandler();
             }
         }
 
@@ -119,5 +118,14 @@ namespace QuantConnect.Lean.Engine.Server
         {
             _commandHandler.DisposeSafely();
         }
+
+        /// <summary>
+        /// Set the command handler to use, protected for testing purposes
+        /// </summary>
+        protected virtual void SetCommandHandler()
+        {
+            _commandHandler = new FileCommandHandler();
+            _commandHandler.Initialize(_job, Algorithm);
+        }
     }
 }

Tests/Common/Commands/CallbackCommandTests.cs

@@ -0,0 +1,129 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.IO;
+using NUnit.Framework;
+using Newtonsoft.Json;
+using QuantConnect.Statistics;
+using QuantConnect.Configuration;
+using System.Collections.Generic;
+using QuantConnect.Algorithm.CSharp;
+using QuantConnect.Lean.Engine.Server;
+using System;
+
+namespace QuantConnect.Tests.Common.Commands
+{
+    [TestFixture]
+    public class CallbackCommandTests
+    {
+        [TestCase(Language.CSharp)]
+        [TestCase(Language.Python)]
+        public void CommanCallback(Language language)
+        {
+            var parameter = new RegressionTests.AlgorithmStatisticsTestParameters(typeof(CallbackCommandRegressionAlgorithm).Name,
+                new Dictionary<string, string> {
+                    {PerformanceMetrics.TotalOrders, "3"},
+                    {"Average Win", "0%"},
+                    {"Average Loss", "0%"},
+                    {"Compounding Annual Return", "0.212%"},
+                    {"Drawdown", "0.000%"},
+                    {"Expectancy", "0"},
+                    {"Net Profit", "0.003%"},
+                    {"Sharpe Ratio", "-5.552"},
+                    {"Sortino Ratio", "0"},
+                    {"Probabilistic Sharpe Ratio", "66.765%"},
+                    {"Loss Rate", "0%"},
+                    {"Win Rate", "0%"},
+                    {"Profit-Loss Ratio", "0"},
+                    {"Alpha", "-0.01"},
+                    {"Beta", "0.003"},
+                    {"Annual Standard Deviation", "0.001"},
+                    {"Annual Variance", "0"},
+                    {"Information Ratio", "-8.919"},
+                    {"Tracking Error", "0.222"},
+                    {"Treynor Ratio", "-1.292"},
+                    {"Total Fees", "$3.00"},
+                    {"Estimated Strategy Capacity", "$670000000.00"},
+                    {"Lowest Capacity Asset", "IBM R735QTJ8XC9X"},
+                    {"Portfolio Turnover", "0.06%"}
+                },
+                language,
+                AlgorithmStatus.Completed);
+
+            Config.Set("lean-manager-type", typeof(TestLocalLeanManager).Name);
+
+            var result = AlgorithmRunner.RunLocalBacktest(parameter.Algorithm,
+                parameter.Statistics,
+                parameter.Language,
+                parameter.ExpectedFinalStatus);
+        }
+
+        internal class TestLocalLeanManager : LocalLeanManager
+        {
+            private bool _sentCommands;
+            public override void Update()
+            {
+                if (!_sentCommands && Algorithm.Time.TimeOfDay > TimeSpan.FromHours(9.50))
+                {
+                    _sentCommands = true;
+                    var commands = new List<Dictionary<string, object>>
+                    {
+                        new()
+                        {
+                            { "$type", "" },
+                            { "id", 1 },
+                            { "Symbol", "SPY" },
+                            { "Parameters", new Dictionary<string, decimal> { {  "quantity", 1 } } },
+                            { "unused", 99 }
+                        },
+                        new()
+                        {
+                            { "$type", "VoidCommand" },
+                            { "id", null },
+                            { "Quantity", 1 },
+                            { "targettime", Algorithm.Time },
+                            { "target", new [] { "BAC" } },
+                            { "Parameters", new Dictionary<string, string> { {  "tag", "a tag" }, { "something", "else" } } },
+                        },
+                        new()
+                        {
+                            { "id", "2" },
+                            { "$type", "BoolCommand" },
+                            { "Result", true },
+                            { "unused", new [] { 99 } }
+                        },
+                        new()
+                        {
+                            { "$type", "BoolCommand" },
+                            { "Result", null },
+                        }
+                    };
+
+                    for (var i = 1; i <= commands.Count; i++)
+                    {
+                        var command = commands[i - 1];
+                        command["id"] = i;
+                        File.WriteAllText($"command-{i}.json", JsonConvert.SerializeObject(command));
+                    }
+                    base.Update();
+                }
+            }
+            public override void OnAlgorithmStart()
+            {
+                SetCommandHandler();
+            }
+        }
+    }
+}

Tests/Common/Data/DataQueueHandlerSubscriptionManagerTests.cs

@@ -113,6 +113,33 @@ namespace QuantConnect.Tests.Common.Data
             Assert.IsFalse(subscriptionManager.IsSubscribed(Symbols.AAPL, TickType.Quote));
         }
 
+        [TestCase(TickType.Trade, MarketDataType.TradeBar, 1)]
+        [TestCase(TickType.Trade, MarketDataType.QuoteBar, 0)]
+        [TestCase(TickType.Quote, MarketDataType.QuoteBar, 1)]
+        [TestCase(TickType.OpenInterest, MarketDataType.Tick, 1)]
+        [TestCase(TickType.OpenInterest, MarketDataType.TradeBar, 0)]
+        public void GetSubscribeSymbolsBySpecificTickType(TickType tickType, MarketDataType dataType, int expectedCount)
+        {
+            using var fakeDataQueueHandler = new FakeDataQueuehandlerSubscriptionManager((tickType) => tickType!.ToString());
+
+            switch (dataType)
+            {
+                case MarketDataType.TradeBar:
+                    fakeDataQueueHandler.Subscribe(GetSubscriptionDataConfig<TradeBar>(Symbols.AAPL, Resolution.Minute));
+                    break;
+                case MarketDataType.QuoteBar:
+                    fakeDataQueueHandler.Subscribe(GetSubscriptionDataConfig<QuoteBar>(Symbols.AAPL, Resolution.Minute));
+                    break;
+                case MarketDataType.Tick:
+                    fakeDataQueueHandler.Subscribe(GetSubscriptionDataConfig<OpenInterest>(Symbols.AAPL, Resolution.Minute));
+                    break;
+            }
+
+            var subscribeSymbols = fakeDataQueueHandler.GetSubscribedSymbols(tickType).ToList();
+
+            Assert.That(subscribeSymbols.Count, Is.EqualTo(expectedCount));
+        }
+
         #region helper
 
         private SubscriptionDataConfig GetSubscriptionDataConfig(Type T, Symbol symbol, Resolution resolution, TickType? tickType = null)