Add new CustomWeight, AlphaModel for AlphaStreams

- Add new CustomWeight PCM for alpha streams
- Add new AlphaStreams AlphaModule that will handle security additions
  and removals, removing this logic from AlphaStreamsBasicTemplateAlgo

Algorithm.CSharp/AlphaStreamsBasicTemplateAlgorithm.cs

@@ -13,15 +13,13 @@
  * limitations under the License.
 */
 
-using System.Linq;
-using QuantConnect.Data;
 using QuantConnect.Orders;
 using QuantConnect.Interfaces;
 using QuantConnect.Brokerages;
 using QuantConnect.Securities;
 using System.Collections.Generic;
-using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Data.Custom.AlphaStreams;
+using QuantConnect.Algorithm.Framework.Alphas;
 using QuantConnect.Algorithm.Framework.Execution;
 using QuantConnect.Algorithm.Framework.Portfolio;
 
@@ -32,8 +30,6 @@ namespace QuantConnect.Algorithm.CSharp
     /// </summary>
     public class AlphaStreamsBasicTemplateAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
-        private Dictionary<Symbol, HashSet<Symbol>> _symbolsPerAlpha = new Dictionary<Symbol, HashSet<Symbol>>();
-
         /// <summary>
         /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
         /// </summary>
@@ -42,10 +38,10 @@ namespace QuantConnect.Algorithm.CSharp
             SetStartDate(2018, 04, 04);
             SetEndDate(2018, 04, 06);
 
+            SetAlpha(new AlphaStreamAlphaModule());
             SetExecution(new ImmediateExecutionModel());
             Settings.MinimumOrderMarginPortfolioPercentage = 0.01m;
             SetPortfolioConstruction(new EqualWeightingAlphaStreamsPortfolioConstructionModel());
-
             SetSecurityInitializer(new BrokerageModelSecurityInitializer(new DefaultBrokerageModel(),
                 new FuncSecuritySeeder(GetLastKnownPrices)));
 
@@ -55,77 +51,11 @@ namespace QuantConnect.Algorithm.CSharp
             }
         }
 
-        /// <summary>
-        /// OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
-        /// </summary>
-        /// <param name="data">Slice object keyed by symbol containing the stock data</param>
-        public override void OnData(Slice data)
-        {
-            foreach (var portfolioState in data.Get<AlphaStreamsPortfolioState>().Values)
-            {
-                ProcessPortfolioState(portfolioState);
-            }
-        }
-
         public override void OnOrderEvent(OrderEvent orderEvent)
         {
             Log($"OnOrderEvent: {orderEvent}");
         }
 
-        public override void OnSecuritiesChanged(SecurityChanges changes)
-        {
-            changes.FilterCustomSecurities = false;
-            foreach (var addedSecurity in changes.AddedSecurities)
-            {
-                if (addedSecurity.Symbol.IsCustomDataType<AlphaStreamsPortfolioState>())
-                {
-                    if (!_symbolsPerAlpha.ContainsKey(addedSecurity.Symbol))
-                    {
-                        _symbolsPerAlpha[addedSecurity.Symbol] = new HashSet<Symbol>();
-                    }
-                    // warmup alpha state, adding target securities
-                    ProcessPortfolioState(addedSecurity.Cache.GetData<AlphaStreamsPortfolioState>());
-                }
-            }
-
-            Log($"OnSecuritiesChanged: {changes}");
-        }
-
-        private bool UsedBySomeAlpha(Symbol asset)
-        {
-            return _symbolsPerAlpha.Any(pair => pair.Value.Contains(asset));
-        }
-
-        private void ProcessPortfolioState(AlphaStreamsPortfolioState portfolioState)
-        {
-            if (portfolioState == null)
-            {
-                return;
-            }
-
-            var alphaId = portfolioState.Symbol;
-            if (!_symbolsPerAlpha.TryGetValue(alphaId, out var currentSymbols))
-            {
-                _symbolsPerAlpha[alphaId] = currentSymbols = new HashSet<Symbol>();
-            }
-
-            var newSymbols = new HashSet<Symbol>(currentSymbols.Count);
-            foreach (var symbol in portfolioState.PositionGroups?.SelectMany(positionGroup => positionGroup.Positions).Select(state => state.Symbol) ?? Enumerable.Empty<Symbol>())
-            {
-                // only add it if it's not used by any alpha (already added check)
-                if (newSymbols.Add(symbol) && !UsedBySomeAlpha(symbol))
-                {
-                    AddSecurity(symbol, resolution: UniverseSettings.Resolution, extendedMarketHours: UniverseSettings.ExtendedMarketHours);
-                }
-            }
-            _symbolsPerAlpha[alphaId] = newSymbols;
-
-            foreach (var symbol in currentSymbols.Where(symbol => !UsedBySomeAlpha(symbol)))
-            {
-                RemoveSecurity(symbol);
-            }
-        }
-
         /// <summary>
         /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
         /// </summary>

Algorithm.CSharp/AlphaStreamsUniverseSelectionTemplateAlgorithm.cs

@@ -14,9 +14,11 @@
 */
 
 using System;
+using QuantConnect.Interfaces;
 using System.Collections.Generic;
 using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Data.Custom.AlphaStreams;
+using QuantConnect.Algorithm.Framework.Alphas;
 using QuantConnect.Algorithm.Framework.Execution;
 using QuantConnect.Algorithm.Framework.Portfolio;
 using QuantConnect.Algorithm.Framework.Selection;
@@ -26,7 +28,7 @@ namespace QuantConnect.Algorithm.CSharp
     /// <summary>
     /// Example algorithm consuming an alpha streams portfolio state and trading based on it
     /// </summary>
-    public class AlphaStreamsUniverseSelectionTemplateAlgorithm : AlphaStreamsBasicTemplateAlgorithm
+    public class AlphaStreamsUniverseSelectionTemplateAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
         /// <summary>
         /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
@@ -36,6 +38,7 @@ namespace QuantConnect.Algorithm.CSharp
             SetStartDate(2018, 04, 04);
             SetEndDate(2018, 04, 06);
 
+            SetAlpha(new AlphaStreamAlphaModule());
             SetExecution(new ImmediateExecutionModel());
             Settings.MinimumOrderMarginPortfolioPercentage = 0.01m;
             SetPortfolioConstruction(new EqualWeightingAlphaStreamsPortfolioConstructionModel());
@@ -65,23 +68,63 @@ namespace QuantConnect.Algorithm.CSharp
             }
         }
 
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public Language[] Languages { get; } = { Language.CSharp };
 
         /// <summary>
         /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
         /// </summary>
-        public override Dictionary<string, string> ExpectedStatistics
+        public virtual Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
-            get
-            {
-                var result = base.ExpectedStatistics;
-                result["Compounding Annual Return"] = "-13.200%";
-                result["Information Ratio"] = "2.827";
-                result["Tracking Error"] = "0.248";
-                result["Fitness Score"] = "0.011";
-                result["Return Over Maximum Drawdown"] = "-113.513";
-                result["Portfolio Turnover"] = "0.023";
-                return result;
-            }
-        }
+            {"Total Trades", "2"},
+            {"Average Win", "0%"},
+            {"Average Loss", "-0.12%"},
+            {"Compounding Annual Return", "-13.200%"},
+            {"Drawdown", "0.200%"},
+            {"Expectancy", "-1"},
+            {"Net Profit", "-0.116%"},
+            {"Sharpe Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "0%"},
+            {"Loss Rate", "100%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "2.827"},
+            {"Tracking Error", "0.248"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "$0.00"},
+            {"Estimated Strategy Capacity", "$83000.00"},
+            {"Lowest Capacity Asset", "BTCUSD XJ"},
+            {"Fitness Score", "0.011"},
+            {"Kelly Criterion Estimate", "0"},
+            {"Kelly Criterion Probability Value", "0"},
+            {"Sortino Ratio", "79228162514264337593543950335"},
+            {"Return Over Maximum Drawdown", "-113.513"},
+            {"Portfolio Turnover", "0.023"},
+            {"Total Insights Generated", "0"},
+            {"Total Insights Closed", "0"},
+            {"Total Insights Analysis Completed", "0"},
+            {"Long Insight Count", "0"},
+            {"Short Insight Count", "0"},
+            {"Long/Short Ratio", "100%"},
+            {"Estimated Monthly Alpha Value", "$0"},
+            {"Total Accumulated Estimated Alpha Value", "$0"},
+            {"Mean Population Estimated Insight Value", "$0"},
+            {"Mean Population Direction", "0%"},
+            {"Mean Population Magnitude", "0%"},
+            {"Rolling Averaged Population Direction", "0%"},
+            {"Rolling Averaged Population Magnitude", "0%"},
+            {"OrderListHash", "2b94bc50a74caebe06c075cdab1bc6da"}
+        };
     }
 }

Algorithm.Framework/Alphas/AlphaStreamAlphaModule.cs

@@ -0,0 +1,120 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.Linq;
+using QuantConnect.Data;
+using System.Collections.Generic;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Data.Custom.AlphaStreams;
+
+namespace QuantConnect.Algorithm.Framework.Alphas
+{
+    /// <summary>
+    /// Alpha model that will handle adding and removing securities from the algorithm based on the current portfolio of the different alphas
+    /// </summary>
+    public sealed class AlphaStreamAlphaModule : AlphaModel
+    {
+        private Dictionary<Symbol, HashSet<Symbol>> _symbolsPerAlpha = new Dictionary<Symbol, HashSet<Symbol>>();
+
+        /// <summary>
+        /// Initialize new <see cref="AlphaStreamAlphaModule"/>
+        /// </summary>
+        public AlphaStreamAlphaModule(string name = null)
+        {
+            Name = name ?? "AlphaStreamAlphaModule";
+        }
+
+        /// <summary>
+        /// Updates this alpha model with the latest data from the algorithm.
+        /// This is called each time the algorithm receives data for subscribed securities
+        /// </summary>
+        /// <param name="algorithm">The algorithm instance</param>
+        /// <param name="data">The new data available</param>
+        /// <returns>The new insights generated</returns>
+        public override IEnumerable<Insight> Update(QCAlgorithm algorithm, Slice data)
+        {
+            foreach (var portfolioState in data.Get<AlphaStreamsPortfolioState>().Values)
+            {
+                ProcessPortfolioState(algorithm, portfolioState);
+            }
+
+            return Enumerable.Empty<Insight>();
+        }
+
+        /// <summary>
+        /// Event fired each time the we add/remove securities from the data feed
+        /// </summary>
+        /// <param name="algorithm">The algorithm instance that experienced the change in securities</param>
+        /// <param name="changes">The security additions and removals from the algorithm</param>
+        public override void OnSecuritiesChanged(QCAlgorithm algorithm, SecurityChanges changes)
+        {
+            changes.FilterCustomSecurities = false;
+            foreach (var addedSecurity in changes.AddedSecurities)
+            {
+                if (addedSecurity.Symbol.IsCustomDataType<AlphaStreamsPortfolioState>())
+                {
+                    if (!_symbolsPerAlpha.ContainsKey(addedSecurity.Symbol))
+                    {
+                        _symbolsPerAlpha[addedSecurity.Symbol] = new HashSet<Symbol>();
+                    }
+                    // warmup alpha state, adding target securities
+                    ProcessPortfolioState(algorithm, addedSecurity.Cache.GetData<AlphaStreamsPortfolioState>());
+                }
+            }
+
+            algorithm.Log($"OnSecuritiesChanged: {changes}");
+        }
+
+        /// <summary>
+        /// Will handle adding and removing securities from the algorithm based on the current portfolio of the different alphas
+        /// </summary>
+        private void ProcessPortfolioState(QCAlgorithm algorithm, AlphaStreamsPortfolioState portfolioState)
+        {
+            if (portfolioState == null)
+            {
+                return;
+            }
+
+            var alphaId = portfolioState.Symbol;
+            if (!_symbolsPerAlpha.TryGetValue(alphaId, out var currentSymbols))
+            {
+                _symbolsPerAlpha[alphaId] = currentSymbols = new HashSet<Symbol>();
+            }
+
+            var newSymbols = new HashSet<Symbol>(currentSymbols.Count);
+            foreach (var symbol in portfolioState.PositionGroups?.SelectMany(positionGroup => positionGroup.Positions).Select(state => state.Symbol) ?? Enumerable.Empty<Symbol>())
+            {
+                // only add it if it's not used by any alpha (already added check)
+                if (newSymbols.Add(symbol) && !UsedBySomeAlpha(symbol))
+                {
+                    algorithm.AddSecurity(symbol,
+                        resolution: algorithm.UniverseSettings.Resolution,
+                        extendedMarketHours: algorithm.UniverseSettings.ExtendedMarketHours);
+                }
+            }
+            _symbolsPerAlpha[alphaId] = newSymbols;
+
+            foreach (var symbol in currentSymbols.Where(symbol => !UsedBySomeAlpha(symbol)))
+            {
+                algorithm.RemoveSecurity(symbol);
+            }
+        }
+
+        private bool UsedBySomeAlpha(Symbol asset)
+        {
+            return _symbolsPerAlpha.Any(pair => pair.Value.Contains(asset));
+        }
+    }
+}

Algorithm.Framework/CustomWeightingAlphaStreamsPortfolioConstructionModel.cs

@@ -0,0 +1,51 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.Linq;
+using QuantConnect.Logging;
+using System.Collections.Generic;
+using QuantConnect.Algorithm.Framework.Portfolio;
+
+namespace QuantConnect.Algorithm.Framework
+{
+    /// <summary>
+    /// Custom weighting alpha streams portfolio construction model that will generate aggregated security targets taking into account all the alphas positions
+    /// and a custom weighting factor for each alpha, which is also factored by the relation of the alphas portfolio value and the current algorithms portfolio value
+    /// </summary>
+    public class CustomWeightingAlphaStreamsPortfolioConstructionModel : EqualWeightingAlphaStreamsPortfolioConstructionModel
+    {
+        private Dictionary<string, decimal> _alphaWeights;
+
+        /// <summary>
+        /// Specify a custom set of alpha portfolio weights to use
+        /// </summary>
+        /// <param name="alphaWeights">The alpha portfolio weights</param>
+        public void SetAlphaWeights(Dictionary<string, decimal> alphaWeights)
+        {
+            Log.Trace($"CustomWeightingAlphaStreamsPortfolioConstructionModel.SetAlphaWeights(): new weights: [{string.Join(",", alphaWeights.Select(pair => $"{pair.Key}:{pair.Value}"))}]");
+            _alphaWeights = alphaWeights;
+        }
+
+        /// <summary>
+        /// Get's the weight for an alpha
+        /// </summary>
+        /// <param name="alphaId">The algorithm instance that experienced the change in securities</param>
+        /// <returns>The alphas weight</returns>
+        public override decimal GetAlphaWeight(string alphaId)
+        {
+            return !_alphaWeights.TryGetValue(alphaId, out var alphaWeight) ? 0 : alphaWeight;
+        }
+    }
+}

Algorithm.Framework/Portfolio/AlphaStreamsPortfolioConstructionModel.cs

@@ -0,0 +1,58 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System.Collections.Generic;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Algorithm.Framework.Alphas;
+
+namespace QuantConnect.Algorithm.Framework.Portfolio
+{
+    /// <summary>
+    /// Base alpha streams portfolio construction model
+    /// </summary>
+    public class AlphaStreamsPortfolioConstructionModel : IPortfolioConstructionModel
+    {
+        /// <summary>
+        /// Get's the weight for an alpha
+        /// </summary>
+        /// <param name="alphaId">The algorithm instance that experienced the change in securities</param>
+        /// <returns>The alphas weight</returns>
+        public virtual decimal GetAlphaWeight(string alphaId)
+        {
+            throw new System.NotImplementedException();
+        }
+
+        /// <summary>
+        /// Event fired each time the we add/remove securities from the data feed
+        /// </summary>
+        /// <param name="algorithm">The algorithm instance that experienced the change in securities</param>
+        /// <param name="changes">The security additions and removals from the algorithm</param>
+        public virtual void OnSecuritiesChanged(QCAlgorithm algorithm, SecurityChanges changes)
+        {
+            throw new System.NotImplementedException();
+        }
+
+        /// <summary>
+        /// Create portfolio targets from the specified insights
+        /// </summary>
+        /// <param name="algorithm">The algorithm instance</param>
+        /// <param name="insights">The insights to create portfolio targets from</param>
+        /// <returns>An enumerable of portfolio targets to be sent to the execution model</returns>
+        public virtual IEnumerable<IPortfolioTarget> CreateTargets(QCAlgorithm algorithm, Insight[] insights)
+        {
+            throw new System.NotImplementedException();
+        }
+    }
+}

Algorithm.Framework/Portfolio/EqualWeightingAlphaStreamsPortfolioConstructionModel.cs

@@ -29,7 +29,7 @@ namespace QuantConnect.Algorithm.Framework.Portfolio
     /// and an equal weighting factor for each alpha, which is also factored by the relation of the alphas portfolio value and the current algorithms portfolio value,
     /// overriding <see cref="GetAlphaWeight"/> allows custom weighting implementations
     /// </summary>
-    public class EqualWeightingAlphaStreamsPortfolioConstructionModel : IPortfolioConstructionModel
+    public class EqualWeightingAlphaStreamsPortfolioConstructionModel : AlphaStreamsPortfolioConstructionModel
     {
         private bool _rebalance;
         private Dictionary<Symbol, PortfolioTarget> _targetsPerSymbol;
@@ -48,7 +48,7 @@ namespace QuantConnect.Algorithm.Framework.Portfolio
         /// <param name="algorithm">The algorithm instance</param>
         /// <param name="insights">The insights to create portfolio targets from</param>
         /// <returns>An enumerable of portfolio targets to be sent to the execution model</returns>
-        public IEnumerable<IPortfolioTarget> CreateTargets(QCAlgorithm algorithm, Insight[] insights)
+        public override IEnumerable<IPortfolioTarget> CreateTargets(QCAlgorithm algorithm, Insight[] insights)
         {
             if (_targetsPerSymbol == null)
             {
@@ -104,12 +104,22 @@ namespace QuantConnect.Algorithm.Framework.Portfolio
             }
         }
 
+        /// <summary>
+        /// Get's the weight for an alpha
+        /// </summary>
+        /// <param name="alphaId">The algorithm instance that experienced the change in securities</param>
+        /// <returns>The alphas weight</returns>
+        public override decimal GetAlphaWeight(string alphaId)
+        {
+            return 1m / _targetsPerSymbolPerAlpha.Count;
+        }
+
         /// <summary>
         /// Event fired each time the we add/remove securities from the data feed
         /// </summary>
         /// <param name="algorithm">The algorithm instance that experienced the change in securities</param>
         /// <param name="changes">The security additions and removals from the algorithm</param>
-        public void OnSecuritiesChanged(QCAlgorithm algorithm, SecurityChanges changes)
+        public override void OnSecuritiesChanged(QCAlgorithm algorithm, SecurityChanges changes)
         {
             changes.FilterCustomSecurities = false;
 
@@ -153,10 +163,10 @@ namespace QuantConnect.Algorithm.Framework.Portfolio
         /// Determines the portfolio weight to give a specific alpha. Default implementation just returns equal weighting
         /// </summary>
         /// <param name="portfolioState">The alphas portfolio state to get the weight for</param>
-        /// <param name="totalUsablePortfolioValue">This algorithms usable total portfolio value</param>
+        /// <param name="totalUsablePortfolioValue">This algorithms usable total portfolio value, removing the free portfolio value</param>
         /// <param name="cashBook">This algorithms cash book</param>
         /// <returns>The weight to use on this alphas positions</returns>
-        protected virtual decimal GetAlphaWeight(AlphaStreamsPortfolioState portfolioState,
+        private decimal GetAlphaWeight(AlphaStreamsPortfolioState portfolioState,
             decimal totalUsablePortfolioValue,
             CashBook cashBook)
         {
@@ -168,8 +178,7 @@ namespace QuantConnect.Algorithm.Framework.Portfolio
                 return 0;
             }
 
-            var equalWeightFactor = 1m / _targetsPerSymbolPerAlpha.Count;
-            return totalUsablePortfolioValue * equalWeightFactor / alphaPortfolioValueInOurAccountCurrency;
+            return totalUsablePortfolioValue * GetAlphaWeight(portfolioState.AlphaId) / alphaPortfolioValueInOurAccountCurrency;
         }
 
         private bool ProcessPortfolioState(AlphaStreamsPortfolioState portfolioState, QCAlgorithm algorithm)