Round AverageLoss to Avoid Very Small Numbers (#6003)

* Refactor error message for reproducability

* WIP Refactor GetMaximumOrderQuantityForTargetBuyingPower

* Additional tweaks and tests

* Address shorted margin case

* Reinstate tests with new modified function

* Update regression

* Some cleanup

* Expand test set

* Refactor solution

* Address review

* Final adjustments and cleanup

* Expand error message for GetAmountToOrder for safety and reproducibility

* Address reviews

Algorithm.CSharp/FractionalQuantityRegressionAlgorithm.cs

@@ -139,7 +139,7 @@ namespace QuantConnect.Algorithm.CSharp
             {"Mean Population Magnitude", "0%"},
             {"Rolling Averaged Population Direction", "0%"},
             {"Rolling Averaged Population Magnitude", "0%"},
-            {"OrderListHash", "2b3ac55337ce5619fc0388ccdac72c54"}
+            {"OrderListHash", "604291218c630343a896bfa2f3104932"}
         };
     }
 }

Common/Securities/BuyingPowerModel.cs

@@ -14,6 +14,7 @@
 */
 
 using System;
+using System.Diagnostics.CodeAnalysis;
 using QuantConnect.Orders;
 using QuantConnect.Orders.Fees;
 using static QuantConnect.StringExtensions;
@@ -377,74 +378,56 @@ namespace QuantConnect.Securities
 
             // we use initial margin requirement here to avoid the duplicate PortfolioTarget.Percent situation:
             // PortfolioTarget.Percent(1) -> fills -> PortfolioTarget.Percent(1) _could_ detect free buying power if we use Maintenance requirement here
-            var currentSignedUsedMargin = this.GetInitialMarginRequirement(parameters.Security, parameters.Security.Holdings.Quantity);
-
-            // remove directionality, we'll work in the land of absolutes
-            var absFinalOrderMargin = Math.Abs(signedTargetFinalMarginValue - currentSignedUsedMargin);
-            var direction = signedTargetFinalMarginValue > currentSignedUsedMargin ? OrderDirection.Buy : OrderDirection.Sell;
+            var signedCurrentUsedMargin = this.GetInitialMarginRequirement(parameters.Security, parameters.Security.Holdings.Quantity);
 
             // determine the unit price in terms of the account currency
             var utcTime = parameters.Security.LocalTime.ConvertToUtc(parameters.Security.Exchange.TimeZone);
-            // determine the margin required for 1 unit, positive since we are working with absolutes
+
+            // determine the margin required for 1 unit
             var absUnitMargin = this.GetInitialMarginRequirement(parameters.Security, 1);
             if (absUnitMargin == 0)
             {
                 return new GetMaximumOrderQuantityResult(0, parameters.Security.Symbol.GetZeroPriceMessage());
             }
 
-            // compute the initial order quantity
-            var absOrderQuantity = Math.Abs(GetAmountToOrder(currentSignedUsedMargin, signedTargetFinalMarginValue, absUnitMargin,
-                parameters.Security.SymbolProperties.LotSize));
-            if (absOrderQuantity == 0)
-            {
-                string reason = null;
-                if (!parameters.SilenceNonErrorReasons)
-                {
-                    reason = $"The order quantity is less than the lot size of {parameters.Security.SymbolProperties.LotSize} " +
-                             "and has been rounded to zero.";
-                }
-                return new GetMaximumOrderQuantityResult(0, reason, false);
-            }
-
+            // Check that the change of margin is above our models minimum percentage change
+            var absDifferenceOfMargin = Math.Abs(signedTargetFinalMarginValue - signedCurrentUsedMargin);
             if (!BuyingPowerModelExtensions.AboveMinimumOrderMarginPortfolioPercentage(parameters.Portfolio,
-                parameters.MinimumOrderMarginPortfolioPercentage, absFinalOrderMargin))
+                parameters.MinimumOrderMarginPortfolioPercentage, absDifferenceOfMargin))
             {
-                var minimumValue = totalPortfolioValue * parameters.MinimumOrderMarginPortfolioPercentage;
                 string reason = null;
                 if (!parameters.SilenceNonErrorReasons)
                 {
-                    reason = $"The target order margin {absFinalOrderMargin} is less than the minimum {minimumValue}.";
+                    var minimumValue = totalPortfolioValue * parameters.MinimumOrderMarginPortfolioPercentage;
+                    reason = $"The target order margin {absDifferenceOfMargin} is less than the minimum {minimumValue}.";
                 }
                 return new GetMaximumOrderQuantityResult(0, reason, false);
             }
 
             // Use the following loop to converge on a value that places us under our target allocation when adjusted for fees
             var lastOrderQuantity = 0m;     // For safety check
+            decimal orderFees = 0m;
+            decimal signedTargetHoldingsMargin;
+            decimal orderQuantity;
 
-            var signedTargetHoldingsMargin = ((direction == OrderDirection.Sell ? -1 : 1) * absOrderQuantity + parameters.Security.Holdings.Quantity) * absUnitMargin;
-            decimal orderFees = 0;
             do
             {
-                // If our order target holdings is larger than our target margin allocated we need to recalculate our order size
-                if (Math.Abs(signedTargetHoldingsMargin) > Math.Abs(signedTargetFinalMarginValue))
+                // Calculate our order quantity
+                orderQuantity = GetAmountToOrder(parameters.Security, signedTargetFinalMarginValue, absUnitMargin, out signedTargetHoldingsMargin);
+                if (orderQuantity == 0)
                 {
-                    absOrderQuantity = Math.Abs(GetAmountToOrder(currentSignedUsedMargin, signedTargetFinalMarginValue, absUnitMargin,
-                        parameters.Security.SymbolProperties.LotSize, absOrderQuantity * (direction == OrderDirection.Sell ? -1 : 1)));
-                }
+                    string reason = null;
+                    if (!parameters.SilenceNonErrorReasons)
+                    {
+                        reason = Invariant($"The order quantity is less than the lot size of {parameters.Security.SymbolProperties.LotSize} ") +
+                            Invariant($"and has been rounded to zero. Target order margin {signedTargetFinalMarginValue - signedCurrentUsedMargin}. ");
+                    }
 
-                if (absOrderQuantity <= 0)
-                {
-                    var sign = direction == OrderDirection.Buy ? 1 : -1;
-                    return new GetMaximumOrderQuantityResult(0,
-                        Invariant($"The order quantity is less than the lot size of {parameters.Security.SymbolProperties.LotSize} ") +
-                        Invariant($"and has been rounded to zero.Target order margin {absFinalOrderMargin * sign}. Order fees ") +
-                        Invariant($"{orderFees}. Order quantity {absOrderQuantity * sign}. Margin unit {absUnitMargin}."),
-                        false
-                    );
+                    return new GetMaximumOrderQuantityResult(0, reason, false);
                 }
 
                 // generate the order
-                var order = new MarketOrder(parameters.Security.Symbol, absOrderQuantity, utcTime);
+                var order = new MarketOrder(parameters.Security.Symbol, orderQuantity, utcTime);
                 var fees = parameters.Security.FeeModel.GetOrderFee(
                     new OrderFeeParameters(parameters.Security,
                         order)).Value;
@@ -452,96 +435,112 @@ namespace QuantConnect.Securities
 
                 // Update our target portfolio margin allocated when considering fees, then calculate the new FinalOrderMargin
                 signedTargetFinalMarginValue = (totalPortfolioValue - orderFees - totalPortfolioValue * RequiredFreeBuyingPowerPercent) * parameters.TargetBuyingPower;
-                absFinalOrderMargin = Math.Abs(signedTargetFinalMarginValue - currentSignedUsedMargin);
 
-                // Start safe check after first loop
-                if (lastOrderQuantity == absOrderQuantity)
+                // Start safe check after first loop, stops endless recursion
+                if (lastOrderQuantity == orderQuantity)
                 {
-                    var sign = direction == OrderDirection.Buy ? 1 : -1;
                     var message =
                         Invariant($"GetMaximumOrderQuantityForTargetBuyingPower failed to converge on the target margin: {signedTargetFinalMarginValue}; ") +
-                        Invariant($"the following information can be used to reproduce the issue. Total Portfolio Cash: {parameters.Portfolio.Cash}; ") +
-                        Invariant($"Leverage: {parameters.Security.Leverage}; Order Fee: {orderFees}; Lot Size: {parameters.Security.SymbolProperties.LotSize}; ") +
-                        Invariant($"Per Unit Margin: {absUnitMargin}; Current Holdings: {parameters.Security.Holdings}; Target Percentage: %{parameters.TargetBuyingPower * 100}; ") +
-                        Invariant($"Current Order Target Margin: {absFinalOrderMargin * sign}; Current Order Margin: {absOrderQuantity * absUnitMargin * sign}");
+                        Invariant($"the following information can be used to reproduce the issue. Total Portfolio Cash: {parameters.Portfolio.Cash}; Security : {parameters.Security.Symbol.ID}; ") +
+                        Invariant($"Price : {parameters.Security.Close}; Leverage: {parameters.Security.Leverage}; Order Fee: {orderFees}; Lot Size: {parameters.Security.SymbolProperties.LotSize}; ") +
+                        Invariant($"Current Holdings: {parameters.Security.Holdings.Quantity} @ {parameters.Security.Holdings.AveragePrice}; Target Percentage: %{parameters.TargetBuyingPower * 100};");
+
+                    // Need to add underlying value to message to reproduce with options
+                    if (parameters.Security is Option.Option option && option.Underlying != null)
+                    {
+                        var underlying = option.Underlying;
+                        message += Invariant($" Underlying Security: {underlying.Symbol.ID}; Underlying Price: {underlying.Close}; Underlying Holdings: {underlying.Holdings.Quantity} @ {underlying.Holdings.AveragePrice};");
+                    }
+
                     throw new ArgumentException(message);
                 }
-                lastOrderQuantity = absOrderQuantity;
-                
-                // Update our target holdings margin
-                signedTargetHoldingsMargin = ((direction == OrderDirection.Sell ? -1 : 1) * absOrderQuantity + parameters.Security.Holdings.Quantity) * absUnitMargin;
+                lastOrderQuantity = orderQuantity;
+
             }
             // Ensure that our target holdings margin will be less than or equal to our target allocated margin
             while (Math.Abs(signedTargetHoldingsMargin) > Math.Abs(signedTargetFinalMarginValue));
 
             // add directionality back in
-            return new GetMaximumOrderQuantityResult((direction == OrderDirection.Sell ? -1 : 1) * absOrderQuantity);
+            return new GetMaximumOrderQuantityResult(orderQuantity);
         }
 
         /// <summary>
         /// Helper function that determines the amount to order to get to a given target safely.
         /// Meaning it will either be at or just below target always.
         /// </summary>
-        /// <param name="currentMargin">Current margin</param>
-        /// <param name="targetMargin">Target margin</param>
-        /// <param name="perUnitMargin">Margin required for each unit</param>
-        /// <param name="lotSize">Lot size of the security we are ordering</param>
+        /// <param name="security">Security we are to determine order size for</param>
+        /// <param name="targetMargin">Target margin allocated</param>
+        /// <param name="marginForOneUnit">Margin requirement for one unit; used in our initial order guess</param>
+        /// <param name="finalMargin">Output the final margin allocated to this security</param>
         /// <returns>The size of the order to get safely to our target</returns>
-        public static decimal GetAmountToOrder(decimal currentMargin, decimal targetMargin, decimal perUnitMargin, decimal lotSize, decimal? currentOrderSize = null)
+        public decimal GetAmountToOrder([NotNull]Security security, decimal targetMargin, decimal marginForOneUnit, out decimal finalMargin)
         {
-            // Determine the amount to order to put us at our target
-            var orderSize = (targetMargin - currentMargin) / perUnitMargin;
+            var lotSize = security.SymbolProperties.LotSize;
 
-            // Determine if we are under our target
-            var underTarget = false;
-            // For negative target, we are under if target is a larger negative number
-            if (targetMargin < 0 && targetMargin - currentMargin < 0)
-            {
-                underTarget = true;
-            }
-            // For positive target, we are under if target is a larger positive number
-            else if (targetMargin > 0 && targetMargin - currentMargin > 0)
-            {
-                underTarget = true;
-            }
+            // Start with order size that puts us back to 0, in theory this means current margin is 0
+            // so we can calculate holdings to get to the new target margin directly. This is very helpful for
+            // odd cases where margin requirements aren't linear.
+            var orderSize = -security.Holdings.Quantity;
 
-            // Determine our rounding mode
-            MidpointRounding roundingMode;
-            if (underTarget)
-            {
-                // Negative orders need to be rounded "up" so we don't go over target
-                // Positive orders need to be rounded "down" so we don't go over target
-                roundingMode = orderSize < 0
-                    ? MidpointRounding.ToPositiveInfinity
-                    : MidpointRounding.ToNegativeInfinity;
-            }
-            else
-            {
-                // Negative orders need to be rounded "down" so we are under our target
-                // Positive orders need to be rounded "up" so we are under our target
-                roundingMode = orderSize < 0
-                    ? MidpointRounding.ToNegativeInfinity
-                    : MidpointRounding.ToPositiveInfinity;
-            }
+            // Use the margin for one unit to make our initial guess.
+            orderSize += targetMargin / marginForOneUnit;
 
-            // For handling precision errors in OrderSize calculation
-            if (currentOrderSize.HasValue && orderSize % lotSize == 0 && orderSize == currentOrderSize.Value)
-            {
-                // Force an adjustment
-                if (roundingMode == MidpointRounding.ToPositiveInfinity)
-                {
-                    orderSize += lotSize;
-                }
-                else
-                {
-                    orderSize -= lotSize;
-                }
-
-                return orderSize;
-            }
+            // Determine the rounding mode for this order size
+            var roundingMode = targetMargin < 0
+                // Ending in short position; orders need to be rounded towards positive so we end up under our target
+                ? MidpointRounding.ToPositiveInfinity
+                // Ending in long position; orders need to be rounded towards negative so we end up under our target
+                : MidpointRounding.ToNegativeInfinity;
 
             // Round this order size appropriately
-            return orderSize.DiscretelyRoundBy(lotSize, roundingMode);
+            orderSize = orderSize.DiscretelyRoundBy(lotSize, roundingMode);
+
+            // Use our model to calculate this final margin as a final check
+            finalMargin = this.GetInitialMarginRequirement(security,
+                    orderSize + security.Holdings.Quantity);
+
+            // Until our absolute final margin is equal to or below target we need to adjust; ensures we don't overshoot target
+            // This isn't usually the case, but for non-linear margin per unit cases this may be necessary.
+            // For example https://www.quantconnect.com/forum/discussion/12470, (covered in OptionMarginBuyingPowerModelTests)
+            var marginDifference = finalMargin - targetMargin;
+            while ((targetMargin < 0 && marginDifference < 0) || (targetMargin > 0 && marginDifference > 0))
+            {
+                // TODO: Can this be smarter about its adjustment, instead of just stepping by lotsize?
+                // We adjust according to the target margin being a short or long
+                orderSize += targetMargin < 0 ? lotSize : -lotSize;
+
+                // Recalculate final margin with this adjusted orderSize
+                finalMargin = this.GetInitialMarginRequirement(security,
+                    orderSize + security.Holdings.Quantity);
+
+                // Safety check, does not occur in any of our testing, but to be sure we don't enter a endless loop
+                // have this guy check that the difference between the two is not growing.
+                var newDifference = finalMargin - targetMargin;
+                if (Math.Abs(newDifference) > Math.Abs(marginDifference) && Math.Sign(newDifference) == Math.Sign(marginDifference))
+                {
+                    // We have a problem and are correcting in the wrong direction
+                    var errorMessage =
+                        "BuyingPowerModel().GetAmountToOrder(): Margin is being adjusted in the wrong direction." +
+                        $"Reproduce this issue with the following variables, Target Margin: {targetMargin}; MarginForOneUnit: {marginForOneUnit};" +
+                        $"Security Holdings: {security.Holdings.Quantity} @ {security.Holdings.AveragePrice};" +
+                        $"LotSize: {security.SymbolProperties.LotSize}; Price: {security.Close}; Leverage: {security.Leverage}";
+
+                    // Need to add underlying value to message to reproduce with options
+                    if (security is Option.Option option && option.Underlying != null)
+                    {
+                        var underlying = option.Underlying;
+                        errorMessage +=
+                            $" Underlying Security: {underlying.Symbol.ID}; Underlying Price: {underlying.Close};" +
+                            $" Underlying Holdings: {underlying.Holdings.Quantity} @ {underlying.Holdings.AveragePrice};";
+                    }
+
+                    throw new ArgumentException(errorMessage);
+                }
+
+                marginDifference = newDifference;
+            }
+
+            return orderSize;
         }
 
         /// <summary>

Common/Securities/Index/IndexSymbol.cs

@@ -25,10 +25,10 @@ namespace QuantConnect.Securities.Index
         private static readonly Dictionary<string, string> _indexMarket = new Dictionary<string, string>
         {
             { "SPX", Market.CBOE },
-            { "NDX", Market.CBOE },
+            { "NDX", "NASDAQ" },
             { "VIX", Market.CBOE },
             { "SPXW", Market.CBOE },
-            { "NQX", Market.CBOE },
+            { "NQX", "NASDAQ" },
             { "VIXW", Market.CBOE }
         };
 

Indicators/Identity.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -19,7 +19,7 @@ namespace QuantConnect.Indicators
     ///     Represents an indicator that is a ready after ingesting a single sample and
     ///     always returns the same value as it is given.
     /// </summary>
-    public class Identity : Indicator
+    public class Identity : Indicator, IIndicatorWarmUpPeriodProvider
     {
         /// <summary>
         ///     Initializes a new instance of the Identity indicator with the specified name
@@ -38,6 +38,11 @@ namespace QuantConnect.Indicators
             get { return Samples > 0; }
         }
 
+        /// <summary>
+        /// Required period, in data points, for the indicator to be ready and fully initialized
+        /// </summary>
+        public int WarmUpPeriod => 1;
+
         /// <summary>
         ///     Computes the next value of this indicator from the given state
         /// </summary>
@@ -48,4 +53,4 @@ namespace QuantConnect.Indicators
             return input.Value;
         }
     }
-}
+}

Indicators/PivotPointsHighLow.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -26,8 +26,8 @@ namespace QuantConnect.Indicators
     /// </summary>
     public class PivotPointsHighLow : IndicatorBase<IBaseDataBar>, IIndicatorWarmUpPeriodProvider
     {
-        private readonly int _lengthHigh;
-        private readonly int _lengthLow;
+        private readonly int _surroundingBarsCountForHighPoint;
+        private readonly int _surroundingBarsCountForLowPoint;
         private readonly RollingWindow<IBaseDataBar> _windowHighs;
         private readonly RollingWindow<IBaseDataBar> _windowLows;
         // Stores information of that last N pivot points
@@ -46,40 +46,42 @@ namespace QuantConnect.Indicators
         /// <summary>
         /// Required period, in data points, for the indicator to be ready and fully initialized.
         /// </summary>
-        public int WarmUpPeriod { get; }
+        public int WarmUpPeriod { get; protected set; }
 
         /// <summary>
         /// Creates a new instance of <see cref="PivotPointsHighLow"/> indicator with an equal high and low length
         /// </summary>
-        /// <param name="length">The length parameter here defines the number of surround bars that we compare against the current bar high and lows for the max/min </param>
+        /// <param name="surroundingBarsCount">The length parameter here defines the number of surrounding bars that we compare against the current bar high and lows for the max/min </param>
         /// <param name="lastStoredValues">The number of last stored indicator values</param>
-        public PivotPointsHighLow(int length, int lastStoredValues = 100)
-            : this($"PivotPointsHighLow({length})", length, length, lastStoredValues)
+        public PivotPointsHighLow(int surroundingBarsCount, int lastStoredValues = 100)
+            : this($"PivotPointsHighLow({surroundingBarsCount})", surroundingBarsCount, surroundingBarsCount, lastStoredValues)
         { }
 
         /// <summary>
         /// Creates a new instance of <see cref="PivotPointsHighLow"/> indicator
         /// </summary>
-        /// <param name="lengthHigh">The number of surrounding bars whose high values should be less than the current bar's for the bar high to be marked as high pivot point</param>
-        /// <param name="lengthLow">The number of surrounding bars whose low values should be more than the current bar's for the bar low to be marked as low pivot point</param>
+        /// <param name="surroundingBarsCountForHighPoint">The number of surrounding bars whose high values should be less than the current bar's for the bar high to be marked as high pivot point</param>
+        /// <param name="surroundingBarsCountForLowPoint">The number of surrounding bars whose low values should be more than the current bar's for the bar low to be marked as low pivot point</param>
         /// <param name="lastStoredValues">The number of last stored indicator values</param>
-        public PivotPointsHighLow(int lengthHigh, int lengthLow, int lastStoredValues = 100)
-            : this($"PivotPointsHighLow({lengthHigh},{lengthLow})", lengthHigh, lengthLow, lastStoredValues)
+        public PivotPointsHighLow(int surroundingBarsCountForHighPoint, int surroundingBarsCountForLowPoint, int lastStoredValues = 100)
+            : this($"PivotPointsHighLow({surroundingBarsCountForHighPoint},{surroundingBarsCountForLowPoint})", surroundingBarsCountForHighPoint, surroundingBarsCountForLowPoint, lastStoredValues)
         { }
 
+
         /// <summary>
         /// Creates a new instance of <see cref="PivotPointsHighLow"/> indicator
         /// </summary>
         /// <param name="name">The name of an indicator</param>
-        /// <param name="lengthHigh">The number of surrounding bars whose high values should be less than the current bar's for the bar high to be marked as high pivot point</param>
-        /// <param name="lengthLow">The number of surrounding bars whose low values should be more than the current bar's for the bar low to be marked as low pivot point</param>
+        /// <param name="surroundingBarsCountForHighPoint">The number of surrounding bars whose high values should be less than the current bar's for the bar high to be marked as high pivot point</param>
+        /// <param name="surroundingBarsCountForLowPoint">The number of surrounding bars whose low values should be more than the current bar's for the bar low to be marked as low pivot point</param>
         /// <param name="lastStoredValues">The number of last stored indicator values</param>
-        public PivotPointsHighLow(string name, int lengthHigh, int lengthLow, int lastStoredValues = 100) : base(name)
+        public PivotPointsHighLow(string name, int surroundingBarsCountForHighPoint, int surroundingBarsCountForLowPoint, int lastStoredValues = 100)
+            : base(name)
         {
-            _lengthHigh = lengthHigh;
-            _lengthLow = lengthLow;
-            _windowHighs = new RollingWindow<IBaseDataBar>(2 * lengthHigh + 1);
-            _windowLows = new RollingWindow<IBaseDataBar>(2 * _lengthLow + 1);
+            _surroundingBarsCountForHighPoint = surroundingBarsCountForHighPoint;
+            _surroundingBarsCountForLowPoint = surroundingBarsCountForLowPoint;
+            _windowHighs = new RollingWindow<IBaseDataBar>(2 * surroundingBarsCountForHighPoint + 1);
+            _windowLows = new RollingWindow<IBaseDataBar>(2 * _surroundingBarsCountForLowPoint + 1);
             _windowPivotPoints = new RollingWindow<PivotPoint>(lastStoredValues);
             WarmUpPeriod = Math.Max(_windowHighs.Size, _windowLows.Size);
         }
@@ -94,65 +96,104 @@ namespace QuantConnect.Indicators
             _windowHighs.Add(input);
             _windowLows.Add(input);
 
-            PivotPoint high = null, low = null;
+            PivotPoint highPoint = null, lowPoint = null;
 
             if (_windowHighs.IsReady)
             {
-                var isHigh = true;
-                var middlePoint = _windowHighs[_lengthHigh];
-                for (var k = 0; k < _windowHighs.Size && isHigh; k++)
-                {
-                    // Skip the middle point
-                    if (k == _lengthHigh)
-                    {
-                        continue;
-                    }
-
-                    // Check if current high is below middle point high
-                    isHigh = _windowHighs[k].High < middlePoint.High;
-                }
-
-                if (isHigh)
-                {
-                    high = new PivotPoint(PivotPointType.High, middlePoint.High, middlePoint.EndTime);
-                }
+                highPoint = FindNextHighPivotPoint(_windowHighs, _surroundingBarsCountForHighPoint);
             }
 
             if (_windowLows.IsReady)
             {
-                var isLow = true;
-                var middlePoint = _windowLows[_lengthLow];
-                for (var k = 0; k < _windowLows.Size && isLow; k++)
-                {
-                    if (k == _lengthLow)
-                    {
-                        continue;
-                    }
-
-                    isLow = _windowLows[k].Low > middlePoint.Low;
-                }
-
-                if (isLow)
-                {
-                    low = new PivotPoint(PivotPointType.Low, middlePoint.Low, middlePoint.EndTime);
-                }
+                lowPoint = FindNextLowPivotPoint(_windowLows, _surroundingBarsCountForLowPoint);
             }
 
-            if (high != null)
+            OnNewPivotPointFormed(highPoint);
+            OnNewPivotPointFormed(lowPoint);
+
+            return ConvertToComputedValue(highPoint, lowPoint);
+        }
+
+        /// <summary>
+        /// Looks for the next low pivot point.
+        /// </summary>
+        /// <param name="windowLows">rolling window that tracks the lows</param>
+        /// <param name="midPointIndexOrSurroundingBarsCount">The midpoint index or surrounding bars count for lows</param>
+        /// <returns>pivot point if found else null</returns>
+        protected virtual PivotPoint FindNextLowPivotPoint(RollingWindow<IBaseDataBar> windowLows, int midPointIndexOrSurroundingBarsCount)
+        {
+            var isLow = true;
+            var middlePoint = windowLows[midPointIndexOrSurroundingBarsCount];
+            for (var k = 0; k < windowLows.Size && isLow; k++)
             {
-                OnNewPivotPointFormed(high);
-                if (low != null)
+                if (k == midPointIndexOrSurroundingBarsCount)
+                {
+                    continue;
+                }
+
+                isLow = windowLows[k].Low > middlePoint.Low;
+            }
+
+            PivotPoint low = null;
+            if (isLow)
+            {
+                low = new PivotPoint(PivotPointType.Low, middlePoint.Low, middlePoint.EndTime);
+            }
+
+            return low;
+        }
+
+        /// <summary>
+        /// Looks for the next high pivot point.
+        /// </summary>
+        /// <param name="windowHighs">rolling window that tracks the highs</param>
+        /// <param name="midPointIndexOrSurroundingBarsCount">The midpoint index or surrounding bars count for highs</param>
+        /// <returns>pivot point if found else null</returns>
+        protected virtual PivotPoint FindNextHighPivotPoint(RollingWindow<IBaseDataBar> windowHighs, int midPointIndexOrSurroundingBarsCount)
+        {
+            var isHigh = true;
+            var middlePoint = windowHighs[midPointIndexOrSurroundingBarsCount];
+            for (var k = 0; k < windowHighs.Size && isHigh; k++)
+            {
+                // Skip the middle point
+                if (k == midPointIndexOrSurroundingBarsCount)
+                {
+                    continue;
+                }
+
+                // Check if current high is below middle point high
+                isHigh = windowHighs[k].High < middlePoint.High;
+            }
+
+            PivotPoint high = null;
+            if (isHigh)
+            {
+                high = new PivotPoint(PivotPointType.High, middlePoint.High, middlePoint.EndTime);
+            }
+
+            return high;
+        }
+
+        /// <summary>
+        /// Method for converting high and low pivot points to a decimal value.
+        /// </summary>
+        /// <param name="highPoint">new high point or null</param>
+        /// <param name="lowPoint">new low point or null</param>
+        /// <returns>a decimal value representing the values of high and low pivot points</returns>
+        protected virtual decimal ConvertToComputedValue(PivotPoint highPoint, PivotPoint lowPoint)
+        {
+            if (highPoint != null)
+            {
+                if (lowPoint != null)
                 {
                     // Can be the bar forms both high and low pivot points at the same time
-                    OnNewPivotPointFormed(low);
                     return (decimal)PivotPointType.Both;
                 }
                 return (decimal)PivotPointType.High;
             }
 
-            if (low != null)
+            if (lowPoint != null)
             {
-                OnNewPivotPointFormed(low);
                 return (decimal)PivotPointType.Low;
             }
 
@@ -204,10 +245,13 @@ namespace QuantConnect.Indicators
         /// <summary>
         /// Invokes NewPivotPointFormed event
         /// </summary>
-        protected virtual void OnNewPivotPointFormed(PivotPoint pivotPoint)
+        private void OnNewPivotPointFormed(PivotPoint pivotPoint)
         {
-            _windowPivotPoints.Add(pivotPoint);
-            NewPivotPointFormed?.Invoke(this, new PivotPointsEventArgs(pivotPoint));
+            if (pivotPoint != null)
+            {
+                _windowPivotPoints.Add(pivotPoint);
+                NewPivotPointFormed?.Invoke(this, new PivotPointsEventArgs(pivotPoint));
+            }
         }
     }
 

Indicators/RelativeStrengthIndex.cs

@@ -13,6 +13,8 @@
  * limitations under the License.
 */
 
+using System;
+
 namespace QuantConnect.Indicators
 {
     /// <summary>
@@ -98,7 +100,8 @@ namespace QuantConnect.Indicators
             var averageLoss = AverageLoss < 0 ? 0 : AverageLoss.Current.Value;
             var averageGain = AverageGain < 0 ? 0 : AverageGain.Current.Value;
 
-            if (averageLoss == 0m)
+            // Round AverageLoss to avoid computing RSI with very small numbers that lead to overflow exception on the division operation below
+            if (Math.Round(averageLoss, 10) == 0m)
             {
                 // all up days is 100
                 return 100m;
@@ -120,4 +123,4 @@ namespace QuantConnect.Indicators
             base.Reset();
         }
     }
-}
+}

Indicators/WindowIndicator.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -20,7 +20,7 @@ namespace QuantConnect.Indicators
     /// <summary>
     /// Represents an indicator that acts on a rolling window of data
     /// </summary>
-    public abstract class WindowIndicator<T> : IndicatorBase<T>
+    public abstract class WindowIndicator<T> : IndicatorBase<T>, IIndicatorWarmUpPeriodProvider
         where T : IBaseData
     {
         // a window of data over a certain look back period
@@ -47,6 +47,11 @@ namespace QuantConnect.Indicators
         /// </summary>
         public override bool IsReady => _window.IsReady;
 
+        /// <summary>
+        /// Required period, in data points, to the indicator to be ready and fully initialized
+        /// </summary>
+        public int WarmUpPeriod => Period;
+
         /// <summary>
         /// Computes the next value of this indicator from the given state
         /// </summary>
@@ -75,4 +80,4 @@ namespace QuantConnect.Indicators
         /// <returns>A new value for this indicator</returns>
         protected abstract decimal ComputeNextValue(IReadOnlyWindow<T> window, T input);
     }
-}
+}

Launcher/config.json

@@ -415,6 +415,8 @@
       "real-time-handler": "QuantConnect.Lean.Engine.RealTime.LiveTradingRealTimeHandler",
       "transaction-handler": "QuantConnect.Lean.Engine.TransactionHandlers.BrokerageTransactionHandler"
     },
+    
+    // defines the 'live-kraken' environment
     "live-kraken": {
       "live-mode": true,
 
@@ -425,7 +427,8 @@
       "result-handler": "QuantConnect.Lean.Engine.Results.LiveTradingResultHandler",
       "data-feed-handler": "QuantConnect.Lean.Engine.DataFeeds.LiveTradingDataFeed",
       "real-time-handler": "QuantConnect.Lean.Engine.RealTime.LiveTradingRealTimeHandler",
-      "transaction-handler": "QuantConnect.Lean.Engine.TransactionHandlers.BrokerageTransactionHandler"
+      "transaction-handler": "QuantConnect.Lean.Engine.TransactionHandlers.BrokerageTransactionHandler",
+      "history-provider": "BrokerageHistoryProvider"
     },
     "live-ftx": {
       "live-mode": true,

Tests/Common/Securities/BuyingPowerModelTests.cs

@@ -14,7 +14,9 @@
 */
 
 using System;
+using NodaTime;
 using NUnit.Framework;
+using QuantConnect.Data.Market;
 using QuantConnect.Securities;
 
 namespace QuantConnect.Tests.Common.Securities
@@ -22,6 +24,15 @@ namespace QuantConnect.Tests.Common.Securities
     [TestFixture]
     public class BuyingPowerModelTests
     {
+        private BuyingPowerModel _model;
+
+        [OneTimeSetUp]
+        public void Setup()
+        {
+            _model = new BuyingPowerModel();
+        }
+
+
         // Current Order Margin 
         [TestCase(-40, 25, -900, 1, 4)]   	    // -1000
         [TestCase(-36, 25, -880, 1, 1)]  	    // -900
@@ -39,10 +50,12 @@ namespace QuantConnect.Tests.Common.Securities
         [TestCase(-40.5, 12.5, 1508, .5, 161)]	// -506.25
         public void OrderCalculation(decimal currentHoldings, decimal perUnitMargin, decimal targetMargin, decimal lotSize, decimal expectedOrderSize)
         {
-            var currentHoldingsMargin = currentHoldings * perUnitMargin;
+            var spy = SetupSecurity(currentHoldings, lotSize, perUnitMargin);
+
+            var currentHoldingsMargin = _model.GetInitialMarginRequirement(spy, spy.Holdings.Quantity);
 
             // Determine the order size to get us to our target margin
-            var orderSize = BuyingPowerModel.GetAmountToOrder(currentHoldingsMargin, targetMargin, perUnitMargin, lotSize);
+            var orderSize = _model.GetAmountToOrder(spy, targetMargin, perUnitMargin, out _);
             Assert.AreEqual(expectedOrderSize, orderSize);
 
             // Determine the final margin and assert we have met our target condition
@@ -69,15 +82,17 @@ namespace QuantConnect.Tests.Common.Securities
 
         public void OrderAdjustmentCalculation(decimal currentOrderSize, decimal perUnitMargin, decimal targetMargin, decimal lotSize, decimal expectedOrderSize)
         {
-            var currentOrderMargin = currentOrderSize * perUnitMargin;
+            var spy = SetupSecurity(currentOrderSize, lotSize, perUnitMargin);
+
+            var currentHoldingsMargin = _model.GetInitialMarginRequirement(spy, spy.Holdings.Quantity);
 
             // Determine the adjustment to get us to our target margin and apply it
             // Use our GetAmountToOrder for determining adjustment to reach the end goal
             var orderAdjustment =
-                BuyingPowerModel.GetAmountToOrder(currentOrderMargin, targetMargin, perUnitMargin, lotSize);
+                _model.GetAmountToOrder(spy, targetMargin, perUnitMargin, out _);
 
             // Apply the change in margin
-            var resultMargin = currentOrderMargin + (orderAdjustment * perUnitMargin);
+            var resultMargin = currentHoldingsMargin + (orderAdjustment * perUnitMargin);
 
             // Assert after our adjustment we have met our target condition
             Assert.IsTrue(Math.Abs(resultMargin) <= Math.Abs(targetMargin));
@@ -86,5 +101,30 @@ namespace QuantConnect.Tests.Common.Securities
             var adjustOrderSize = currentOrderSize + orderAdjustment;
             Assert.AreEqual(expectedOrderSize, adjustOrderSize);
         }
+
+        /// <summary>
+        /// Helper method for tests, sets up an equity security with our properties
+        /// </summary>
+        /// <returns>Equity with the given setup values</returns>
+        private static Security SetupSecurity(decimal currentHoldings, decimal lotSize, decimal perUnitMargin)
+        {
+            var spy = new QuantConnect.Securities.Equity.Equity(Symbols.SPY, SecurityExchangeHours.AlwaysOpen(DateTimeZone.Utc), new Cash("$", 0, 1),
+                new SymbolProperties(null, "$", 1, 0.01m, lotSize, null, 0), null, null, new SecurityCache());
+
+            spy.Holdings.SetHoldings(perUnitMargin, currentHoldings);
+            spy.SetLeverage(1);
+
+            spy.SetMarketPrice(new TradeBar
+            {
+                Time = DateTime.Now,
+                Symbol = spy.Symbol,
+                Open = perUnitMargin,
+                High = perUnitMargin,
+                Low = perUnitMargin,
+                Close = perUnitMargin
+            });
+
+            return spy;
+        }
     }
 }

Tests/Common/Securities/OptionMarginBuyingPowerModelTests.cs

@@ -19,6 +19,7 @@ using QuantConnect.Algorithm;
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
 using QuantConnect.Orders;
+using QuantConnect.Orders.Fees;
 using QuantConnect.Securities;
 using QuantConnect.Securities.Option;
 using QuantConnect.Tests.Engine.DataFeeds;
@@ -410,5 +411,71 @@ namespace QuantConnect.Tests.Common.Securities
             Assert.AreEqual(10000m + algorithm.Portfolio.CashBook[Currencies.USD].ValueInAccountCurrency,
                 quantity.Value);
         }
+
+        // This test set showcases some odd behaviour by our OptionMarginModel margin requirement calculation.
+
+        // ~-1.5% Target (~-15K). If we are already shorted or long we reduce holdings to 0, this is because the requirement for a
+        // short option position is at base ~-200K, but the issue is if we have zero holdings it allows us to buy -31 contracts for
+        // 478 margin requirement per unit. This is because the margin requirement seems to be contingent upon the current holdings.
+        [TestCase(-31, 31, -.015)] // Short to Short (-31 + 31 = 0)
+        [TestCase(0, -31, -.015)] // Open Short (0 + -31 = -31)
+        [TestCase(31, -31, -.015)] // Long To Short (31 + -31 = 0)
+
+        // -40% Target (~-400k), All end up at different allocations.
+        // This is because of the initial margin requirement calculations.
+        [TestCase(-31, -380, -0.40)] // Short to Shorter (-31 + -380 = -411)
+        [TestCase(0, -836, -0.40)] // Open Short (0 + -836 = -836)
+        [TestCase(31, -467, -0.40)] // Long To Short (31 + -467 = -436)
+
+        // 40% Target (~400k), All end up at different allocations.
+        // This is because of the initial margin requirement calculations.
+        [TestCase(-31, 855, 0.40)] // Short to Long (-31 + 855 = 824)
+        [TestCase(0, 836, 0.40)] // Open Long (0 + 836 = 836)
+        [TestCase(31, 818, 0.40)] // Long To Longer (31 + 818 = 849)
+
+        // ~0.04% Target (~400). This is below the needed margin for one unit. We end up at 0 holdings for all cases.
+        [TestCase(-31, 31, 0.0004)] // Short to Long (-31 + 31 = 0)
+        [TestCase(0, 0, 0.0004)] // Open Long (0 + 0 = 0)
+        [TestCase(31, -31, 0.0004)] // Long To Longer (31 + -31 = 0)
+        public void CallOTM_MarginRequirement(int startingHoldings, int expectedOrderSize, decimal targetPercentage)
+        {
+            // Initialize algorithm
+            var algorithm = new QCAlgorithm();
+            algorithm.SetFinishedWarmingUp();
+            algorithm.Transactions.SetOrderProcessor(new FakeOrderProcessor());
+
+            algorithm.SetCash(1000000);
+            algorithm.SubscriptionManager.SetDataManager(new DataManagerStub(algorithm));
+
+            var optionSymbol = Symbols.CreateOptionSymbol("SPY", OptionRight.Call, 411m, DateTime.UtcNow);
+            var option = algorithm.AddOptionContract(optionSymbol);
+
+            option.Holdings.SetHoldings(4.74m, startingHoldings);
+            option.FeeModel = new ConstantFeeModel(0);
+            option.SetLeverage(1);
+
+            // Update option data
+            UpdatePrice(option, 4.78m);
+
+            // Update the underlying data
+            UpdatePrice(option.Underlying, 395.51m);
+
+            var model = new OptionMarginModel();
+            var result = model.GetMaximumOrderQuantityForTargetBuyingPower(algorithm.Portfolio, option, targetPercentage, 0);
+            Assert.AreEqual(expectedOrderSize, result.Quantity);
+        }
+
+        private static void UpdatePrice(Security security, decimal close)
+        {
+            security.SetMarketPrice(new TradeBar
+            {
+                Time = DateTime.Now,
+                Symbol = security.Symbol,
+                Open = close,
+                High = close,
+                Low = close,
+                Close = close
+            });
+        }
     }
 }

Tests/Indicators/IdentityTests.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -60,5 +60,21 @@ namespace QuantConnect.Tests.Indicators
             Assert.IsFalse(identity.IsReady);
             Assert.AreEqual(0, identity.Samples);
         }
+
+        [Test]
+        public void WarmsUpProperly() 
+        {
+            var identityIndicator = new Identity("Example");
+            var time = new DateTime(2020, 8, 1);
+            var period = ((IIndicatorWarmUpPeriodProvider)identityIndicator).WarmUpPeriod;
+
+            Assert.IsFalse(identityIndicator.IsReady);
+
+            for (var i = 0; i < period; i++)
+            {
+                identityIndicator.Update(time.AddDays(i), i);
+                Assert.AreEqual(i == period - 1, identityIndicator.IsReady);
+            }
+        }
     }
 }

Tests/Indicators/WindowIdentityTests.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  * 
@@ -76,5 +76,19 @@ namespace QuantConnect.Tests.Indicators
             var indicator = new WindowIdentity(14);
             TestHelper.TestIndicator(indicator, "Close", 1e-2); // test file only has
         }
+
+        [Test]
+        public void WarmsUpProperly()
+        {
+            var windowIdentityIndicator = new WindowIdentity("Example", 13);
+            var time = new DateTime(2020, 8, 1);
+            var period = ((IIndicatorWarmUpPeriodProvider)windowIdentityIndicator).WarmUpPeriod;
+
+            for (var i = 0; i < period; i++)
+            {
+                windowIdentityIndicator.Update(time.AddDays(i), i);
+                Assert.AreEqual(i == period - 1, windowIdentityIndicator.IsReady);
+            }
+        }
     }
 }

ToolBox/CoinApi/CoinApiSymbolMapper.cs

@@ -17,10 +17,11 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Net;
 using Newtonsoft.Json;
 using QuantConnect.Brokerages;
 using QuantConnect.Configuration;
+using QuantConnect.Logging;
+using QuantConnect.Securities;
 
 namespace QuantConnect.ToolBox.CoinApi
 {
@@ -41,7 +42,9 @@ namespace QuantConnect.ToolBox.CoinApi
         {
             { Market.GDAX, "COINBASE" },
             { Market.Bitfinex, "BITFINEX" },
-            { Market.Binance, "BINANCE" }
+            { Market.Binance, "BINANCE" },
+            { Market.FTX, "FTX" },
+            { Market.Kraken, "KRAKEN" }
         };
         private static readonly Dictionary<string, string> MapExchangeIdsToMarkets =
             MapMarketsToExchangeIds.ToDictionary(x => x.Value, x => x.Key);
@@ -197,24 +200,25 @@ namespace QuantConnect.ToolBox.CoinApi
             // There were cases of entries in the CoinApiSymbols list with the following pattern:
             // <Exchange>_SPOT_<BaseCurrency>_<QuoteCurrency>_<ExtraSuffix>
             // Those cases should be ignored for SPOT prices.
-            _symbolMap = result
+            foreach (var x in  result
                 .Where(x => x.SymbolType == "SPOT" &&
                     x.SymbolId.Split('_').Length == 4 &&
                     // exclude Bitfinex BCH pre-2018-fork as for now we don't have historical mapping data
                     (x.ExchangeId != "BITFINEX" || x.AssetIdBase != "BCH" && x.AssetIdQuote != "BCH")
                     // solves the cases where we request 'binance' and get 'binanceus'
-                    && MapExchangeIdsToMarkets.ContainsKey(x.ExchangeId))
-                .ToDictionary(
-                    x =>
-                    {
-                        var market = MapExchangeIdsToMarkets[x.ExchangeId];
-                        return Symbol.Create(
-                            ConvertCoinApiCurrencyToLeanCurrency(x.AssetIdBase, market) +
-                            ConvertCoinApiCurrencyToLeanCurrency(x.AssetIdQuote, market),
-                            SecurityType.Crypto,
-                            market);
-                    },
-                    x => x.SymbolId);
+                    && MapExchangeIdsToMarkets.ContainsKey(x.ExchangeId)))
+            {
+                var market = MapExchangeIdsToMarkets[x.ExchangeId];
+                var symbol = GetLeanSymbol(x.SymbolId, SecurityType.Crypto, market);
+
+                if (_symbolMap.ContainsKey(symbol))
+                {
+                    // skipping duplicate symbols. Kraken has both USDC/AD & USD/CAD symbols
+                    Log.Error($"CoinApiSymbolMapper(): Duplicate symbol found {symbol} will be skipped!");
+                    continue;
+                }
+                _symbolMap[symbol] = x.SymbolId;
+            }
         }
 
         private static string ConvertCoinApiCurrencyToLeanCurrency(string currency, string market)

ToolBox/CoinApiDataConverter/CoinApiDataConverter.cs

@@ -39,7 +39,9 @@ namespace QuantConnect.ToolBox.CoinApiDataConverter
         {
             Market.GDAX,
             Market.Bitfinex,
-            Market.Binance
+            Market.Binance,
+            Market.FTX,
+            Market.Kraken
         }.ToHashSet();
 
         private readonly DirectoryInfo _rawDataFolder;
@@ -119,7 +121,16 @@ namespace QuantConnect.ToolBox.CoinApiDataConverter
             {
                 try
                 {
-                    var key = candidate.Directory.Parent.Name + apiDataReader.GetCoinApiEntryData(candidate, _processingDate).Symbol.ID;
+                    var entryData = apiDataReader.GetCoinApiEntryData(candidate, _processingDate);
+                    CurrencyPairUtil.DecomposeCurrencyPair(entryData.Symbol, out var baseCurrency,
+                        out var quoteCurrency);
+
+                    if (!candidate.FullName.Contains(baseCurrency) && !candidate.FullName.Contains(quoteCurrency))
+                    {
+                        throw new Exception($"Skipping {candidate.FullName} we have the wrong symbol {entryData.Symbol}!");
+                    }
+
+                    var key = candidate.Directory.Parent.Name + entryData.Symbol.ID;
                     if (filesToProcessKeys.Add(key))
                     {
                         // Separate list from HashSet to preserve ordering of viable candidates

readme.md

@@ -94,7 +94,7 @@ dotnet QuantConnect.Lean.Launcher.dll
 - Install [dotnet 5](https://docs.microsoft.com/en-us/dotnet/core/install/linux):
 - Compile Lean Solution:
 ```
-dotnet QuantConnect.Lean.sln
+dotnet build QuantConnect.Lean.sln
 ```
 - Run Lean:
 ```