Accept symbol array or IEnumerable for TRIN (#5473)

* Make random data generator generate symbols with entries in SPD

* Cap symbol count in generator when needed

Algorithm/QCAlgorithm.Indicators.cs

@@ -1887,6 +1887,17 @@ namespace QuantConnect.Algorithm
         /// <param name="resolution">The resolution</param>
         /// <returns>The Arms Index indicator for the requested symbol over the specified period</returns>
         public ArmsIndex TRIN(IEnumerable<Symbol> symbols, Resolution? resolution = null)
+        {
+            return TRIN(symbols.ToArray(), resolution);
+        }
+
+        /// <summary>
+        /// Creates a new Arms Index indicator
+        /// </summary>
+        /// <param name="symbols">The symbols whose Arms Index we want</param>
+        /// <param name="resolution">The resolution</param>
+        /// <returns>The Arms Index indicator for the requested symbol over the specified period</returns>
+        public ArmsIndex TRIN(Symbol[] symbols, Resolution? resolution = null)
         {
             var name = CreateIndicatorName(QuantConnect.Symbol.None, "TRIN", resolution ?? GetSubscription(symbols.First()).Resolution);
             var trin = new ArmsIndex(name);

Tests/ToolBox/RandomDataGenerator/RandomValueGeneratorTests.cs

@@ -98,11 +98,13 @@ namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
         }
 
         [Test]
-        [TestCase(SecurityType.Cfd, Market.FXCM)]
-        [TestCase(SecurityType.Base, Market.USA)]
-        [TestCase(SecurityType.Forex, Market.FXCM)]
         [TestCase(SecurityType.Equity, Market.USA)]
+        [TestCase(SecurityType.Cfd, Market.FXCM)]
+        [TestCase(SecurityType.Cfd, Market.Oanda)]
+        [TestCase(SecurityType.Forex, Market.FXCM)]
+        [TestCase(SecurityType.Forex, Market.Oanda)]
         [TestCase(SecurityType.Crypto, Market.GDAX)]
+        [TestCase(SecurityType.Crypto, Market.Bitfinex)]
         public void NextSymbol_CreatesSymbol_WithRequestedSecurityTypeAndMarket(SecurityType securityType, string market)
         {
             var symbol = randomValueGenerator.NextSymbol(securityType, market);
@@ -112,27 +114,54 @@ namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
         }
 
         [Test]
-        [TestCase(SecurityType.Cfd)]
-        [TestCase(SecurityType.Base)]
-        [TestCase(SecurityType.Forex)]
-        [TestCase(SecurityType.Equity)]
-        [TestCase(SecurityType.Crypto)]
-        public void NextSymbol_CreatesSymbol_WithThreeCharacterTicker(SecurityType securityType)
+        [TestCase(SecurityType.Equity, Market.USA)]
+        [TestCase(SecurityType.Cfd, Market.FXCM)]
+        [TestCase(SecurityType.Cfd, Market.Oanda)]
+        [TestCase(SecurityType.Forex, Market.FXCM)]
+        [TestCase(SecurityType.Forex, Market.Oanda)]
+        [TestCase(SecurityType.Crypto, Market.GDAX)]
+        [TestCase(SecurityType.Crypto, Market.Bitfinex)]
+        public void NextSymbol_CreatesSymbol_WithEntryInSymbolPropertiesDatabase(SecurityType securityType, string market)
         {
-            var defaultMarket = DefaultBrokerageModel.DefaultMarketMap[securityType];
-            var symbol = randomValueGenerator.NextSymbol(securityType, defaultMarket);
+            var symbol = randomValueGenerator.NextSymbol(securityType, market);
 
-            // for derivatives, check the underlying ticker
-            if (securityType == SecurityType.Option || securityType == SecurityType.Future)
+            var db = SymbolPropertiesDatabase.FromDataFolder();
+            if (db.ContainsKey(market, SecurityDatabaseKey.Wildcard, securityType))
             {
-                symbol = symbol.Underlying;
+                // there is a wildcard entry, so no need to check whether there is a specific entry for the symbol
+                Assert.Pass();
+            }
+            else
+            {
+                // there is no wildcard entry, so there should be a specific entry for the symbol instead
+                Assert.IsTrue(db.ContainsKey(market, symbol, securityType));
             }
-
-            Assert.AreEqual(3, symbol.Value.Length);
         }
 
         [Test]
-        public void NextOptionSymbol_CreatesOptionSymbol_WithCorrectSecurityTypeAndEquitUnderlying()
+        [TestCase(SecurityType.Cfd, Market.FXCM)]
+        [TestCase(SecurityType.Cfd, Market.Oanda)]
+        [TestCase(SecurityType.Forex, Market.FXCM)]
+        [TestCase(SecurityType.Forex, Market.Oanda)]
+        [TestCase(SecurityType.Crypto, Market.GDAX)]
+        [TestCase(SecurityType.Crypto, Market.Bitfinex)]
+        public void NextSymbol_ThrowsNoTickersAvailableException_WhenAllSymbolsGenerated(SecurityType securityType, string market)
+        {
+            var db = SymbolPropertiesDatabase.FromDataFolder();
+            var symbolCount = db.GetSymbolPropertiesList(market, securityType).Count();
+
+            for (var i = 0; i < symbolCount; i++)
+            {
+                randomValueGenerator.NextSymbol(securityType, market);
+            }
+
+            Assert.Throws<NoTickersAvailableException>(() =>
+                randomValueGenerator.NextSymbol(securityType, market)
+            );
+        }
+
+        [Test]
+        public void NextOptionSymbol_CreatesOptionSymbol_WithCorrectSecurityTypeAndEquityUnderlying()
         {
             var minExpiry = new DateTime(2000, 01, 01);
             var maxExpiry = new DateTime(2001, 01, 01);
@@ -334,6 +363,7 @@ namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
             Assert.AreEqual(Market.CME, symbol.ID.Market);
             Assert.AreEqual(SecurityType.Future, symbol.SecurityType);
         }
+
         [Test]
         public void NextFuture_CreatesSymbol_WithFutureWithValidFridayExpiry()
         {
@@ -346,5 +376,36 @@ namespace QuantConnect.Tests.ToolBox.RandomDataGenerator
             Assert.LessOrEqual(expiry, maxExpiry);
             Assert.AreEqual(DayOfWeek.Friday, expiry.DayOfWeek);
         }
+
+        [Test]
+        public void NextFuture_CreatesSymbol_WithEntryInSymbolPropertiesDatabase()
+        {
+            var minExpiry = new DateTime(2000, 01, 01);
+            var maxExpiry = new DateTime(2001, 01, 01);
+            var symbol = randomValueGenerator.NextFuture(Market.CME, minExpiry, maxExpiry);
+
+            var db = SymbolPropertiesDatabase.FromDataFolder();
+            Assert.IsTrue(db.ContainsKey(Market.CME, symbol, SecurityType.Future));
+        }
+
+        [Test]
+        [TestCase(SecurityType.Equity, Market.USA, true)]
+        [TestCase(SecurityType.Cfd, Market.FXCM, false)]
+        [TestCase(SecurityType.Cfd, Market.Oanda, false)]
+        [TestCase(SecurityType.Forex, Market.FXCM, false)]
+        [TestCase(SecurityType.Forex, Market.Oanda, false)]
+        [TestCase(SecurityType.Crypto, Market.GDAX, false)]
+        [TestCase(SecurityType.Crypto, Market.Bitfinex, false)]
+        [TestCase(SecurityType.Option, Market.USA, true)]
+        [TestCase(SecurityType.Future, Market.CME, true)]
+        [TestCase(SecurityType.Future, Market.CBOE, true)]
+        public void GetAvailableSymbolCount(SecurityType securityType, string market, bool expectInfinity)
+        {
+            var expected = expectInfinity
+                ? int.MaxValue
+                : SymbolPropertiesDatabase.FromDataFolder().GetSymbolPropertiesList(market, securityType).Count();
+
+            Assert.AreEqual(expected, randomValueGenerator.GetAvailableSymbolCount(securityType, market));
+        }
     }
 }

ToolBox/RandomDataGenerator/IRandomValueGenerator.cs

@@ -86,6 +86,7 @@ namespace QuantConnect.ToolBox.RandomDataGenerator
 
         /// <summary>
         /// Generates a new random <see cref="Symbol"/> object of the specified security type.
+        /// All returned symbols have a matching entry in the symbol properties database.
         /// </summary>
         /// <remarks>
         /// A valid implementation will keep track of generated symbol objects to ensure duplicates
@@ -94,8 +95,9 @@ namespace QuantConnect.ToolBox.RandomDataGenerator
         /// <exception cref="ArgumentException">Throw when specifying <see cref="SecurityType.Option"/> or
         /// <see cref="SecurityType.Future"/>. To generate symbols for the derivative security types, please
         /// use <see cref="NextOption"/> and <see cref="NextFuture"/> respectively</exception>
+        /// <exception cref="NoTickersAvailableException">Thrown when there are no tickers left to use for new symbols.</exception>
         /// <param name="securityType">The security type of the generated symbol</param>
-        /// <param name="market"></param>
+        /// <param name="market">The market of the generated symbol</param>
         /// <returns>A new symbol object of the specified security type</returns>
         Symbol NextSymbol(SecurityType securityType, string market);
 
@@ -110,7 +112,7 @@ namespace QuantConnect.ToolBox.RandomDataGenerator
         /// Standard contracts expiry on the third Friday.
         /// Weekly contracts expiry every week on Friday
         /// </remarks>
-        /// <param name="market"></param>
+        /// <param name="market">The market of the generated symbol</param>
         /// <param name="minExpiry">The minimum expiry date, inclusive</param>
         /// <param name="maxExpiry">The maximum expiry date, inclusive</param>
         /// <param name="underlyingPrice">The option's current underlying price</param>
@@ -122,10 +124,19 @@ namespace QuantConnect.ToolBox.RandomDataGenerator
         /// Generates a new random future <see cref="Symbol"/>. The generates future contract symbol will have an
         /// expiry between the specified <paramref name="minExpiry"/> and <paramref name="maxExpiry"/>.
         /// </summary>
-        /// <param name="market"></param>
+        /// <param name="market">The market of the generated symbol</param>
         /// <param name="minExpiry">The minimum expiry date, inclusive</param>
         /// <param name="maxExpiry">The maximum expiry date, inclusive</param>
         /// <returns>A new future contract symbol with the specified expiration parameters</returns>
         Symbol NextFuture(string market, DateTime minExpiry, DateTime maxExpiry);
+
+        /// <summary>
+        /// Returns the number of symbols with the specified parameters can be generated.
+        /// Returns int.MaxValue if there is no limit for the given parameters.
+        /// </summary>
+        /// <param name="securityType">The security type of the generated symbols</param>
+        /// <param name="market">The market of the generated symbols</param>
+        /// <returns>The number of available symbols for the given parameters, or int.MaxValue if no limit</returns>
+        int GetAvailableSymbolCount(SecurityType securityType, string market);
     }
-}
+}

ToolBox/RandomDataGenerator/NoTickersAvailableException.cs

@@ -0,0 +1,13 @@
+namespace QuantConnect.ToolBox.RandomDataGenerator
+{
+    /// <summary>
+    /// Exception thrown when there are no tickers left to generate for a certain combination of security type and market.
+    /// </summary>
+    public class NoTickersAvailableException : RandomValueGeneratorException
+    {
+        public NoTickersAvailableException(SecurityType securityType, string market)
+            : base($"Failed to generate {securityType} symbol for {market}, there are no tickers left")
+        {
+        }
+    }
+}

ToolBox/RandomDataGenerator/RandomDataGeneratorProgram.cs

@@ -100,6 +100,14 @@ namespace QuantConnect.ToolBox.RandomDataGenerator
                 random = new Random(settings.RandomSeed);
                 randomValueGenerator = new RandomValueGenerator(settings.RandomSeed);
             }
+
+            var maxSymbolCount = randomValueGenerator.GetAvailableSymbolCount(settings.SecurityType, settings.Market);
+            if (settings.SymbolCount > maxSymbolCount)
+            {
+                output.Warn.WriteLine($"Limiting symbol count to {maxSymbolCount}, we don't have more {settings.SecurityType} tickers for {settings.Market}");
+                settings.SymbolCount = maxSymbolCount;
+            }
+
             var symbolGenerator = new SymbolGenerator(settings, randomValueGenerator);
             var tickGenerator = new TickGenerator(settings, randomValueGenerator);
 

ToolBox/RandomDataGenerator/RandomValueGenerator.cs

@@ -261,8 +261,20 @@ namespace QuantConnect.ToolBox.RandomDataGenerator
                 throw new ArgumentException("Please use NextOption or NextFuture for SecurityType.Option and SecurityType.Future respectively.");
             }
 
-            // let's make symbols all have 3 chars as it's acceptable for all permitted security types in this method
-            var ticker = NextUpperCaseString(3, 3);
+            string ticker;
+
+            // we must return a symbol matching an entry in the symbol properties database
+            // if there is a wildcard entry, we can generate a truly random symbol
+            // if there is no wildcard entry, the symbols we can generate are limited by the entries in the database
+            if (_symbolPropertiesDatabase.ContainsKey(market, SecurityDatabaseKey.Wildcard, securityType))
+            {
+                // let's make symbols all have 3 chars as it's acceptable for all security types with wildcard entries
+                ticker = NextUpperCaseString(3, 3);
+            }
+            else
+            {
+                ticker = NextTickerFromSymbolPropertiesDatabase(securityType, market);
+            }
 
             // by chance we may generate a ticker that actually exists, and if map files exist that match this
             // ticker then we'll end up resolving the first trading date for use in the SID, otherwise, all
@@ -303,15 +315,53 @@ namespace QuantConnect.ToolBox.RandomDataGenerator
 
         public virtual Symbol NextFuture(string market, DateTime minExpiry, DateTime maxExpiry)
         {
-            // futures are usually two characters
-            var ticker = NextUpperCaseString(2, 2);
+            // get a valid ticker from the symbol properties database
+            var ticker = NextTickerFromSymbolPropertiesDatabase(SecurityType.Future, market);
 
-            var marketHours = _marketHoursDatabase.GetExchangeHours(market, null, SecurityType.Future);
+            var marketHours = _marketHoursDatabase.GetExchangeHours(market, ticker, SecurityType.Future);
             var expiry = GetRandomExpiration(marketHours, minExpiry, maxExpiry);
 
             return Symbol.CreateFuture(ticker, market, expiry);
         }
 
+        public virtual int GetAvailableSymbolCount(SecurityType securityType, string market)
+        {
+            // there is no limit to the number of option/future contracts we can generate
+            if (securityType == SecurityType.Option || securityType == SecurityType.Future)
+            {
+                return int.MaxValue;
+            }
+
+            // check the symbol properties database to determine how many symbols we can generate
+            // if there is a wildcard entry, we can generate as many symbols as we want
+            // if there is no wildcard entry, we can only generate as many symbols as there are entries
+            return _symbolPropertiesDatabase.ContainsKey(market, SecurityDatabaseKey.Wildcard, securityType)
+                ? int.MaxValue
+                : _symbolPropertiesDatabase.GetSymbolPropertiesList(market, securityType).Count();
+        }
+
+        private string NextTickerFromSymbolPropertiesDatabase(SecurityType securityType, string market)
+        {
+            // prevent returning a ticker matching any previously generated symbol
+            var existingTickers = _symbols
+                .Where(sym => sym.ID.Market == market && sym.ID.SecurityType == securityType)
+                .Select(sym => sym.Value);
+
+            // get the available tickers from the symbol properties database and remove previously generated tickers
+            var availableTickers = _symbolPropertiesDatabase.GetSymbolPropertiesList(market, securityType)
+                .Select(kvp => kvp.Key.Symbol)
+                .Except(existingTickers)
+                .ToList();
+
+            // there is a limited number of entries in the symbol properties database so we may run out of tickers
+            if (availableTickers.Count == 0)
+            {
+                throw new NoTickersAvailableException(securityType, market);
+            }
+
+            return availableTickers[_random.Next(availableTickers.Count)];
+        }
+
         private bool IsWithinRange(DateTime value, DateTime min, DateTime max)
         {
             return value >= min && value <= max;