Refactor user define universe handling (#9088)

* Refactor user define universe handling

- Normalize user define universe additions and removals to behave like
  other subscriptions without requiting special handling

* Minor fixes
This commit is contained in:
Martin-Molinero
2025-11-18 12:29:05 -03:00
committed by GitHub
parent afca923cd9
commit 4961844f82
88 changed files with 378 additions and 383 deletions

View File

@@ -87,7 +87,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 24;
public long DataPoints => 26;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -82,7 +82,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 24;
public long DataPoints => 25;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -115,7 +115,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 61;
public long DataPoints => 62;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -117,7 +117,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 37597;
public long DataPoints => 37598;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -169,7 +169,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 5798;
public long DataPoints => 5800;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -116,7 +116,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 3814;
public long DataPoints => 3818;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -96,7 +96,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 1658167;
public long DataPoints => 1658168;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -83,7 +83,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 11202;
public long DataPoints => 15042;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -110,7 +110,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 7063;
public long DataPoints => 7065;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -101,7 +101,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 1578;
public long DataPoints => 1579;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -119,7 +119,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 126221;
public long DataPoints => 126222;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -139,7 +139,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 17486;
public long DataPoints => 17487;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -81,7 +81,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public virtual long DataPoints => 47132;
public virtual long DataPoints => 47140;
/// <summary>
/// Data Points count of the algorithm history
@@ -117,7 +117,7 @@ namespace QuantConnect.Algorithm.CSharp
{"Beta", "0"},
{"Annual Standard Deviation", "0"},
{"Annual Variance", "0"},
{"Information Ratio", "-5.732"},
{"Information Ratio", "-6.035"},
{"Tracking Error", "0.05"},
{"Treynor Ratio", "0"},
{"Total Fees", "$0.00"},

View File

@@ -26,6 +26,6 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public override long DataPoints => 46264;
public override long DataPoints => 46271;
}
}

View File

@@ -105,7 +105,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 7122;
public long DataPoints => 7123;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -120,7 +120,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 17099;
public long DataPoints => 17100;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -153,7 +153,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 86;
public long DataPoints => 87;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -213,7 +213,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 2849;
public long DataPoints => 2850;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -136,7 +136,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 63;
public long DataPoints => 64;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -135,7 +135,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 31;
public long DataPoints => 32;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -112,7 +112,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 309282;
public long DataPoints => 309286;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -200,7 +200,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 212196;
public long DataPoints => 212198;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -165,7 +165,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 212196;
public long DataPoints => 212198;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -176,7 +176,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 212196;
public long DataPoints => 212198;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -201,7 +201,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 212196;
public long DataPoints => 212198;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -174,7 +174,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 212196;
public long DataPoints => 212198;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -185,7 +185,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 212196;
public long DataPoints => 212198;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -168,7 +168,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 212196;
public long DataPoints => 212198;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -182,7 +182,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 212196;
public long DataPoints => 212198;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -167,7 +167,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 212196;
public long DataPoints => 212198;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -178,7 +178,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 212942;
public long DataPoints => 212944;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -118,7 +118,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 32143;
public long DataPoints => 32144;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -38,7 +38,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public override long DataPoints => 195;
public override long DataPoints => 196;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -170,7 +170,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public virtual long DataPoints => 19908;
public virtual long DataPoints => 19909;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -156,7 +156,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 19908;
public long DataPoints => 19909;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -38,7 +38,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public override long DataPoints => 185;
public override long DataPoints => 186;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -178,7 +178,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public virtual long DataPoints => 15941;
public virtual long DataPoints => 15942;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -185,7 +185,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 19890;
public long DataPoints => 19891;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -169,7 +169,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 19984;
public long DataPoints => 19985;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -183,7 +183,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 19908;
public long DataPoints => 19909;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -163,7 +163,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 15941;
public long DataPoints => 15942;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -186,7 +186,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 19890;
public long DataPoints => 19891;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -162,7 +162,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 19984;
public long DataPoints => 19985;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -106,7 +106,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public virtual long DataPoints => 796;
public virtual long DataPoints => 797;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -186,7 +186,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 454077;
public long DataPoints => 454078;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -171,7 +171,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 2821;
public long DataPoints => 2822;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -80,7 +80,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 107;
public long DataPoints => 108;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -92,7 +92,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 4025;
public long DataPoints => 4026;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -261,7 +261,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 4358;
public long DataPoints => 4359;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -113,7 +113,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 2155693;
public long DataPoints => 2155694;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -112,7 +112,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public virtual long DataPoints => 16638;
public virtual long DataPoints => 16640;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -160,7 +160,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 212196;
public long DataPoints => 212198;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -86,12 +86,12 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 10857;
public long DataPoints => 10869;
/// <summary>
/// Data Points count of the algorithm history
/// </summary>
public int AlgorithmHistoryDataPoints => 787;
public int AlgorithmHistoryDataPoints => 788;
/// <summary>
/// Final status of the algorithm

View File

@@ -73,7 +73,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 34;
public long DataPoints => 35;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -115,7 +115,7 @@ namespace QuantConnect.Algorithm.CSharp.RegressionTests
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 4543;
public long DataPoints => 4544;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -106,18 +106,17 @@ namespace QuantConnect.Algorithm.CSharp
}
// if we added the etf subscription it will get added and delisted and send us a addition/removal event
var adjusment = AddETFSubscription ? 1 : 0;
var expectedChangesCount = _universeSymbolCount + adjusment;
var expectedChangesCount = _universeSymbolCount;
if (_universeSelectionDone)
{
// "_universeSymbolCount + 1" because selection is done right away,
// so AddedSecurities includes all ETF constituents (including APPL) plus GDVD
_universeAdded |= changes.AddedSecurities.Count == expectedChangesCount;
// manually added securities are added right away, the etf universe selection happens a few days later when data available
// AAPL was already added so it wont be counted
_universeAdded |= changes.AddedSecurities.Count == (expectedChangesCount - 1);
}
// TODO: shouldn't be sending AAPL as a removed security since it was added by another universe
_universeRemoved |= changes.RemovedSecurities.Count == expectedChangesCount &&
_universeRemoved |= changes.RemovedSecurities.Count == (expectedChangesCount + (AddETFSubscription ? 1 : 0)) &&
UtcTime.Date >= _delistingDate &&
UtcTime.Date < EndDate;
}
@@ -151,7 +150,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public virtual long DataPoints => 692;
public virtual long DataPoints => 826;
/// <summary>
/// Data Points count of the algorithm history
@@ -171,31 +170,31 @@ namespace QuantConnect.Algorithm.CSharp
{"Total Orders", "1"},
{"Average Win", "0%"},
{"Average Loss", "0%"},
{"Compounding Annual Return", "30.084%"},
{"Compounding Annual Return", "26.315%"},
{"Drawdown", "5.400%"},
{"Expectancy", "0"},
{"Start Equity", "100000"},
{"End Equity", "104393.19"},
{"Net Profit", "4.393%"},
{"Sharpe Ratio", "1.543"},
{"Sortino Ratio", "2.111"},
{"Probabilistic Sharpe Ratio", "58.028%"},
{"End Equity", "103892.62"},
{"Net Profit", "3.893%"},
{"Sharpe Ratio", "1.291"},
{"Sortino Ratio", "1.876"},
{"Probabilistic Sharpe Ratio", "53.929%"},
{"Loss Rate", "0%"},
{"Win Rate", "0%"},
{"Profit-Loss Ratio", "0"},
{"Alpha", "0.166"},
{"Beta", "0.717"},
{"Annual Standard Deviation", "0.136"},
{"Alpha", "0.13"},
{"Beta", "0.697"},
{"Annual Standard Deviation", "0.139"},
{"Annual Variance", "0.019"},
{"Information Ratio", "1.254"},
{"Tracking Error", "0.118"},
{"Treynor Ratio", "0.293"},
{"Total Fees", "$2.06"},
{"Estimated Strategy Capacity", "$160000000.00"},
{"Information Ratio", "0.889"},
{"Tracking Error", "0.122"},
{"Treynor Ratio", "0.257"},
{"Total Fees", "$2.04"},
{"Estimated Strategy Capacity", "$260000000.00"},
{"Lowest Capacity Asset", "AAPL R735QTJ8XC9X"},
{"Portfolio Turnover", "0.83%"},
{"Drawdown Recovery", "23"},
{"OrderListHash", "527cba5cfdcac4b0f667bb354e80a1fe"}
{"OrderListHash", "cdf9a800c8ec7d5f9f750f32c2622f5a"}
};
}
}

View File

@@ -26,7 +26,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public override long DataPoints => 511;
public override long DataPoints => 623;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -157,7 +157,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 618;
public long DataPoints => 751;
/// <summary>
/// Data Points count of the algorithm history
@@ -189,13 +189,13 @@ namespace QuantConnect.Algorithm.CSharp
{"Loss Rate", "0%"},
{"Win Rate", "0%"},
{"Profit-Loss Ratio", "0"},
{"Alpha", "-0.084"},
{"Beta", "0.591"},
{"Alpha", "-0.118"},
{"Beta", "0.445"},
{"Annual Standard Deviation", "0.078"},
{"Annual Variance", "0.006"},
{"Information Ratio", "-1.408"},
{"Tracking Error", "0.065"},
{"Treynor Ratio", "-0.125"},
{"Information Ratio", "-2.01"},
{"Tracking Error", "0.086"},
{"Treynor Ratio", "-0.166"},
{"Total Fees", "$22.93"},
{"Estimated Strategy Capacity", "$74000000.00"},
{"Lowest Capacity Asset", "AAPL R735QTJ8XC9X"},

View File

@@ -86,7 +86,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 15885;
public long DataPoints => 15886;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -187,7 +187,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public virtual long DataPoints => 4036;
public virtual long DataPoints => 4072;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -44,7 +44,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public override long DataPoints => 85;
public override long DataPoints => 101;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -83,7 +83,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public override long DataPoints => 115;
public override long DataPoints => 139;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -55,7 +55,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public override long DataPoints => 3170;
public override long DataPoints => 3172;
/// <summary>
/// This is used by the regression test system to indicate what the expected statistics are from running the algorithm

View File

@@ -100,7 +100,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 24288;
public long DataPoints => 24289;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -134,7 +134,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 7000;
public long DataPoints => 7001;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -114,7 +114,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public long DataPoints => 228;
public long DataPoints => 229;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -33,6 +33,6 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public override long DataPoints => 5298;
public override long DataPoints => 5299;
}
}

View File

@@ -99,7 +99,7 @@ namespace QuantConnect.Algorithm.CSharp
/// <summary>
/// Data Points count of all timeslices of algorithm
/// </summary>
public virtual long DataPoints => 3763;
public virtual long DataPoints => 3764;
/// <summary>
/// Data Points count of the algorithm history

View File

@@ -59,16 +59,16 @@ class ETFConstituentUniverseCompositeDelistingRegressionAlgorithm(QCAlgorithm):
raise AssertionError("New securities added after ETF constituents were delisted")
# Since we added the etf subscription it will get delisted and send us a removal event
expected_changes_count = self.universe_symbol_count + 1
expected_changes_count = self.universe_symbol_count
if self.universe_selection_done:
# "_universe_symbol_count + 1" because selection is done right away,
# so AddedSecurities includes all ETF constituents (including APPL) plus GDVD
self.universe_added = self.universe_added or len(changes.added_securities) == expected_changes_count
self.universe_added = self.universe_added or len(changes.added_securities) == (expected_changes_count - 1)
# TODO: shouldn't be sending AAPL as a removed security since it was added by another universe
self.universe_removed = self.universe_removed or (
len(changes.removed_securities) == expected_changes_count and
len(changes.removed_securities) == (expected_changes_count + 1) and
self.utc_time.date() >= self.delisting_date and
self.utc_time.date() < self.end_date.date())

View File

@@ -59,7 +59,7 @@ class ETFConstituentUniverseCompositeDelistingRegressionAlgorithmNoAddEquityETF(
raise AssertionError("New securities added after ETF constituents were delisted")
if self.universe_selection_done:
self.universe_added = self.universe_added or len(changes.added_securities) == self.universe_symbol_count
self.universe_added = self.universe_added or len(changes.added_securities) == self.universe_symbol_count - 1
# TODO: shouldn't be sending AAPL as a removed security since it was added by another universe
self.universe_removed = self.universe_removed or (

View File

@@ -16,6 +16,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Collections.Specialized;
using NodaTime;
using QuantConnect.Algorithm.Selection;
using QuantConnect.Data;
@@ -33,7 +34,7 @@ namespace QuantConnect.Algorithm
// this removes temporal dependencies from w/in initialize method
// original motivation: adding equity/options to enforce equity raw data mode
private readonly object _pendingUniverseAdditionsLock = new object();
private readonly List<UserDefinedUniverseAddition> _pendingUserDefinedUniverseSecurityAdditions = new List<UserDefinedUniverseAddition>();
private readonly List<UserDefinedUniverseUpdate> _pendingUserDefinedUniverseSecurityChanges = new();
private bool _pendingUniverseAdditions;
private ConcurrentSet<Symbol> _rawNormalizationWarningSymbols = new ConcurrentSet<Symbol>();
private readonly int _rawNormalizationWarningSymbolsMaxCount = 10;
@@ -69,7 +70,7 @@ namespace QuantConnect.Algorithm
// rewrite securities w/ derivatives to be in raw mode
lock (_pendingUniverseAdditionsLock)
{
if (!_pendingUniverseAdditions && _pendingUserDefinedUniverseSecurityAdditions.Count == 0)
if (!_pendingUniverseAdditions && _pendingUserDefinedUniverseSecurityChanges.Count == 0)
{
// no point in looping through everything if there's no pending changes
return;
@@ -78,7 +79,7 @@ namespace QuantConnect.Algorithm
var requiredHistoryRequests = new Dictionary<Security, Resolution>();
foreach (var security in Securities.Select(kvp => kvp.Value).Union(
_pendingUserDefinedUniverseSecurityAdditions.Select(x => x.Security)))
_pendingUserDefinedUniverseSecurityChanges.Where(x => x.IsAddition).Select(x => x.Security)))
{
// check for any derivative securities and mark the underlying as raw
if (security.Type == SecurityType.Equity &&
@@ -169,11 +170,26 @@ namespace QuantConnect.Algorithm
}
// add subscriptionDataConfig to their respective user defined universes
foreach (var userDefinedUniverseAddition in _pendingUserDefinedUniverseSecurityAdditions)
foreach (var userDefinedUniverseAddition in _pendingUserDefinedUniverseSecurityChanges)
{
foreach (var subscriptionDataConfig in userDefinedUniverseAddition.SubscriptionDataConfigs)
var changedCollection = false;
var action = NotifyCollectionChangedAction.Add;
if (userDefinedUniverseAddition.IsAddition)
{
userDefinedUniverseAddition.Universe.Add(subscriptionDataConfig);
foreach (var subscriptionDataConfig in userDefinedUniverseAddition.SubscriptionDataConfigs)
{
changedCollection |= userDefinedUniverseAddition.Universe.Add(subscriptionDataConfig);
}
}
else
{
action = NotifyCollectionChangedAction.Replace;
changedCollection |= userDefinedUniverseAddition.Universe.Remove(userDefinedUniverseAddition.Security);
}
if (changedCollection)
{
UniverseManager.Update(userDefinedUniverseAddition.Universe.Symbol, userDefinedUniverseAddition.Universe, action);
}
}
@@ -183,7 +199,7 @@ namespace QuantConnect.Algorithm
UniverseManager.ProcessChanges();
_pendingUniverseAdditions = false;
_pendingUserDefinedUniverseSecurityAdditions.Clear();
_pendingUserDefinedUniverseSecurityChanges.Clear();
}
if (!_rawNormalizationWarningSymbols.IsNullOrEmpty())
@@ -645,8 +661,7 @@ namespace QuantConnect.Algorithm
{
lock (_pendingUniverseAdditionsLock)
{
_pendingUserDefinedUniverseSecurityAdditions.Add(
new UserDefinedUniverseAddition(userDefinedUniverse, configurations, security));
_pendingUserDefinedUniverseSecurityChanges.Add(new UserDefinedUniverseUpdate(userDefinedUniverse, configurations, security));
}
}
else
@@ -743,13 +758,14 @@ namespace QuantConnect.Algorithm
/// Helper class used to store <see cref="UserDefinedUniverse"/> additions.
/// They will be consumed at <see cref="OnEndOfTimeStep"/>
/// </summary>
private class UserDefinedUniverseAddition
private class UserDefinedUniverseUpdate
{
public bool IsAddition => SubscriptionDataConfigs != null;
public Security Security { get; }
public UserDefinedUniverse Universe { get; }
public List<SubscriptionDataConfig> SubscriptionDataConfigs { get; }
public UserDefinedUniverseAddition(
public UserDefinedUniverseUpdate(
UserDefinedUniverse universe,
List<SubscriptionDataConfig> subscriptionDataConfigs,
Security security)

View File

@@ -2467,9 +2467,9 @@ namespace QuantConnect.Algorithm
var optionUniverse = universe as OptionContractUniverse;
if (optionUniverse != null)
{
foreach (var subscriptionDataConfig in configs.Concat(underlyingConfigs))
lock (_pendingUniverseAdditionsLock)
{
optionUniverse.Add(subscriptionDataConfig);
_pendingUserDefinedUniverseSecurityChanges.Add(new UserDefinedUniverseUpdate(optionUniverse, [.. configs, .. underlyingConfigs], option));
}
}
@@ -2632,14 +2632,15 @@ namespace QuantConnect.Algorithm
{
lock (_pendingUniverseAdditionsLock)
{
// for existing universes we need to purge pending additions too, also handled at OnEndOfTimeStep()
_pendingUserDefinedUniverseSecurityChanges.RemoveAll(addition => addition.Security.Symbol == symbol);
// we need to handle existing universes and pending to be added universes, that will be pushed
// at the end of this time step see OnEndOfTimeStep()
foreach (var universe in UniverseManager.Select(x => x.Value).OfType<UserDefinedUniverse>())
foreach (var universe in UniverseManager.Where(x => x.Value.ContainsMember(security)).Select(x => x.Value).OfType<UserDefinedUniverse>())
{
universe.Remove(symbol);
_pendingUserDefinedUniverseSecurityChanges.Add(new UserDefinedUniverseUpdate(universe, null, security));
}
// for existing universes we need to purge pending additions too, also handled at OnEndOfTimeStep()
_pendingUserDefinedUniverseSecurityAdditions.RemoveAll(addition => addition.Security.Symbol == symbol);
}
}
return true;

View File

@@ -27,6 +27,7 @@ using QuantConnect.Orders.Fees;
using QuantConnect.Securities;
using QuantConnect.Securities.Option;
using QuantConnect.Util;
using System.Collections.Specialized;
namespace QuantConnect.Brokerages.Backtesting
{
@@ -553,10 +554,12 @@ namespace QuantConnect.Brokerages.Backtesting
var universe = ukvp.Value;
if (universe.ContainsMember(security.Symbol))
{
var userUniverse = universe as UserDefinedUniverse;
if (userUniverse != null)
if (universe is UserDefinedUniverse userUniverse)
{
userUniverse.Remove(security.Symbol);
if (userUniverse.Remove(security.Symbol))
{
Algorithm.UniverseManager.Update(userUniverse.Symbol, userUniverse, NotifyCollectionChangedAction.Replace);
}
}
else
{
@@ -564,6 +567,7 @@ namespace QuantConnect.Brokerages.Backtesting
}
}
}
Algorithm.UniverseManager.ProcessChanges();
if (!Algorithm.IsWarmingUp)
{

View File

@@ -1,4 +1,4 @@
/*
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
@@ -160,6 +160,14 @@ namespace QuantConnect.Data
return !config.IsCustomData && !config.Symbol.Value.Contains("UNIVERSE") && config.SecurityType == SecurityType.Equity;
}
/// <summary>
/// True if this configuration is associated with an asset which can have delisting events
/// </summary>
public static bool CanBeDelisted(this SubscriptionDataConfig config)
{
return config.SecurityType.IsOption() || config.SecurityType == SecurityType.Future || config.SecurityType == SecurityType.Equity;
}
/// <summary>
/// Initializes a new instance of the <see cref="BaseData"/> type defined in <paramref name="config"/> with the symbol properly set
/// </summary>

View File

@@ -27,15 +27,15 @@ namespace QuantConnect.Securities
/// <summary>
/// Manages the algorithm's collection of universes
/// </summary>
public class UniverseManager : IDictionary<Symbol, Universe>, INotifyCollectionChanged
public class UniverseManager : IDictionary<Symbol, Universe>
{
private readonly Queue<NotifyCollectionChangedEventArgs> _pendingChanges = new();
private readonly Queue<UniverseManagerChanged> _pendingChanges = new();
private readonly ConcurrentDictionary<Symbol, Universe> _universes;
/// <summary>
/// Event fired when a universe is added or removed
/// </summary>
public event NotifyCollectionChangedEventHandler CollectionChanged;
public event EventHandler<UniverseManagerChanged> CollectionChanged;
/// <summary>
/// Read-only dictionary containing all active securities. An active security is
@@ -173,9 +173,28 @@ namespace QuantConnect.Securities
{
if (_universes.TryAdd(key, value))
{
lock(_pendingChanges)
lock (_pendingChanges)
{
_pendingChanges.Enqueue(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, value));
_pendingChanges.Enqueue(new UniverseManagerChanged(NotifyCollectionChangedAction.Add, value));
}
}
}
/// <summary>
/// Updates an element with the provided key and value to the <see cref="System.Collections.Generic.IDictionary{TKey, TValue}"/>.
/// </summary>
/// <param name="key">The object to use as the key of the element to add.
/// </param><param name="value">The object to use as the value of the element to add.</param>
/// <exception cref="System.ArgumentNullException"><paramref name="key"/> is null.</exception>
/// <exception cref="System.ArgumentException">An element with the same key already exists in the <see cref="System.Collections.Generic.IDictionary{TKey, TValue}"/>.</exception>
/// <exception cref="System.NotSupportedException">The <see cref="System.Collections.Generic.IDictionary{TKey, TValue}"/> is read-only.</exception>
public void Update(Symbol key, Universe value, NotifyCollectionChangedAction action)
{
if (_universes.ContainsKey(key) && !_pendingChanges.Any(x => x.Value == value))
{
lock (_pendingChanges)
{
_pendingChanges.Enqueue(new UniverseManagerChanged(action, value));
}
}
}
@@ -185,7 +204,7 @@ namespace QuantConnect.Securities
/// </summary>
public void ProcessChanges()
{
NotifyCollectionChangedEventArgs universeChange;
UniverseManagerChanged universeChange;
do
{
lock (_pendingChanges)
@@ -214,7 +233,7 @@ namespace QuantConnect.Securities
if (_universes.TryRemove(key, out universe))
{
universe.Dispose();
OnCollectionChanged(new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, universe));
OnCollectionChanged(new UniverseManagerChanged(NotifyCollectionChangedAction.Remove, universe));
return true;
}
return false;
@@ -287,7 +306,7 @@ namespace QuantConnect.Securities
/// Event invocator for the <see cref="CollectionChanged"/> event
/// </summary>
/// <param name="e"></param>
protected virtual void OnCollectionChanged(NotifyCollectionChangedEventArgs e)
protected virtual void OnCollectionChanged(UniverseManagerChanged e)
{
CollectionChanged?.Invoke(this, e);
}

View File

@@ -0,0 +1,45 @@
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
using System.Collections.Specialized;
using QuantConnect.Data.UniverseSelection;
namespace QuantConnect.Securities
{
/// <summary>
/// Event dto class fired when a universe reports a change
/// </summary>
public class UniverseManagerChanged
{
/// <summary>
/// The action that occurred
/// </summary>
public NotifyCollectionChangedAction Action { get; }
/// <summary>
/// Universe reporting a change
/// </summary>
public Universe Value { get; }
/// <summary>
/// Creates a new instance
/// </summary>
public UniverseManagerChanged(NotifyCollectionChangedAction action, Universe value)
{
Action = action;
Value = value;
}
}
}

View File

@@ -25,7 +25,6 @@ using QuantConnect.Interfaces;
using QuantConnect.Logging;
using QuantConnect.Securities.Future;
using QuantConnect.Util;
using static QuantConnect.Messages;
namespace QuantConnect
{
@@ -336,7 +335,7 @@ namespace QuantConnect
{
throw new ArgumentException(Messages.SecurityIdentifier.PropertiesDoNotMatchAnySecurityType, nameof(properties));
}
_hashCode = unchecked (symbol.GetHashCode() * 397) ^ properties.GetHashCode();
_hashCode = Math.Abs(unchecked (symbol.GetHashCode() * 397) ^ properties.GetHashCode());
_hashCodeSet = true;
}
@@ -1054,7 +1053,7 @@ namespace QuantConnect
{
if (!_hashCodeSet)
{
_hashCode = unchecked(_symbol.GetHashCode() * 397) ^ _properties.GetHashCode();
_hashCode = Math.Abs(unchecked(_symbol.GetHashCode() * 397) ^ _properties.GetHashCode());
_hashCodeSet = true;
}
return _hashCode;

View File

@@ -82,101 +82,104 @@ namespace QuantConnect.Lean.Engine.DataFeeds
// wire ourselves up to receive notifications when universes are added/removed
algorithm.UniverseManager.CollectionChanged += (sender, args) =>
{
var universe = args.Value;
switch (args.Action)
{
case NotifyCollectionChangedAction.Replace:
case NotifyCollectionChangedAction.Add:
foreach (var universe in args.NewItems.OfType<Universe>())
var config = universe.Configuration;
var start = algorithm.UtcTime;
if (algorithm.GetLocked() && args.Action == NotifyCollectionChangedAction.Add && universe is UserDefinedUniverse)
{
var config = universe.Configuration;
var start = algorithm.UtcTime;
var end = algorithm.LiveMode ? Time.EndOfTime
: algorithm.EndDate.ConvertToUtc(algorithm.TimeZone);
Security security;
if (!algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object if it doesn't exist
security = new Security(
_marketHoursDatabase.GetExchangeHours(config),
config,
algorithm.Portfolio.CashBook[algorithm.AccountCurrency],
SymbolProperties.GetDefault(algorithm.AccountCurrency),
algorithm.Portfolio.CashBook,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
}
// Let's adjust the start time to the previous tradable date
// so universe selection always happens right away at the start of the algorithm.
var universeType = universe.GetType();
if (
// We exclude the UserDefinedUniverse because their selection already happens at the algorithm start time.
// For instance, ETFs universe selection depends its first trigger time to be before the equity universe
// (the UserDefinedUniverse), because the ETFs are EndTime-indexed and that would make their first selection
// time to be before the algorithm start time, with the EndTime being the algorithms's start date,
// and both the Equity and the ETFs constituents first selection to happen together.
!universeType.IsAssignableTo(typeof(UserDefinedUniverse)) &&
// We exclude the ScheduledUniverse because it's already scheduled to run at a specific time.
// Adjusting the start time would cause the first selection trigger time to be before the algorithm start time,
// making the selection to be triggered at the first algorithm time, which would be the exact StartDate.
universeType != typeof(ScheduledUniverse))
{
const int maximumLookback = 60;
var loopCount = 0;
var startLocalTime = start.ConvertFromUtc(security.Exchange.TimeZone);
if (universe.UniverseSettings.Schedule.Initialized)
{
do
{
// determine if there's a scheduled selection time at the current start local time date, note that next
// we get the previous day of the first scheduled date we find, so we are sure the data is available to trigger selection
if (universe.UniverseSettings.Schedule.Get(startLocalTime.Date, startLocalTime.Date).Any())
{
break;
}
startLocalTime = startLocalTime.AddDays(-1);
if (++loopCount >= maximumLookback)
{
// fallback to the original, we found none
startLocalTime = algorithm.UtcTime.ConvertFromUtc(security.Exchange.TimeZone);
if (!_sentUniverseScheduleWarning)
{
// just in case
_sentUniverseScheduleWarning = true;
algorithm.Debug($"Warning: Found no valid start time for scheduled universe, will use default");
}
}
} while (loopCount < maximumLookback);
}
startLocalTime = Time.GetStartTimeForTradeBars(security.Exchange.Hours, startLocalTime,
// disable universe selection on extended market hours, for example futures/index options have a sunday pre market we are not interested on
Time.OneDay, 1, extendedMarketHours: false, config.DataTimeZone,
LeanData.UseDailyStrictEndTimes(algorithm.Settings, config.Type, security.Symbol, Time.OneDay, security.Exchange.Hours));
start = startLocalTime.ConvertToUtc(security.Exchange.TimeZone);
}
AddSubscription(
new SubscriptionRequest(true,
universe,
security,
config,
start,
end));
// If it is an add, after initialize, we will set time 1 tick ahead to properly sync data
// with next timeslice, avoid emitting now twice, if it is a remove then we will set time to now
// we do the same in the 'DataManager' when handling FF resolution changes
start = start.AddTicks(1);
}
var end = algorithm.LiveMode ? Time.EndOfTime
: algorithm.EndDate.ConvertToUtc(algorithm.TimeZone);
Security security;
if (!algorithm.Securities.TryGetValue(config.Symbol, out security))
{
// create a canonical security object if it doesn't exist
security = new Security(
_marketHoursDatabase.GetExchangeHours(config),
config,
algorithm.Portfolio.CashBook[algorithm.AccountCurrency],
SymbolProperties.GetDefault(algorithm.AccountCurrency),
algorithm.Portfolio.CashBook,
RegisteredSecurityDataTypesProvider.Null,
new SecurityCache()
);
}
// Let's adjust the start time to the previous tradable date
// so universe selection always happens right away at the start of the algorithm.
var universeType = universe.GetType();
if (
// We exclude the UserDefinedUniverse because their selection already happens at the algorithm start time.
// For instance, ETFs universe selection depends its first trigger time to be before the equity universe
// (the UserDefinedUniverse), because the ETFs are EndTime-indexed and that would make their first selection
// time to be before the algorithm start time, with the EndTime being the algorithms's start date,
// and both the Equity and the ETFs constituents first selection to happen together.
!universeType.IsAssignableTo(typeof(UserDefinedUniverse)) &&
// We exclude the ScheduledUniverse because it's already scheduled to run at a specific time.
// Adjusting the start time would cause the first selection trigger time to be before the algorithm start time,
// making the selection to be triggered at the first algorithm time, which would be the exact StartDate.
universeType != typeof(ScheduledUniverse))
{
const int maximumLookback = 60;
var loopCount = 0;
var startLocalTime = start.ConvertFromUtc(security.Exchange.TimeZone);
if (universe.UniverseSettings.Schedule.Initialized)
{
do
{
// determine if there's a scheduled selection time at the current start local time date, note that next
// we get the previous day of the first scheduled date we find, so we are sure the data is available to trigger selection
if (universe.UniverseSettings.Schedule.Get(startLocalTime.Date, startLocalTime.Date).Any())
{
break;
}
startLocalTime = startLocalTime.AddDays(-1);
if (++loopCount >= maximumLookback)
{
// fallback to the original, we found none
startLocalTime = algorithm.UtcTime.ConvertFromUtc(security.Exchange.TimeZone);
if (!_sentUniverseScheduleWarning)
{
// just in case
_sentUniverseScheduleWarning = true;
algorithm.Debug($"Warning: Found no valid start time for scheduled universe, will use default");
}
}
} while (loopCount < maximumLookback);
}
startLocalTime = Time.GetStartTimeForTradeBars(security.Exchange.Hours, startLocalTime,
// disable universe selection on extended market hours, for example futures/index options have a sunday pre market we are not interested on
Time.OneDay, 1, extendedMarketHours: false, config.DataTimeZone,
LeanData.UseDailyStrictEndTimes(algorithm.Settings, config.Type, security.Symbol, Time.OneDay, security.Exchange.Hours));
start = startLocalTime.ConvertToUtc(security.Exchange.TimeZone);
}
AddSubscription(
new SubscriptionRequest(true,
universe,
security,
config,
start,
end));
break;
case NotifyCollectionChangedAction.Remove:
foreach (var universe in args.OldItems.OfType<Universe>())
// removing the subscription will be handled by the SubscriptionSynchronizer
// in the next loop as well as executing a UniverseSelection one last time.
if (!universe.DisposeRequested)
{
// removing the subscription will be handled by the SubscriptionSynchronizer
// in the next loop as well as executing a UniverseSelection one last time.
if (!universe.DisposeRequested)
{
universe.Dispose();
}
universe.Dispose();
}
break;
@@ -196,7 +199,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
.SelectMany(subscription => subscription.SubscriptionRequests)
.ToList();
if(requests.Count > 0)
if (requests.Count > 0)
{
Log.Trace($"DataManager(): Fill forward resolution has changed from {changedEvent.Old} to {changedEvent.New} at utc: {algorithm.UtcTime}. " +
$"Restarting {requests.Count} subscriptions...");
@@ -274,7 +277,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
{
// guarantee the configuration is present in our config collection
// this is related to GH issue 3877: where we added a configuration which we also removed
if(_subscriptionManagerSubscriptions.TryAdd(request.Configuration, request.Configuration))
if (_subscriptionManagerSubscriptions.TryAdd(request.Configuration, request.Configuration))
{
_subscriptionDataConfigsEnumerator = null;
}
@@ -283,10 +286,14 @@ namespace QuantConnect.Lean.Engine.DataFeeds
Subscription subscription;
if (DataFeedSubscriptions.TryGetValue(request.Configuration, out subscription))
{
// duplicate subscription request
subscription.AddSubscriptionRequest(request);
// only result true if the existing subscription is internal, we actually added something from the users perspective
return subscription.Configuration.IsInternalFeed;
if (!subscription.EndOfStream)
{
// duplicate subscription request
subscription.AddSubscriptionRequest(request);
// only result true if the existing subscription is internal, we actually added something from the users perspective
return subscription.Configuration.IsInternalFeed;
}
DataFeedSubscriptions.TryRemove(request.Configuration, out _);
}
if (request.Configuration.DataNormalizationMode == DataNormalizationMode.ScaledRaw)
@@ -313,7 +320,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
Log.Trace($"DataManager.AddSubscription(): Added {request.Configuration}." +
$" Start: {request.StartTimeUtc}. End: {request.EndTimeUtc}");
}
else if(Log.DebuggingEnabled)
else if (Log.DebuggingEnabled)
{
// for performance lets not create the message string if debugging is not enabled
// this can be executed many times and its in the algorithm thread
@@ -380,7 +387,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
{
Log.Trace($"DataManager.RemoveSubscription(): Removed {configuration}");
}
else if(Log.DebuggingEnabled)
else if (Log.DebuggingEnabled)
{
// for performance lets not create the message string if debugging is not enabled
// this can be executed many times and its in the algorithm thread
@@ -437,7 +444,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
{
lock (_subscriptionManagerSubscriptions)
{
if(_subscriptionDataConfigsEnumerator == null)
if (_subscriptionDataConfigsEnumerator == null)
{
_subscriptionDataConfigsEnumerator = _subscriptionManagerSubscriptions.Values.ToList();
}
@@ -539,7 +546,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
)
{
return Add(symbol, resolution, fillForward, extendedMarketHours, isFilteredSubscription, isInternalFeed, isCustomData,
new List<Tuple<Type, TickType>> { new Tuple<Type, TickType>(dataType, LeanData.GetCommonTickTypeForCommonDataTypes(dataType, symbol.SecurityType))},
new List<Tuple<Type, TickType>> { new Tuple<Type, TickType>(dataType, LeanData.GetCommonTickTypeForCommonDataTypes(dataType, symbol.SecurityType)) },
dataNormalizationMode, dataMappingMode, contractDepthOffset)
.First();
}

View File

@@ -70,7 +70,10 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
tradableEventProviders.Add(new MappingEventProvider());
}
tradableEventProviders.Add(new DelistingEventProvider());
if (config.CanBeDelisted())
{
tradableEventProviders.Add(new DelistingEventProvider());
}
var enumerator = new AuxiliaryDataEnumerator(
config,

View File

@@ -14,10 +14,7 @@
*
*/
using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Linq;
using QuantConnect.Data;
using QuantConnect.Data.Market;
@@ -34,7 +31,6 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
/// </summary>
public class TimeTriggeredUniverseSubscriptionEnumeratorFactory : ISubscriptionEnumeratorFactory
{
private readonly ITimeProvider _timeProvider;
private readonly ITimeTriggeredUniverse _universe;
private readonly MarketHoursDatabase _marketHoursDatabase;
@@ -43,11 +39,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
/// </summary>
/// <param name="universe">The user defined universe</param>
/// <param name="marketHoursDatabase">The market hours database</param>
/// <param name="timeProvider">The time provider</param>
public TimeTriggeredUniverseSubscriptionEnumeratorFactory(ITimeTriggeredUniverse universe, MarketHoursDatabase marketHoursDatabase, ITimeProvider timeProvider)
public TimeTriggeredUniverseSubscriptionEnumeratorFactory(ITimeTriggeredUniverse universe, MarketHoursDatabase marketHoursDatabase)
{
_universe = universe;
_timeProvider = timeProvider;
_marketHoursDatabase = marketHoursDatabase;
}
@@ -59,105 +53,9 @@ namespace QuantConnect.Lean.Engine.DataFeeds.Enumerators.Factories
/// <returns>An enumerator reading the subscription request</returns>
public IEnumerator<BaseData> CreateEnumerator(SubscriptionRequest request, IDataProvider dataProvider)
{
var enumerator = (IEnumerator<BaseData>) _universe.GetTriggerTimes(request.StartTimeUtc, request.EndTimeUtc, _marketHoursDatabase)
return _universe.GetTriggerTimes(request.StartTimeUtc, request.EndTimeUtc, _marketHoursDatabase)
.Select(x => new Tick { Time = x, Symbol = request.Configuration.Symbol })
.GetEnumerator();
var universe = request.Universe as UserDefinedUniverse;
if (universe != null)
{
enumerator = new InjectionEnumerator(enumerator);
// Trigger universe selection when security added/removed after Initialize
universe.CollectionChanged += (sender, args) =>
{
// If it is an add we will set time 1 tick ahead to properly sync data
// with next timeslice, avoid emitting now twice, if it is a remove then we will set time to now
// we do the same in the 'DataManager' when handling FF resolution changes
IList items;
DateTime time;
if (args.Action == NotifyCollectionChangedAction.Add)
{
items = args.NewItems;
time = _timeProvider.GetUtcNow().AddTicks(1);
}
else if (args.Action == NotifyCollectionChangedAction.Remove)
{
items = args.OldItems;
time = _timeProvider.GetUtcNow();
}
else
{
items = null;
time = DateTime.MinValue;
}
// Check that we have our items and time
if (items == null || time == DateTime.MinValue) return;
var symbol = items.OfType<Symbol>().FirstOrDefault();
if(symbol == null) return;
// the data point time should always be in exchange timezone
time = time.ConvertFromUtc(request.Configuration.ExchangeTimeZone);
var collection = new BaseDataCollection(time, symbol);
((InjectionEnumerator) enumerator).InjectDataPoint(collection);
};
}
return enumerator;
}
private class InjectionEnumerator : IEnumerator<BaseData>
{
private volatile bool _wasInjected;
private readonly IEnumerator<BaseData> _underlyingEnumerator;
public BaseData Current { get; private set; }
object IEnumerator.Current => Current;
public InjectionEnumerator(IEnumerator<BaseData> underlyingEnumerator)
{
_underlyingEnumerator = underlyingEnumerator;
}
public void InjectDataPoint(BaseData baseData)
{
// we use a lock because the main algorithm thread is the one injecting and the base exchange is the thread pulling MoveNext()
lock (_underlyingEnumerator)
{
_wasInjected = true;
Current = baseData;
}
}
public void Dispose()
{
_underlyingEnumerator.Dispose();
}
public bool MoveNext()
{
lock (_underlyingEnumerator)
{
if (_wasInjected)
{
_wasInjected = false;
return true;
}
_underlyingEnumerator.MoveNext();
Current = _underlyingEnumerator.Current;
return true;
}
}
public void Reset()
{
_underlyingEnumerator.Reset();
}
}
}
}

View File

@@ -162,12 +162,6 @@ namespace QuantConnect.Lean.Engine.DataFeeds
enumerator = AddScheduleWrapper(request, enumerator, null);
if (request.IsUniverseSubscription && request.Universe is UserDefinedUniverse)
{
// for user defined universe we do not use a worker task, since calls to AddData can happen in any moment
// and we have to be able to inject selection data points into the enumerator
return SubscriptionUtils.Create(request, enumerator, _algorithm.Settings.DailyPreciseEndTime);
}
return SubscriptionUtils.CreateAndScheduleWorker(request, enumerator, _factorFileProvider, true, _algorithm.Settings.DailyPreciseEndTime);
}
@@ -187,9 +181,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
ISubscriptionEnumeratorFactory factory = _subscriptionFactory;
if (request.Universe is ITimeTriggeredUniverse)
{
factory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse,
_marketHoursDatabase,
_timeProvider);
factory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(request.Universe as ITimeTriggeredUniverse, _marketHoursDatabase);
}
else if (request.Configuration.Type == typeof(FundamentalUniverse))
{

View File

@@ -328,21 +328,21 @@ namespace QuantConnect.Lean.Engine.DataFeeds
var tzOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);
IEnumerator<BaseData> enumerator = null;
var timeTriggered = request.Universe as ITimeTriggeredUniverse;
if (timeTriggered != null)
if (request.Universe is ITimeTriggeredUniverse timeTriggered)
{
Log.Trace($"LiveTradingDataFeed.CreateUniverseSubscription(): Creating user defined universe: {config.Symbol.ID}");
// spoof a tick on the requested interval to trigger the universe selection function
var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder(), _frontierTimeProvider);
var enumeratorFactory = new TimeTriggeredUniverseSubscriptionEnumeratorFactory(timeTriggered, MarketHoursDatabase.FromDataFolder());
enumerator = enumeratorFactory.CreateEnumerator(request, _dataProvider);
enumerator = new FrontierAwareEnumerator(enumerator, _timeProvider, tzOffsetProvider);
var enqueueable = new EnqueueableEnumerator<BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
if (request.Universe is not UserDefinedUniverse)
{
var enqueueable = new EnqueueableEnumerator<BaseData>();
_customExchange.AddEnumerator(new EnumeratorHandler(config.Symbol, enumerator, enqueueable));
enumerator = enqueueable;
}
}
else if (config.Type.IsAssignableTo(typeof(ETFConstituentUniverse)) ||
config.Type.IsAssignableTo(typeof(FundamentalUniverse)) ||

View File

@@ -134,6 +134,11 @@ namespace QuantConnect.Lean.Engine.DataFeeds
if (IsUniverseSelectionSubscription
|| subscriptionRequest.IsUniverseSubscription)
{
if (subscriptionRequest.Universe is UserDefinedUniverse)
{
// for different reasons a user defined universe can trigger a subscription request, likes additions/removals
return false;
}
throw new Exception("Subscription.AddSubscriptionRequest(): Universe selection" +
" subscriptions should not have more than 1 SubscriptionRequest");
}

View File

@@ -1,4 +1,4 @@
/*
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
@@ -70,8 +70,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
// will add new universe selection data points when is has too
// so lets move it next to check if there is any
subscription.Current == null
&& subscription.IsUniverseSelectionSubscription
&& subscription.UtcStartTime != _utcNow)
&& subscription.IsUniverseSelectionSubscription)
{
subscription.MoveNext();
}

View File

@@ -82,7 +82,7 @@ namespace QuantConnect.Lean.Engine.DataFeeds
}
var exchangeHours = request.Security.Exchange.Hours;
var enqueueable = new EnqueueableEnumerator<SubscriptionData>(true);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.Configuration.ExchangeTimeZone, request.StartTimeUtc, request.EndTimeUtc);
var timeZoneOffsetProvider = new TimeZoneOffsetProvider(request.ExchangeHours.TimeZone, request.StartTimeUtc, request.EndTimeUtc);
var subscription = new Subscription(request, enqueueable, timeZoneOffsetProvider);
var config = subscription.Configuration;
enablePriceScale = enablePriceScale && config.PricesShouldBeScaled();

View File

@@ -1,4 +1,4 @@
/*
/*
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
*
@@ -38,7 +38,7 @@ namespace QuantConnect.Tests.Common.Securities
manager.CollectionChanged += (sender, args) =>
{
if (args.NewItems.OfType<object>().Single() != universe)
if (args.Value != universe)
{
Assert.Fail("Expected args.NewItems to have exactly one element equal to universe");
}
@@ -63,7 +63,7 @@ namespace QuantConnect.Tests.Common.Securities
manager.CollectionChanged += (sender, args) =>
{
if (args.NewItems.OfType<object>().Single() != universe)
if (args.Value != universe)
{
Assert.Fail("Expected args.NewItems to have exactly one element equal to universe");
}
@@ -89,7 +89,7 @@ namespace QuantConnect.Tests.Common.Securities
manager.Add(universe.Configuration.Symbol, universe);
manager.CollectionChanged += (sender, args) =>
{
if (args.OldItems.OfType<object>().Single() != universe)
if (args.Value != universe)
{
Assert.Fail("Expected args.OldItems to have exactly one element equal to universe");
}

View File

@@ -71,27 +71,27 @@ class CustomBrokerageMessageHandler(DefaultBrokerageMessageHandler):
{PerformanceMetrics.TotalOrders, expectedOrdersCount.ToStringInvariant()},
{"Average Win", "0%"},
{"Average Loss", "0%"},
{"Compounding Annual Return", "-10.771%"},
{"Compounding Annual Return", "-11.597%"},
{"Drawdown", "0.200%"},
{"Expectancy", "0"},
{"Net Profit", "-0.146%"},
{"Sharpe Ratio", "-5.186"},
{"Sortino Ratio", "-6.53"},
{"Net Profit", "-0.157%"},
{"Sharpe Ratio", "-5.199"},
{"Sortino Ratio", "-6.546"},
{"Probabilistic Sharpe Ratio", "24.692%"},
{"Loss Rate", "0%"},
{"Win Rate", "0%"},
{"Profit-Loss Ratio", "0"},
{"Alpha", "0.059"},
{"Beta", "-0.072"},
{"Alpha", "0.058"},
{"Beta", "-0.07"},
{"Annual Standard Deviation", "0.016"},
{"Annual Variance", "0"},
{"Information Ratio", "-8.629"},
{"Tracking Error", "0.239"},
{"Treynor Ratio", "1.154"},
{"Total Fees", "$50.00"},
{"Estimated Strategy Capacity", "$17000000.00"},
{"Information Ratio", "-8.635"},
{"Tracking Error", "0.238"},
{"Treynor Ratio", "1.157"},
{"Total Fees", "$49.00"},
{"Estimated Strategy Capacity", "$26000000.00"},
{"Lowest Capacity Asset", "SPY R735QTJ8XC9X"},
{"Portfolio Turnover", "1.45%"}
{"Portfolio Turnover", "1.42%"}
},
language,
AlgorithmStatus.Completed);
@@ -114,27 +114,27 @@ class CustomBrokerageMessageHandler(DefaultBrokerageMessageHandler):
{PerformanceMetrics.TotalOrders, expectedOrdersCount.ToStringInvariant()},
{"Average Win", "0%"},
{"Average Loss", "0%"},
{"Compounding Annual Return", "-10.771%"},
{"Compounding Annual Return", "-11.597%"},
{"Drawdown", "0.200%"},
{"Expectancy", "0"},
{"Net Profit", "-0.146%"},
{"Sharpe Ratio", "-5.186"},
{"Sortino Ratio", "-6.53"},
{"Net Profit", "-0.157%"},
{"Sharpe Ratio", "-5.199"},
{"Sortino Ratio", "-6.546"},
{"Probabilistic Sharpe Ratio", "24.692%"},
{"Loss Rate", "0%"},
{"Win Rate", "0%"},
{"Profit-Loss Ratio", "0"},
{"Alpha", "0.059"},
{"Beta", "-0.072"},
{"Alpha", "0.058"},
{"Beta", "-0.07"},
{"Annual Standard Deviation", "0.016"},
{"Annual Variance", "0"},
{"Information Ratio", "-8.629"},
{"Tracking Error", "0.239"},
{"Treynor Ratio", "1.154"},
{"Total Fees", "$50.00"},
{"Estimated Strategy Capacity", "$17000000.00"},
{"Information Ratio", "-8.635"},
{"Tracking Error", "0.238"},
{"Treynor Ratio", "1.157"},
{"Total Fees", "$49.00"},
{"Estimated Strategy Capacity", "$26000000.00"},
{"Lowest Capacity Asset", "SPY R735QTJ8XC9X"},
{"Portfolio Turnover", "1.45%"}
{"Portfolio Turnover", "1.42%"}
},
language,
AlgorithmStatus.Completed);

View File

@@ -1134,7 +1134,7 @@ namespace QuantConnect.Tests.Engine.DataFeeds
}
},
endDate: endDate,
secondsTimeStep: 60);
secondsTimeStep: 5);
Assert.IsTrue(emittedData);
}