Remove dates present in IB generic entry part 6 (#9197)

* Remove dates present in generic entries

* Remove dates present in IB generic entry

AlgorithmFactory/Loader.cs

@@ -19,7 +19,6 @@ using System.IO;
 using System.Linq;
 using System.Reflection;
 using System.Runtime.InteropServices;
-using System.Runtime.Loader;
 using System.Security.Policy;
 using Python.Runtime;
 using QuantConnect.Interfaces;
@@ -180,11 +179,9 @@ namespace QuantConnect.AlgorithmFactory
                     {
                         PythonEngine.Exec(
                             @"
-from logging import captureWarnings
-from os import devnull
-from sys import stdout
-stdout = open(devnull, 'w')
-captureWarnings(True)"
+import logging, os, sys
+sys.stdout = open(os.devnull, 'w')
+logging.captureWarnings(True)"
                         );
                     }
                 }
@@ -214,14 +211,40 @@ captureWarnings(True)"
 
             try
             {
+                byte[] debugInformationBytes = null;
+
                 // if the assembly is located in the base directory then don't bother loading the pdbs
                 // manually, they'll be loaded automatically by the .NET runtime.
-                var assemblyFile = new FileInfo(assemblyPath);
-                var directoryName = assemblyFile.DirectoryName;
+                var directoryName = new FileInfo(assemblyPath).DirectoryName;
+                if (directoryName != null && directoryName.TrimEnd(Path.DirectorySeparatorChar) != AppDomain.CurrentDomain.BaseDirectory.TrimEnd(Path.DirectorySeparatorChar))
+                {
+                    // see if the pdb exists
+                    var mdbFilename = assemblyPath + ".mdb";
+                    var pdbFilename = assemblyPath.Substring(0, assemblyPath.Length - 4) + ".pdb";
+                    if (File.Exists(pdbFilename))
+                    {
+                        debugInformationBytes = File.ReadAllBytes(pdbFilename);
+                    }
+                    // see if the mdb exists
+                    if (File.Exists(mdbFilename))
+                    {
+                        debugInformationBytes = File.ReadAllBytes(mdbFilename);
+                    }
+                }
 
                 //Load the assembly:
-                var loader = new PluginLoadContext(directoryName);
-                var assembly = loader.LoadFromAssemblyPath(assemblyFile.FullName);
+                Assembly assembly;
+                if (debugInformationBytes == null)
+                {
+                    Log.Trace("Loader.TryCreateILAlgorithm(): Loading only the algorithm assembly");
+                    assembly = Assembly.LoadFrom(assemblyPath);
+                }
+                else
+                {
+                    Log.Trace("Loader.TryCreateILAlgorithm(): Loading debug information with algorithm");
+                    var assemblyBytes = File.ReadAllBytes(assemblyPath);
+                    assembly = Assembly.Load(assemblyBytes, debugInformationBytes);
+                }
 
                 //Get the list of extention classes in the library:
                 var types = GetExtendedTypeNames(assembly);
@@ -254,6 +277,7 @@ captureWarnings(True)"
                 {
                     Log.Trace("Loader.TryCreateILAlgorithm(): Loaded " + algorithmInstance.GetType().Name);
                 }
+
             }
             catch (ReflectionTypeLoadException err)
             {
@@ -278,7 +302,7 @@ captureWarnings(True)"
         /// <returns>String list of types available.</returns>
         public static List<string> GetExtendedTypeNames(Assembly assembly)
         {
-            List<string> types;
+            var types = new List<string>();
             try
             {
                 Type[] assemblyTypes;
@@ -306,22 +330,20 @@ captureWarnings(True)"
                 {
                     types = (from t in assemblyTypes
                              where t.IsClass                                    // require class
-                             && !t.IsAbstract                                // require concrete impl
-                             && AlgorithmInterfaceType.IsAssignableFrom(t)   // require derived from IAlgorithm
-                             && t.FullName != AlgorithmBaseTypeFullName      // require not equal to QuantConnect.QCAlgorithm
-                             && t.FullName != FrameworkBaseTypeFullName      // require not equal to QuantConnect.QCAlgorithmFramework
-                             && t.GetConstructor(Type.EmptyTypes) != null    // require default ctor
+                             where !t.IsAbstract                                // require concrete impl
+                             where AlgorithmInterfaceType.IsAssignableFrom(t)   // require derived from IAlgorithm
+                             where t.FullName != AlgorithmBaseTypeFullName      // require not equal to QuantConnect.QCAlgorithm
+                             where t.FullName != FrameworkBaseTypeFullName      // require not equal to QuantConnect.QCAlgorithmFramework
+                             where t.GetConstructor(Type.EmptyTypes) != null    // require default ctor
                              select t.FullName).ToList();
                 }
                 else
                 {
-                    types = [];
                     Log.Error("API.GetExtendedTypeNames(): No types found in assembly.");
                 }
             }
             catch (Exception err)
             {
-                types = [];
                 Log.Error(err);
             }
 
@@ -375,37 +397,6 @@ captureWarnings(True)"
             }
         }
 
-        class PluginLoadContext : AssemblyLoadContext
-        {
-            private readonly AssemblyDependencyResolver _resolver;
-
-            public PluginLoadContext(string pluginPath)
-            {
-                _resolver = new AssemblyDependencyResolver(pluginPath);
-                Resolving += (context, assemblyName) =>
-                {
-                    var path = Path.Combine(Composer.PluginDirectory, $"{assemblyName.Name}.dll");
-                    try
-                    {
-                        return context.LoadFromAssemblyPath(path);
-                    }
-                    catch
-                    {
-                        return null;
-                    }
-                };
-            }
-
-            protected override Assembly Load(AssemblyName assemblyName)
-            {
-                var path = _resolver.ResolveAssemblyToPath(assemblyName);
-                if (path != null)
-                {
-                    return LoadFromAssemblyPath(path);
-                }
-                return null;
-            }
-        }
-} // End Algorithm Factory Class
+    } // End Algorithm Factory Class
 
 } // End QC Namespace.

Common/Brokerages/dYdXBrokerageModel.cs

@@ -43,6 +43,21 @@ public class dYdXBrokerageModel : DefaultBrokerageModel
         }
     }
 
+    /// <summary>
+    /// Gets a new buying power model for the security, returning the default model with the security's configured leverage.
+    /// For cash accounts, leverage = 1 is used.
+    /// </summary>
+    /// <param name="security">The security to get a buying power model for</param>
+    /// <returns>The buying power model for this brokerage/security</returns>
+    public override IBuyingPowerModel GetBuyingPowerModel(Security security)
+    {
+        return security?.Type switch
+        {
+            SecurityType.CryptoFuture => new SecurityMarginModel(GetLeverage(security)),
+            _ => base.GetBuyingPowerModel(security)
+        };
+    }
+
     /// <summary>
     /// Provides dYdX fee model
     /// </summary>
@@ -145,4 +160,4 @@ public class dYdXBrokerageModel : DefaultBrokerageModel
         map[SecurityType.CryptoFuture] = marketName;
         return map.ToReadOnlyDictionary();
     }
-}
+}

Common/Securities/MarketHoursDatabase.cs

@@ -31,7 +31,7 @@ namespace QuantConnect.Securities
     /// Provides access to exchange hours and raw data times zones in various markets
     /// </summary>
     [JsonConverter(typeof(MarketHoursDatabaseJsonConverter))]
-    public class MarketHoursDatabase : BaseSecurityDatabase<MarketHoursDatabase, Lazy<MarketHoursDatabase.Entry>>
+    public class MarketHoursDatabase : BaseSecurityDatabase<MarketHoursDatabase, MarketHoursDatabase.Entry>
     {
         private readonly bool _forceExchangeAlwaysOpen = Config.GetBool("force-exchange-always-open");
 
@@ -40,7 +40,7 @@ namespace QuantConnect.Securities
         /// <summary>
         /// Gets all the exchange hours held by this provider
         /// </summary>
-        public List<KeyValuePair<SecurityDatabaseKey, Entry>> ExchangeHoursListing => Entries.ToList(x => new KeyValuePair<SecurityDatabaseKey, Entry>(x.Key, x.Value.Value));
+        public List<KeyValuePair<SecurityDatabaseKey, Entry>> ExchangeHoursListing => Entries.ToList();
 
         /// <summary>
         /// Gets a <see cref="MarketHoursDatabase"/> that always returns <see cref="SecurityExchangeHours.AlwaysOpen"/>
@@ -62,7 +62,7 @@ namespace QuantConnect.Securities
         /// Initializes a new instance of the <see cref="MarketHoursDatabase"/> class
         /// </summary>
         private MarketHoursDatabase()
-            : this(new Dictionary<SecurityDatabaseKey, Entry>())
+            : this(new())
         {
         }
 
@@ -71,16 +71,7 @@ namespace QuantConnect.Securities
         /// </summary>
         /// <param name="exchangeHours">The full listing of exchange hours by key</param>
         public MarketHoursDatabase(Dictionary<SecurityDatabaseKey, Entry> exchangeHours)
-            : this(exchangeHours.ToDictionary(kvp => kvp.Key, kvp => new Lazy<Entry>(kvp.Value)))
-        {
-        }
-
-        /// <summary>
-        /// Initializes a new instance of the <see cref="MarketHoursDatabase"/> class
-        /// </summary>
-        /// <param name="exchangeHours">The full listing of exchange hours by key</param>
-        public MarketHoursDatabase(Dictionary<SecurityDatabaseKey, Lazy<Entry>> exchangeHours)
-            : base(exchangeHours, FromDataFolder, (entry, other) => entry.Value.Update(other.Value))
+            : base(exchangeHours, FromDataFolder, (entry, other) => entry.Update(other))
         {
         }
 
@@ -148,9 +139,7 @@ namespace QuantConnect.Securities
         /// <returns>A new instance of the <see cref="MarketHoursDatabase"/> class</returns>
         public static MarketHoursDatabase FromFile(string path)
         {
-            using var stream = File.OpenRead(path);
-            var result = Extensions.DeserializeJson<MarketHoursDatabaseJsonConverter.MarketHoursDatabaseJson>(stream);
-            return result.Convert();
+            return JsonConvert.DeserializeObject<MarketHoursDatabase>(File.ReadAllText(path));
         }
 
         /// <summary>
@@ -172,7 +161,7 @@ namespace QuantConnect.Securities
             var entry = new Entry(dataTimeZone, exchangeHours);
             lock (DataFolderDatabaseLock)
             {
-                Entries[key] = new Lazy<Entry>(entry);
+                Entries[key] = entry;
                 CustomEntries.Add(key);
             }
             return entry;
@@ -263,22 +252,16 @@ namespace QuantConnect.Securities
 
         private bool TryGetEntryImpl(string market, string symbol, SecurityType securityType, out Entry entry)
         {
-            entry = null;
             var symbolKey = new SecurityDatabaseKey(market, symbol, securityType);
-            if (Entries.TryGetValue(symbolKey, out var lazyEntry)
+            return Entries.TryGetValue(symbolKey, out entry)
                 // now check with null symbol key
-                || Entries.TryGetValue(symbolKey.CreateCommonKey(), out lazyEntry)
+                || Entries.TryGetValue(symbolKey.CreateCommonKey(), out entry)
                 // if FOP check for future
                 || securityType == SecurityType.FutureOption && TryGetEntry(market,
                     FuturesOptionsSymbolMappings.MapFromOption(symbol), SecurityType.Future, out entry)
                 // if custom data type check for type specific entry
                 || (securityType == SecurityType.Base && SecurityIdentifier.TryGetCustomDataType(symbol, out var customType)
-                    && Entries.TryGetValue(new SecurityDatabaseKey(market, $"TYPE.{customType}", securityType), out lazyEntry)))
-            {
-                entry ??= lazyEntry.Value;
-                return true;
-            }
-            return false;
+                    && Entries.TryGetValue(new SecurityDatabaseKey(market, $"TYPE.{customType}", securityType), out entry));
         }
 
         /// <summary>

Common/Util/Composer.cs

@@ -37,19 +37,11 @@ namespace QuantConnect.Util
     /// </summary>
     public class Composer
     {
-        /// <summary>
-        /// The plugin directory source if any
-        /// </summary>
-        public static string PluginDirectory { get; private set; }
-
+        private static string PluginDirectory;
         private static readonly Lazy<Composer> LazyComposer = new Lazy<Composer>(
             () =>
             {
-                var pluginDirectory = Config.Get("plugin-directory");
-                if (!string.IsNullOrEmpty(pluginDirectory))
-                {
-                    PluginDirectory = new DirectoryInfo(pluginDirectory).FullName;
-                }
+                PluginDirectory = Config.Get("plugin-directory");
                 return new Composer();
             });
 
@@ -95,7 +87,7 @@ namespace QuantConnect.Util
 
             var loadFromPluginDir = !string.IsNullOrWhiteSpace(PluginDirectory)
                 && Directory.Exists(PluginDirectory) &&
-                PluginDirectory != primaryDllLookupDirectory;
+                new DirectoryInfo(PluginDirectory).FullName != primaryDllLookupDirectory;
             var fileNames = Directory.EnumerateFiles(primaryDllLookupDirectory, "*.dll");
             if (loadFromPluginDir)
             {
@@ -362,11 +354,6 @@ namespace QuantConnect.Util
                 var catalogs = new ConcurrentBag<ComposablePartCatalog>();
                 Parallel.ForEach(files, file =>
                 {
-                    if (!Path.GetFileName(file).StartsWith($"{nameof(QuantConnect)}.", StringComparison.InvariantCulture))
-                    {
-                        return;
-                    }
-
                     try
                     {
                         // we need to load assemblies so that C# algorithm dependencies are resolved correctly
@@ -383,9 +370,12 @@ namespace QuantConnect.Util
                             assembly = Assembly.LoadFrom(file);
                         }
 
-                        foreach (var type in assembly.ExportedTypes.Where(type => !type.IsAbstract && !type.IsInterface && !type.IsEnum))
+                        if (Path.GetFileName(file).StartsWith($"{nameof(QuantConnect)}.", StringComparison.InvariantCulture))
                         {
-                            exportedTypes.Add(type);
+                            foreach (var type in assembly.ExportedTypes.Where(type => !type.IsAbstract && !type.IsInterface && !type.IsEnum))
+                            {
+                                exportedTypes.Add(type);
+                            }
                         }
                         var asmCatalog = new AssemblyCatalog(assembly);
                         var parts = asmCatalog.Parts.ToArray();

Common/Util/MarketHoursDatabaseJsonConverter.cs

@@ -98,23 +98,33 @@ namespace QuantConnect.Util
             /// <returns>A new instance of the <see cref="MarketHoursDatabase"/> class</returns>
             public MarketHoursDatabase Convert()
             {
-                var result = new Dictionary<SecurityDatabaseKey, Lazy<MarketHoursDatabase.Entry>>(Entries.Count);
-                foreach (var kvp in Entries)
+                // first we parse the entries keys so that later we can sort by security type
+                var entries = new Dictionary<SecurityDatabaseKey, MarketHoursDatabaseEntryJson>(Entries.Count);
+                foreach (var entry in Entries)
                 {
                     try
                     {
-                        var key = SecurityDatabaseKey.Parse(kvp.Key);
-                        result[key] = new Lazy<MarketHoursDatabase.Entry>(() =>
+                        var key = SecurityDatabaseKey.Parse(entry.Key);
+                        if (key != null)
                         {
-                            MarketHoursDatabase.Entry marketEntry = null;
-                            if (key.Symbol != SecurityDatabaseKey.Wildcard)
-                            {
-                                result.TryGetValue(key.CreateCommonKey(), out var marketEntryLazy);
-                                marketEntry = marketEntryLazy?.Value;
-                            }
-                            var underlyingEntry = GetUnderlyingEntry(key, result);
-                            return kvp.Value.Convert(underlyingEntry, marketEntry);
-                        });
+                            entries[key] = entry.Value;
+                        }
+                    }
+                    catch (Exception err)
+                    {
+                        Log.Error(err);
+                    }
+                }
+
+                var result = new Dictionary<SecurityDatabaseKey, MarketHoursDatabase.Entry>(Entries.Count);
+                // we sort so we process generic entries and non options first
+                foreach (var entry in entries.OrderBy(kvp => kvp.Key.Symbol != null ? 1 : 0).ThenBy(kvp => kvp.Key.SecurityType.IsOption() ? 1 : 0))
+                {
+                    try
+                    {
+                        result.TryGetValue(entry.Key.CreateCommonKey(), out var marketEntry);
+                        var underlyingEntry = GetUnderlyingEntry(entry.Key, result);
+                        result[entry.Key] = entry.Value.Convert(underlyingEntry, marketEntry);
                     }
                     catch (Exception err)
                     {
@@ -127,9 +137,9 @@ namespace QuantConnect.Util
             /// <summary>
             /// Helper method to get the already processed underlying entry for options
             /// </summary>
-            private static MarketHoursDatabase.Entry GetUnderlyingEntry(SecurityDatabaseKey key, Dictionary<SecurityDatabaseKey, Lazy<MarketHoursDatabase.Entry>> result)
+            private static MarketHoursDatabase.Entry GetUnderlyingEntry(SecurityDatabaseKey key, Dictionary<SecurityDatabaseKey, MarketHoursDatabase.Entry> result)
             {
-                Lazy<MarketHoursDatabase.Entry> underlyingEntryLazy = null;
+                MarketHoursDatabase.Entry underlyingEntry = null;
                 if (key.SecurityType.IsOption())
                 {
                     // if option, let's get the underlyings entry
@@ -137,15 +147,15 @@ namespace QuantConnect.Util
                     var underlying = OptionSymbol.MapToUnderlying(key.Symbol, key.SecurityType);
                     var underlyingKey = new SecurityDatabaseKey(key.Market, underlying, underlyingSecurityType);
 
-                    if (!result.TryGetValue(underlyingKey, out underlyingEntryLazy)
+                    if (!result.TryGetValue(underlyingKey, out underlyingEntry)
                         // let's retry with the wildcard
                         && underlying != SecurityDatabaseKey.Wildcard)
                     {
                         var underlyingKeyWildCard = new SecurityDatabaseKey(key.Market, SecurityDatabaseKey.Wildcard, underlyingSecurityType);
-                        result.TryGetValue(underlyingKeyWildCard, out underlyingEntryLazy);
+                        result.TryGetValue(underlyingKeyWildCard, out underlyingEntry);
                     }
                 }
-                return underlyingEntryLazy?.Value;
+                return underlyingEntry;
             }
         }