Speed loading improvements

AlgorithmFactory/Loader.cs

@@ -19,6 +19,7 @@ using System.IO;
 using System.Linq;
 using System.Reflection;
 using System.Runtime.InteropServices;
+using System.Runtime.Loader;
 using System.Security.Policy;
 using Python.Runtime;
 using QuantConnect.Interfaces;
@@ -179,9 +180,11 @@ namespace QuantConnect.AlgorithmFactory
                     {
                         PythonEngine.Exec(
                             @"
-import logging, os, sys
-sys.stdout = open(os.devnull, 'w')
-logging.captureWarnings(True)"
+from logging import captureWarnings
+from os import devnull
+from sys import stdout
+stdout = open(devnull, 'w')
+captureWarnings(True)"
                         );
                     }
                 }
@@ -211,40 +214,14 @@ logging.captureWarnings(True)"
 
             try
             {
-                byte[] debugInformationBytes = null;
-
                 // if the assembly is located in the base directory then don't bother loading the pdbs
                 // manually, they'll be loaded automatically by the .NET runtime.
-                var directoryName = new FileInfo(assemblyPath).DirectoryName;
-                if (directoryName != null && directoryName.TrimEnd(Path.DirectorySeparatorChar) != AppDomain.CurrentDomain.BaseDirectory.TrimEnd(Path.DirectorySeparatorChar))
-                {
-                    // see if the pdb exists
-                    var mdbFilename = assemblyPath + ".mdb";
-                    var pdbFilename = assemblyPath.Substring(0, assemblyPath.Length - 4) + ".pdb";
-                    if (File.Exists(pdbFilename))
-                    {
-                        debugInformationBytes = File.ReadAllBytes(pdbFilename);
-                    }
-                    // see if the mdb exists
-                    if (File.Exists(mdbFilename))
-                    {
-                        debugInformationBytes = File.ReadAllBytes(mdbFilename);
-                    }
-                }
+                var assemblyFile = new FileInfo(assemblyPath);
+                var directoryName = assemblyFile.DirectoryName;
 
                 //Load the assembly:
-                Assembly assembly;
-                if (debugInformationBytes == null)
-                {
-                    Log.Trace("Loader.TryCreateILAlgorithm(): Loading only the algorithm assembly");
-                    assembly = Assembly.LoadFrom(assemblyPath);
-                }
-                else
-                {
-                    Log.Trace("Loader.TryCreateILAlgorithm(): Loading debug information with algorithm");
-                    var assemblyBytes = File.ReadAllBytes(assemblyPath);
-                    assembly = Assembly.Load(assemblyBytes, debugInformationBytes);
-                }
+                var loader = new PluginLoadContext(directoryName);
+                var assembly = loader.LoadFromAssemblyPath(assemblyFile.FullName);
 
                 //Get the list of extention classes in the library:
                 var types = GetExtendedTypeNames(assembly);
@@ -277,7 +254,6 @@ logging.captureWarnings(True)"
                 {
                     Log.Trace("Loader.TryCreateILAlgorithm(): Loaded " + algorithmInstance.GetType().Name);
                 }
-
             }
             catch (ReflectionTypeLoadException err)
             {
@@ -302,7 +278,7 @@ logging.captureWarnings(True)"
         /// <returns>String list of types available.</returns>
         public static List<string> GetExtendedTypeNames(Assembly assembly)
         {
-            var types = new List<string>();
+            List<string> types;
             try
             {
                 Type[] assemblyTypes;
@@ -330,20 +306,22 @@ logging.captureWarnings(True)"
                 {
                     types = (from t in assemblyTypes
                              where t.IsClass                                    // require class
-                             where !t.IsAbstract                                // require concrete impl
-                             where AlgorithmInterfaceType.IsAssignableFrom(t)   // require derived from IAlgorithm
-                             where t.FullName != AlgorithmBaseTypeFullName      // require not equal to QuantConnect.QCAlgorithm
-                             where t.FullName != FrameworkBaseTypeFullName      // require not equal to QuantConnect.QCAlgorithmFramework
-                             where t.GetConstructor(Type.EmptyTypes) != null    // require default ctor
+                             && !t.IsAbstract                                // require concrete impl
+                             && AlgorithmInterfaceType.IsAssignableFrom(t)   // require derived from IAlgorithm
+                             && t.FullName != AlgorithmBaseTypeFullName      // require not equal to QuantConnect.QCAlgorithm
+                             && t.FullName != FrameworkBaseTypeFullName      // require not equal to QuantConnect.QCAlgorithmFramework
+                             && t.GetConstructor(Type.EmptyTypes) != null    // require default ctor
                              select t.FullName).ToList();
                 }
                 else
                 {
+                    types = [];
                     Log.Error("API.GetExtendedTypeNames(): No types found in assembly.");
                 }
             }
             catch (Exception err)
             {
+                types = [];
                 Log.Error(err);
             }
 
@@ -397,6 +375,37 @@ logging.captureWarnings(True)"
             }
         }
 
-    } // End Algorithm Factory Class
+        class PluginLoadContext : AssemblyLoadContext
+        {
+            private readonly AssemblyDependencyResolver _resolver;
+
+            public PluginLoadContext(string pluginPath)
+            {
+                _resolver = new AssemblyDependencyResolver(pluginPath);
+                Resolving += (context, assemblyName) =>
+                {
+                    var path = Path.Combine(Composer.PluginDirectory, $"{assemblyName.Name}.dll");
+                    try
+                    {
+                        return context.LoadFromAssemblyPath(path);
+                    }
+                    catch
+                    {
+                        return null;
+                    }
+                };
+            }
+
+            protected override Assembly Load(AssemblyName assemblyName)
+            {
+                var path = _resolver.ResolveAssemblyToPath(assemblyName);
+                if (path != null)
+                {
+                    return LoadFromAssemblyPath(path);
+                }
+                return null;
+            }
+        }
+} // End Algorithm Factory Class
 
 } // End QC Namespace.

Common/Securities/MarketHoursDatabase.cs

@@ -31,7 +31,7 @@ namespace QuantConnect.Securities
     /// Provides access to exchange hours and raw data times zones in various markets
     /// </summary>
     [JsonConverter(typeof(MarketHoursDatabaseJsonConverter))]
-    public class MarketHoursDatabase : BaseSecurityDatabase<MarketHoursDatabase, MarketHoursDatabase.Entry>
+    public class MarketHoursDatabase : BaseSecurityDatabase<MarketHoursDatabase, Lazy<MarketHoursDatabase.Entry>>
     {
         private readonly bool _forceExchangeAlwaysOpen = Config.GetBool("force-exchange-always-open");
 
@@ -40,7 +40,7 @@ namespace QuantConnect.Securities
         /// <summary>
         /// Gets all the exchange hours held by this provider
         /// </summary>
-        public List<KeyValuePair<SecurityDatabaseKey, Entry>> ExchangeHoursListing => Entries.ToList();
+        public List<KeyValuePair<SecurityDatabaseKey, Entry>> ExchangeHoursListing => Entries.ToList(x => new KeyValuePair<SecurityDatabaseKey, Entry>(x.Key, x.Value.Value));
 
         /// <summary>
         /// Gets a <see cref="MarketHoursDatabase"/> that always returns <see cref="SecurityExchangeHours.AlwaysOpen"/>
@@ -62,7 +62,7 @@ namespace QuantConnect.Securities
         /// Initializes a new instance of the <see cref="MarketHoursDatabase"/> class
         /// </summary>
         private MarketHoursDatabase()
-            : this(new())
+            : this(new Dictionary<SecurityDatabaseKey, Entry>())
         {
         }
 
@@ -71,7 +71,16 @@ namespace QuantConnect.Securities
         /// </summary>
         /// <param name="exchangeHours">The full listing of exchange hours by key</param>
         public MarketHoursDatabase(Dictionary<SecurityDatabaseKey, Entry> exchangeHours)
-            : base(exchangeHours, FromDataFolder, (entry, other) => entry.Update(other))
+            : this(exchangeHours.ToDictionary(kvp => kvp.Key, kvp => new Lazy<Entry>(kvp.Value)))
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="MarketHoursDatabase"/> class
+        /// </summary>
+        /// <param name="exchangeHours">The full listing of exchange hours by key</param>
+        public MarketHoursDatabase(Dictionary<SecurityDatabaseKey, Lazy<Entry>> exchangeHours)
+            : base(exchangeHours, FromDataFolder, (entry, other) => entry.Value.Update(other.Value))
         {
         }
 
@@ -139,7 +148,9 @@ namespace QuantConnect.Securities
         /// <returns>A new instance of the <see cref="MarketHoursDatabase"/> class</returns>
         public static MarketHoursDatabase FromFile(string path)
         {
-            return JsonConvert.DeserializeObject<MarketHoursDatabase>(File.ReadAllText(path));
+            using var stream = File.OpenRead(path);
+            var result = Extensions.DeserializeJson<MarketHoursDatabaseJsonConverter.MarketHoursDatabaseJson>(stream);
+            return result.Convert();
         }
 
         /// <summary>
@@ -161,7 +172,7 @@ namespace QuantConnect.Securities
             var entry = new Entry(dataTimeZone, exchangeHours);
             lock (DataFolderDatabaseLock)
             {
-                Entries[key] = entry;
+                Entries[key] = new Lazy<Entry>(entry);
                 CustomEntries.Add(key);
             }
             return entry;
@@ -252,16 +263,22 @@ namespace QuantConnect.Securities
 
         private bool TryGetEntryImpl(string market, string symbol, SecurityType securityType, out Entry entry)
         {
+            entry = null;
             var symbolKey = new SecurityDatabaseKey(market, symbol, securityType);
-            return Entries.TryGetValue(symbolKey, out entry)
+            if (Entries.TryGetValue(symbolKey, out var lazyEntry)
                 // now check with null symbol key
-                || Entries.TryGetValue(symbolKey.CreateCommonKey(), out entry)
+                || Entries.TryGetValue(symbolKey.CreateCommonKey(), out lazyEntry)
                 // if FOP check for future
                 || securityType == SecurityType.FutureOption && TryGetEntry(market,
                     FuturesOptionsSymbolMappings.MapFromOption(symbol), SecurityType.Future, out entry)
                 // if custom data type check for type specific entry
                 || (securityType == SecurityType.Base && SecurityIdentifier.TryGetCustomDataType(symbol, out var customType)
-                    && Entries.TryGetValue(new SecurityDatabaseKey(market, $"TYPE.{customType}", securityType), out entry));
+                    && Entries.TryGetValue(new SecurityDatabaseKey(market, $"TYPE.{customType}", securityType), out lazyEntry)))
+            {
+                entry ??= lazyEntry.Value;
+                return true;
+            }
+            return false;
         }
 
         /// <summary>

Common/Util/Composer.cs

@@ -37,11 +37,19 @@ namespace QuantConnect.Util
     /// </summary>
     public class Composer
     {
-        private static string PluginDirectory;
+        /// <summary>
+        /// The plugin directory source if any
+        /// </summary>
+        public static string PluginDirectory { get; private set; }
+
         private static readonly Lazy<Composer> LazyComposer = new Lazy<Composer>(
             () =>
             {
-                PluginDirectory = Config.Get("plugin-directory");
+                var pluginDirectory = Config.Get("plugin-directory");
+                if (!string.IsNullOrEmpty(pluginDirectory))
+                {
+                    PluginDirectory = new DirectoryInfo(pluginDirectory).FullName;
+                }
                 return new Composer();
             });
 
@@ -87,7 +95,7 @@ namespace QuantConnect.Util
 
             var loadFromPluginDir = !string.IsNullOrWhiteSpace(PluginDirectory)
                 && Directory.Exists(PluginDirectory) &&
-                new DirectoryInfo(PluginDirectory).FullName != primaryDllLookupDirectory;
+                PluginDirectory != primaryDllLookupDirectory;
             var fileNames = Directory.EnumerateFiles(primaryDllLookupDirectory, "*.dll");
             if (loadFromPluginDir)
             {
@@ -354,6 +362,11 @@ namespace QuantConnect.Util
                 var catalogs = new ConcurrentBag<ComposablePartCatalog>();
                 Parallel.ForEach(files, file =>
                 {
+                    if (!Path.GetFileName(file).StartsWith($"{nameof(QuantConnect)}.", StringComparison.InvariantCulture))
+                    {
+                        return;
+                    }
+
                     try
                     {
                         // we need to load assemblies so that C# algorithm dependencies are resolved correctly
@@ -370,12 +383,9 @@ namespace QuantConnect.Util
                             assembly = Assembly.LoadFrom(file);
                         }
 
-                        if (Path.GetFileName(file).StartsWith($"{nameof(QuantConnect)}.", StringComparison.InvariantCulture))
+                        foreach (var type in assembly.ExportedTypes.Where(type => !type.IsAbstract && !type.IsInterface && !type.IsEnum))
                         {
-                            foreach (var type in assembly.ExportedTypes.Where(type => !type.IsAbstract && !type.IsInterface && !type.IsEnum))
-                            {
-                                exportedTypes.Add(type);
-                            }
+                            exportedTypes.Add(type);
                         }
                         var asmCatalog = new AssemblyCatalog(assembly);
                         var parts = asmCatalog.Parts.ToArray();

Common/Util/MarketHoursDatabaseJsonConverter.cs

@@ -98,33 +98,23 @@ namespace QuantConnect.Util
             /// <returns>A new instance of the <see cref="MarketHoursDatabase"/> class</returns>
             public MarketHoursDatabase Convert()
             {
-                // first we parse the entries keys so that later we can sort by security type
-                var entries = new Dictionary<SecurityDatabaseKey, MarketHoursDatabaseEntryJson>(Entries.Count);
-                foreach (var entry in Entries)
+                var result = new Dictionary<SecurityDatabaseKey, Lazy<MarketHoursDatabase.Entry>>(Entries.Count);
+                foreach (var kvp in Entries)
                 {
                     try
                     {
-                        var key = SecurityDatabaseKey.Parse(entry.Key);
-                        if (key != null)
+                        var key = SecurityDatabaseKey.Parse(kvp.Key);
+                        result[key] = new Lazy<MarketHoursDatabase.Entry>(() =>
                         {
-                            entries[key] = entry.Value;
-                        }
-                    }
-                    catch (Exception err)
-                    {
-                        Log.Error(err);
-                    }
-                }
-
-                var result = new Dictionary<SecurityDatabaseKey, MarketHoursDatabase.Entry>(Entries.Count);
-                // we sort so we process generic entries and non options first
-                foreach (var entry in entries.OrderBy(kvp => kvp.Key.Symbol != null ? 1 : 0).ThenBy(kvp => kvp.Key.SecurityType.IsOption() ? 1 : 0))
-                {
-                    try
-                    {
-                        result.TryGetValue(entry.Key.CreateCommonKey(), out var marketEntry);
-                        var underlyingEntry = GetUnderlyingEntry(entry.Key, result);
-                        result[entry.Key] = entry.Value.Convert(underlyingEntry, marketEntry);
+                            MarketHoursDatabase.Entry marketEntry = null;
+                            if (key.Symbol != SecurityDatabaseKey.Wildcard)
+                            {
+                                result.TryGetValue(key.CreateCommonKey(), out var marketEntryLazy);
+                                marketEntry = marketEntryLazy?.Value;
+                            }
+                            var underlyingEntry = GetUnderlyingEntry(key, result);
+                            return kvp.Value.Convert(underlyingEntry, marketEntry);
+                        });
                     }
                     catch (Exception err)
                     {
@@ -137,9 +127,9 @@ namespace QuantConnect.Util
             /// <summary>
             /// Helper method to get the already processed underlying entry for options
             /// </summary>
-            private static MarketHoursDatabase.Entry GetUnderlyingEntry(SecurityDatabaseKey key, Dictionary<SecurityDatabaseKey, MarketHoursDatabase.Entry> result)
+            private static MarketHoursDatabase.Entry GetUnderlyingEntry(SecurityDatabaseKey key, Dictionary<SecurityDatabaseKey, Lazy<MarketHoursDatabase.Entry>> result)
             {
-                MarketHoursDatabase.Entry underlyingEntry = null;
+                Lazy<MarketHoursDatabase.Entry> underlyingEntryLazy = null;
                 if (key.SecurityType.IsOption())
                 {
                     // if option, let's get the underlyings entry
@@ -147,15 +137,15 @@ namespace QuantConnect.Util
                     var underlying = OptionSymbol.MapToUnderlying(key.Symbol, key.SecurityType);
                     var underlyingKey = new SecurityDatabaseKey(key.Market, underlying, underlyingSecurityType);
 
-                    if (!result.TryGetValue(underlyingKey, out underlyingEntry)
+                    if (!result.TryGetValue(underlyingKey, out underlyingEntryLazy)
                         // let's retry with the wildcard
                         && underlying != SecurityDatabaseKey.Wildcard)
                     {
                         var underlyingKeyWildCard = new SecurityDatabaseKey(key.Market, SecurityDatabaseKey.Wildcard, underlyingSecurityType);
-                        result.TryGetValue(underlyingKeyWildCard, out underlyingEntry);
+                        result.TryGetValue(underlyingKeyWildCard, out underlyingEntryLazy);
                     }
                 }
-                return underlyingEntry;
+                return underlyingEntryLazy?.Value;
             }
         }