Add configs for backup universe files and separate underlying/derivative history providers

DataProcessing/config.json

@@ -1,5 +1,8 @@
 {
 	"data-folder": "../../../../Lean/Data/",
 	"history-provider": [ "SubscriptionDataReaderHistoryProvider", "IndexHistoryProvider" ],
-	"universe-generation-symbols": []
+	"universe-generation-underlying-history-provider": "",
+	"universe-generation-derivative-history-provider": "",
+	"universe-generation-symbols": [],
+	"universe-generation-backup-files": false
 }

Lean.DataSource.DerivativeUniverseGenerator/DerivativeUniverseGenerator.cs

@@ -20,6 +20,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using NodaTime;
+using QuantConnect.Configuration;
 using QuantConnect.Data;
 using QuantConnect.Data.Market;
 using QuantConnect.Interfaces;
@@ -41,8 +42,13 @@ public abstract class DerivativeUniverseGenerator
         protected readonly string _dataFolderRoot;
         protected readonly string _outputFolderRoot;
 
+        // whether to generate the universe files as backup files (suffixed with ".backup"),
+        // which Lean can use in live trading as a fallback when the expected universe files are not available yet
+        protected readonly bool _generateBackupFiles;
+
         protected readonly IDataProvider _dataProvider;
-        protected readonly IHistoryProvider _historyProvider;
+        protected readonly IHistoryProvider _underlyingHistoryProvider;
+        protected readonly IHistoryProvider _derivativeHistoryProvider;
         protected readonly IDataCacheProvider _dataCacheProvider;
 
         protected readonly MarketHoursDatabase _marketHoursDatabase;
@@ -72,9 +78,29 @@ public abstract class DerivativeUniverseGenerator
         /// <param name="outputFolderRoot">Path to the output folder</param>
         /// <param name="dataProvider">The data provider to use</param>
         /// <param name="dataCacheProvider">The data cache provider to use</param>
-        /// <param name="historyProvider">The history provider to use</param>
+        /// <param name="historyProvider">The history provider to use for both the underlying and the derivatives</param>
         public DerivativeUniverseGenerator(DateTime processingDate, SecurityType securityType, string market, string dataFolderRoot,
             string outputFolderRoot, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, IHistoryProvider historyProvider)
+            : this(processingDate, securityType, market, dataFolderRoot, outputFolderRoot, dataProvider, dataCacheProvider,
+                historyProvider, historyProvider)
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="DerivativeUniverseGenerator" /> class.
+        /// </summary>
+        /// <param name="processingDate">The processing date</param>
+        /// <param name="securityType">Derivative security type to process</param>
+        /// <param name="market">Market of data to process</param>
+        /// <param name="dataFolderRoot">Path to the data folder</param>
+        /// <param name="outputFolderRoot">Path to the output folder</param>
+        /// <param name="dataProvider">The data provider to use</param>
+        /// <param name="dataCacheProvider">The data cache provider to use</param>
+        /// <param name="underlyingHistoryProvider">The history provider to use for the underlying security</param>
+        /// <param name="derivativeHistoryProvider">The history provider to use for the derivative contracts</param>
+        public DerivativeUniverseGenerator(DateTime processingDate, SecurityType securityType, string market, string dataFolderRoot,
+            string outputFolderRoot, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider,
+            IHistoryProvider underlyingHistoryProvider, IHistoryProvider derivativeHistoryProvider)
         {
             _processingDate = processingDate;
             _securityType = securityType;
@@ -83,8 +109,10 @@ public DerivativeUniverseGenerator(DateTime processingDate, SecurityType securit
             _outputFolderRoot = outputFolderRoot;
             _dataProvider = dataProvider;
             _dataCacheProvider = dataCacheProvider;
-            _historyProvider = historyProvider;
+            _underlyingHistoryProvider = underlyingHistoryProvider;
+            _derivativeHistoryProvider = derivativeHistoryProvider;
             _marketHoursDatabase = MarketHoursDatabase.FromDataFolder();
+            _generateBackupFiles = Config.GetBool("universe-generation-backup-files");
         }
 
         /// <summary>
@@ -263,7 +291,13 @@ protected virtual string GetUniverseFileName(Symbol canonicalSymbol)
             var universeDirectory = LeanData.GenerateUniversesDirectory(_outputFolderRoot, canonicalSymbol);
             Directory.CreateDirectory(universeDirectory);
 
-            return Path.Combine(universeDirectory, $"{_processingDate:yyyyMMdd}.csv");
+            var universeFileName = Path.Combine(universeDirectory, $"{_processingDate:yyyyMMdd}.csv");
+            if (_generateBackupFiles)
+            {
+                universeFileName += ".backup";
+            }
+
+            return universeFileName;
         }
 
         /// <summary>
@@ -294,7 +328,8 @@ protected virtual bool TryGenerateAndWriteUnderlyingLine(Symbol underlyingSymbol
                 LeanData.GetCommonTickTypeForCommonDataTypes(typeof(TradeBar), _securityType));
 
             entry = CreateUniverseEntry(underlyingSymbol);
-            history = GetHistory(new[] { underlyingHistoryRequest }, marketHoursEntry.ExchangeHours.TimeZone, marketHoursEntry);
+            history = GetHistory(new[] { underlyingHistoryRequest }, marketHoursEntry.ExchangeHours.TimeZone, marketHoursEntry,
+                _underlyingHistoryProvider);
             var success = true;
 
             if (history == null || history.Count == 0)
@@ -313,7 +348,7 @@ protected virtual bool TryGenerateAndWriteUnderlyingLine(Symbol underlyingSymbol
         }
 
         private List<Slice> GetHistory(HistoryRequest[] historyRequests,
-            DateTimeZone sliceTimeZone, MarketHoursDatabase.Entry marketHoursEntry)
+            DateTimeZone sliceTimeZone, MarketHoursDatabase.Entry marketHoursEntry, IHistoryProvider historyProvider)
         {
             List<Slice> history = null;
 
@@ -328,7 +363,7 @@ private List<Slice> GetHistory(HistoryRequest[] historyRequests,
                     return request;
                 }).ToArray();
 
-                history = _historyProvider.GetHistory(resolutionHistoryRequests, sliceTimeZone).ToList();
+                history = historyProvider.GetHistory(resolutionHistoryRequests, sliceTimeZone).ToList();
                 if (history != null && history.Count > 0)
                 {
                     return history;
@@ -373,7 +408,8 @@ protected virtual IEnumerable<IDerivativeUniverseFileEntry> GenerateDerivativeEn
                 }
                 else
                 {
-                    var history = GetHistory(historyRequests, marketHoursEntry.ExchangeHours.TimeZone, marketHoursEntry);
+                    var history = GetHistory(historyRequests, marketHoursEntry.ExchangeHours.TimeZone, marketHoursEntry,
+                        _derivativeHistoryProvider);
                     entry = GenerateDerivativeEntry(symbol, history, underlyingHistory);
                 }
 

Lean.DataSource.DerivativeUniverseGenerator/Program.cs

@@ -77,18 +77,17 @@ protected virtual void MainImpl(string[] args, string[] argNamesToIgnore = null)
             api.Initialize(Globals.UserId, Globals.UserToken, Globals.DataFolder);
 
             var dataCacheProvider = new ZipDataCacheProvider(dataProvider);
-            var historyProvider = new HistoryProviderManager();
             var parameters = new HistoryProviderInitializeParameters(null, api, dataProvider, dataCacheProvider, mapFileProvider,
                 factorFileProvider, (_) => { }, true, new DataPermissionManager(), null, new AlgorithmSettings());
-            historyProvider.Initialize(parameters);
+            var (underlyingHistoryProvider, derivativeHistoryProvider) = CreateHistoryProviders(parameters);
 
             var timer = new Stopwatch();
             timer.Start();
 
             foreach (var market in markets)
             {
                 var universeGenerator = GetUniverseGenerator(securityType, market, dataFolderRoot, outputFolderRoot, processingDate,
-                    dataProvider, dataCacheProvider, historyProvider);
+                    dataProvider, dataCacheProvider, underlyingHistoryProvider, derivativeHistoryProvider);
 
                 try
                 {
@@ -112,7 +111,65 @@ protected virtual void MainImpl(string[] args, string[] argNamesToIgnore = null)
 
         protected abstract DerivativeUniverseGenerator GetUniverseGenerator(SecurityType securityType, string market, string dataFolderRoot,
             string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider,
-            HistoryProviderManager historyProvider);
+            HistoryProviderManager underlyingHistoryProvider, HistoryProviderManager derivativeHistoryProvider);
+
+        /// <summary>
+        /// Creates the history providers to use for the underlying securities and for the derivative contracts.
+        /// The "universe-generation-underlying-history-provider" and "universe-generation-derivative-history-provider" configs
+        /// allow overriding the history providers (from the "history-provider" config) to use for each of them.
+        /// When a config is not set, the corresponding history provider falls back to the "history-provider" config,
+        /// so by default a single shared history provider is used for both, just like before these configs existed.
+        /// </summary>
+        private static (HistoryProviderManager UnderlyingHistoryProvider, HistoryProviderManager DerivativeHistoryProvider) CreateHistoryProviders(
+            HistoryProviderInitializeParameters parameters)
+        {
+            var underlyingHistoryProviders = Config.Get("universe-generation-underlying-history-provider");
+            var derivativeHistoryProviders = Config.Get("universe-generation-derivative-history-provider");
+
+            HistoryProviderManager defaultHistoryProvider = null;
+            HistoryProviderManager GetDefaultHistoryProvider() => defaultHistoryProvider ??= CreateHistoryProvider(null, parameters);
+
+            var underlyingHistoryProvider = underlyingHistoryProviders.DeserializeList().IsNullOrEmpty()
+                ? GetDefaultHistoryProvider()
+                : CreateHistoryProvider(underlyingHistoryProviders, parameters);
+
+            var derivativeHistoryProvider = derivativeHistoryProviders.DeserializeList().IsNullOrEmpty()
+                ? GetDefaultHistoryProvider()
+                : derivativeHistoryProviders == underlyingHistoryProviders
+                    ? underlyingHistoryProvider
+                    : CreateHistoryProvider(derivativeHistoryProviders, parameters);
+
+            return (underlyingHistoryProvider, derivativeHistoryProvider);
+        }
+
+        /// <summary>
+        /// Creates and initializes a history provider manager for the given history providers,
+        /// or for the "history-provider" config if none are given.
+        /// </summary>
+        private static HistoryProviderManager CreateHistoryProvider(string historyProviders, HistoryProviderInitializeParameters parameters)
+        {
+            var historyProviderManager = new HistoryProviderManager();
+            if (string.IsNullOrEmpty(historyProviders))
+            {
+                historyProviderManager.Initialize(parameters);
+                return historyProviderManager;
+            }
+
+            // The history provider manager reads the history providers to wrap from the "history-provider" config,
+            // so we temporarily override it while initializing this instance
+            var originalHistoryProviders = Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider");
+            Config.Set("history-provider", historyProviders);
+            try
+            {
+                historyProviderManager.Initialize(parameters);
+            }
+            finally
+            {
+                Config.Set("history-provider", originalHistoryProviders);
+            }
+
+            return historyProviderManager;
+        }
 
         /// <summary>
         /// Validate and extract command line args and configuration options.

Lean.DataSource.FuturesUniverseGenerator/FuturesUniverseGenerator.cs

@@ -36,11 +36,30 @@ public class FuturesUniverseGenerator : DerivativeUniverseGenerator.DerivativeUn
         /// <param name="outputFolderRoot">Path to the output folder</param>
         /// <param name="dataProvider">The data provider to use</param>
         /// <param name="dataCacheProvider">The data cache provider to use</param>
-        /// <param name="historyProvider">The history provider to use</param>
+        /// <param name="historyProvider">The history provider to use for both the underlying and the derivatives</param>
         public FuturesUniverseGenerator(DateTime processingDate, string market, string dataFolderRoot, string outputFolderRoot,
             IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, IHistoryProvider historyProvider)
+            : this(processingDate, market, dataFolderRoot, outputFolderRoot, dataProvider, dataCacheProvider,
+                historyProvider, historyProvider)
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="FuturesUniverseGenerator" /> class.
+        /// </summary>
+        /// <param name="processingDate">The processing date</param>
+        /// <param name="market">Market of data to process</param>
+        /// <param name="dataFolderRoot">Path to the data folder</param>
+        /// <param name="outputFolderRoot">Path to the output folder</param>
+        /// <param name="dataProvider">The data provider to use</param>
+        /// <param name="dataCacheProvider">The data cache provider to use</param>
+        /// <param name="underlyingHistoryProvider">The history provider to use for the underlying security</param>
+        /// <param name="derivativeHistoryProvider">The history provider to use for the future contracts</param>
+        public FuturesUniverseGenerator(DateTime processingDate, string market, string dataFolderRoot, string outputFolderRoot,
+            IDataProvider dataProvider, IDataCacheProvider dataCacheProvider,
+            IHistoryProvider underlyingHistoryProvider, IHistoryProvider derivativeHistoryProvider)
             : base(processingDate, SecurityType.Future, market, dataFolderRoot, outputFolderRoot, dataProvider,
-                  dataCacheProvider, historyProvider)
+                  dataCacheProvider, underlyingHistoryProvider, derivativeHistoryProvider)
         {
         }
 

Lean.DataSource.FuturesUniverseGenerator/Program.cs

@@ -38,10 +38,10 @@ public static void Main(string[] args)
 
         protected override DerivativeUniverseGenerator.DerivativeUniverseGenerator GetUniverseGenerator(SecurityType securityType, string market,
             string dataFolderRoot, string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider,
-            HistoryProviderManager historyProvider)
+            HistoryProviderManager underlyingHistoryProvider, HistoryProviderManager derivativeHistoryProvider)
         {
             return new FuturesUniverseGenerator(processingDate, market, dataFolderRoot, outputFolderRoot, dataProvider,
-                dataCacheProvider, historyProvider);
+                dataCacheProvider, underlyingHistoryProvider, derivativeHistoryProvider);
         }
     }
 }

Lean.DataSource.FuturesUniverseGenerator/config.json

@@ -1,5 +1,8 @@
 {
 	"data-folder": "../../../Data/",
 	"history-provider": [ "SubscriptionDataReaderHistoryProvider" ],
-	"universe-generation-symbols": []
+	"universe-generation-underlying-history-provider": "",
+	"universe-generation-derivative-history-provider": "",
+	"universe-generation-symbols": [],
+	"universe-generation-backup-files": false
 }

Lean.DataSource.OptionsUniverseGenerator/OptionsUniverseGenerator.cs

@@ -42,10 +42,31 @@ public class OptionsUniverseGenerator : DerivativeUniverseGenerator.DerivativeUn
         /// <param name="outputFolderRoot">Path to the output folder</param>
         /// <param name="dataProvider">The data provider to use</param>
         /// <param name="dataCacheProvider">The data cache provider to use</param>
-        /// <param name="historyProvider">The history provider to use</param>
+        /// <param name="historyProvider">The history provider to use for both the underlying and the derivatives</param>
         public OptionsUniverseGenerator(DateTime processingDate, SecurityType securityType, string market, string dataFolderRoot,
             string outputFolderRoot, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, IHistoryProvider historyProvider)
-            : base(processingDate, securityType, market, dataFolderRoot, outputFolderRoot, dataProvider, dataCacheProvider, historyProvider)
+            : this(processingDate, securityType, market, dataFolderRoot, outputFolderRoot, dataProvider, dataCacheProvider,
+                historyProvider, historyProvider)
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="OptionsUniverseGenerator" /> class.
+        /// </summary>
+        /// <param name="processingDate">The processing date</param>
+        /// <param name="securityType">Option security type to process</param>
+        /// <param name="market">Market of data to process</param>
+        /// <param name="dataFolderRoot">Path to the data folder</param>
+        /// <param name="outputFolderRoot">Path to the output folder</param>
+        /// <param name="dataProvider">The data provider to use</param>
+        /// <param name="dataCacheProvider">The data cache provider to use</param>
+        /// <param name="underlyingHistoryProvider">The history provider to use for the underlying security</param>
+        /// <param name="derivativeHistoryProvider">The history provider to use for the option contracts</param>
+        public OptionsUniverseGenerator(DateTime processingDate, SecurityType securityType, string market, string dataFolderRoot,
+            string outputFolderRoot, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider,
+            IHistoryProvider underlyingHistoryProvider, IHistoryProvider derivativeHistoryProvider)
+            : base(processingDate, securityType, market, dataFolderRoot, outputFolderRoot, dataProvider, dataCacheProvider,
+                underlyingHistoryProvider, derivativeHistoryProvider)
         {
             if (!_supportedSecurityTypes.Contains(securityType))
             {

Lean.DataSource.OptionsUniverseGenerator/Program.cs

@@ -39,10 +39,10 @@ public static void Main(string[] args)
 
         protected override DerivativeUniverseGenerator.DerivativeUniverseGenerator GetUniverseGenerator(SecurityType securityType, string market,
             string dataFolderRoot, string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider,
-            HistoryProviderManager historyProvider)
+            HistoryProviderManager underlyingHistoryProvider, HistoryProviderManager derivativeHistoryProvider)
         {
             return new OptionsUniverseGenerator(processingDate, securityType, market, dataFolderRoot, outputFolderRoot,
-                dataProvider, dataCacheProvider, historyProvider);
+                dataProvider, dataCacheProvider, underlyingHistoryProvider, derivativeHistoryProvider);
         }
     }
 }