diff --git a/DataProcessing/config.json b/DataProcessing/config.json index 39ad385..2c60e64 100644 --- a/DataProcessing/config.json +++ b/DataProcessing/config.json @@ -1,5 +1,8 @@ { "data-folder": "../../../../Lean/Data/", "history-provider": [ "SubscriptionDataReaderHistoryProvider", "IndexHistoryProvider" ], - "universe-generation-symbols": [] + "universe-generation-underlying-history-provider": "", + "universe-generation-derivative-history-provider": "", + "universe-generation-symbols": [], + "universe-generation-backup-files": false } \ No newline at end of file diff --git a/Lean.DataSource.DerivativeUniverseGenerator/DerivativeUniverseGenerator.cs b/Lean.DataSource.DerivativeUniverseGenerator/DerivativeUniverseGenerator.cs index 56063a2..b458ac9 100644 --- a/Lean.DataSource.DerivativeUniverseGenerator/DerivativeUniverseGenerator.cs +++ b/Lean.DataSource.DerivativeUniverseGenerator/DerivativeUniverseGenerator.cs @@ -20,6 +20,7 @@ using System.Threading; using System.Threading.Tasks; using NodaTime; +using QuantConnect.Configuration; using QuantConnect.Data; using QuantConnect.Data.Market; using QuantConnect.Interfaces; @@ -41,8 +42,13 @@ public abstract class DerivativeUniverseGenerator protected readonly string _dataFolderRoot; protected readonly string _outputFolderRoot; + // whether to generate the universe files as backup files (suffixed with ".backup"), + // which Lean can use in live trading as a fallback when the expected universe files are not available yet + protected readonly bool _generateBackupFiles; + protected readonly IDataProvider _dataProvider; - protected readonly IHistoryProvider _historyProvider; + protected readonly IHistoryProvider _underlyingHistoryProvider; + protected readonly IHistoryProvider _derivativeHistoryProvider; protected readonly IDataCacheProvider _dataCacheProvider; protected readonly MarketHoursDatabase _marketHoursDatabase; @@ -72,9 +78,29 @@ public abstract class DerivativeUniverseGenerator /// Path to the output folder /// The data provider to use /// The data cache provider to use - /// The history provider to use + /// The history provider to use for both the underlying and the derivatives public DerivativeUniverseGenerator(DateTime processingDate, SecurityType securityType, string market, string dataFolderRoot, string outputFolderRoot, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, IHistoryProvider historyProvider) + : this(processingDate, securityType, market, dataFolderRoot, outputFolderRoot, dataProvider, dataCacheProvider, + historyProvider, historyProvider) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The processing date + /// Derivative security type to process + /// Market of data to process + /// Path to the data folder + /// Path to the output folder + /// The data provider to use + /// The data cache provider to use + /// The history provider to use for the underlying security + /// The history provider to use for the derivative contracts + public DerivativeUniverseGenerator(DateTime processingDate, SecurityType securityType, string market, string dataFolderRoot, + string outputFolderRoot, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, + IHistoryProvider underlyingHistoryProvider, IHistoryProvider derivativeHistoryProvider) { _processingDate = processingDate; _securityType = securityType; @@ -83,8 +109,10 @@ public DerivativeUniverseGenerator(DateTime processingDate, SecurityType securit _outputFolderRoot = outputFolderRoot; _dataProvider = dataProvider; _dataCacheProvider = dataCacheProvider; - _historyProvider = historyProvider; + _underlyingHistoryProvider = underlyingHistoryProvider; + _derivativeHistoryProvider = derivativeHistoryProvider; _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + _generateBackupFiles = Config.GetBool("universe-generation-backup-files"); } /// @@ -263,7 +291,13 @@ protected virtual string GetUniverseFileName(Symbol canonicalSymbol) var universeDirectory = LeanData.GenerateUniversesDirectory(_outputFolderRoot, canonicalSymbol); Directory.CreateDirectory(universeDirectory); - return Path.Combine(universeDirectory, $"{_processingDate:yyyyMMdd}.csv"); + var universeFileName = Path.Combine(universeDirectory, $"{_processingDate:yyyyMMdd}.csv"); + if (_generateBackupFiles) + { + universeFileName += ".backup"; + } + + return universeFileName; } /// @@ -294,7 +328,8 @@ protected virtual bool TryGenerateAndWriteUnderlyingLine(Symbol underlyingSymbol LeanData.GetCommonTickTypeForCommonDataTypes(typeof(TradeBar), _securityType)); entry = CreateUniverseEntry(underlyingSymbol); - history = GetHistory(new[] { underlyingHistoryRequest }, marketHoursEntry.ExchangeHours.TimeZone, marketHoursEntry); + history = GetHistory(new[] { underlyingHistoryRequest }, marketHoursEntry.ExchangeHours.TimeZone, marketHoursEntry, + _underlyingHistoryProvider); var success = true; if (history == null || history.Count == 0) @@ -313,7 +348,7 @@ protected virtual bool TryGenerateAndWriteUnderlyingLine(Symbol underlyingSymbol } private List GetHistory(HistoryRequest[] historyRequests, - DateTimeZone sliceTimeZone, MarketHoursDatabase.Entry marketHoursEntry) + DateTimeZone sliceTimeZone, MarketHoursDatabase.Entry marketHoursEntry, IHistoryProvider historyProvider) { List history = null; @@ -328,7 +363,7 @@ private List GetHistory(HistoryRequest[] historyRequests, return request; }).ToArray(); - history = _historyProvider.GetHistory(resolutionHistoryRequests, sliceTimeZone).ToList(); + history = historyProvider.GetHistory(resolutionHistoryRequests, sliceTimeZone).ToList(); if (history != null && history.Count > 0) { return history; @@ -373,7 +408,8 @@ protected virtual IEnumerable GenerateDerivativeEn } else { - var history = GetHistory(historyRequests, marketHoursEntry.ExchangeHours.TimeZone, marketHoursEntry); + var history = GetHistory(historyRequests, marketHoursEntry.ExchangeHours.TimeZone, marketHoursEntry, + _derivativeHistoryProvider); entry = GenerateDerivativeEntry(symbol, history, underlyingHistory); } diff --git a/Lean.DataSource.DerivativeUniverseGenerator/Program.cs b/Lean.DataSource.DerivativeUniverseGenerator/Program.cs index 80e3916..6709aab 100644 --- a/Lean.DataSource.DerivativeUniverseGenerator/Program.cs +++ b/Lean.DataSource.DerivativeUniverseGenerator/Program.cs @@ -77,10 +77,9 @@ protected virtual void MainImpl(string[] args, string[] argNamesToIgnore = null) api.Initialize(Globals.UserId, Globals.UserToken, Globals.DataFolder); var dataCacheProvider = new ZipDataCacheProvider(dataProvider); - var historyProvider = new HistoryProviderManager(); var parameters = new HistoryProviderInitializeParameters(null, api, dataProvider, dataCacheProvider, mapFileProvider, factorFileProvider, (_) => { }, true, new DataPermissionManager(), null, new AlgorithmSettings()); - historyProvider.Initialize(parameters); + var (underlyingHistoryProvider, derivativeHistoryProvider) = CreateHistoryProviders(parameters); var timer = new Stopwatch(); timer.Start(); @@ -88,7 +87,7 @@ protected virtual void MainImpl(string[] args, string[] argNamesToIgnore = null) foreach (var market in markets) { var universeGenerator = GetUniverseGenerator(securityType, market, dataFolderRoot, outputFolderRoot, processingDate, - dataProvider, dataCacheProvider, historyProvider); + dataProvider, dataCacheProvider, underlyingHistoryProvider, derivativeHistoryProvider); try { @@ -112,7 +111,65 @@ protected virtual void MainImpl(string[] args, string[] argNamesToIgnore = null) protected abstract DerivativeUniverseGenerator GetUniverseGenerator(SecurityType securityType, string market, string dataFolderRoot, string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, - HistoryProviderManager historyProvider); + HistoryProviderManager underlyingHistoryProvider, HistoryProviderManager derivativeHistoryProvider); + + /// + /// Creates the history providers to use for the underlying securities and for the derivative contracts. + /// The "universe-generation-underlying-history-provider" and "universe-generation-derivative-history-provider" configs + /// allow overriding the history providers (from the "history-provider" config) to use for each of them. + /// When a config is not set, the corresponding history provider falls back to the "history-provider" config, + /// so by default a single shared history provider is used for both, just like before these configs existed. + /// + private static (HistoryProviderManager UnderlyingHistoryProvider, HistoryProviderManager DerivativeHistoryProvider) CreateHistoryProviders( + HistoryProviderInitializeParameters parameters) + { + var underlyingHistoryProviders = Config.Get("universe-generation-underlying-history-provider"); + var derivativeHistoryProviders = Config.Get("universe-generation-derivative-history-provider"); + + HistoryProviderManager defaultHistoryProvider = null; + HistoryProviderManager GetDefaultHistoryProvider() => defaultHistoryProvider ??= CreateHistoryProvider(null, parameters); + + var underlyingHistoryProvider = underlyingHistoryProviders.DeserializeList().IsNullOrEmpty() + ? GetDefaultHistoryProvider() + : CreateHistoryProvider(underlyingHistoryProviders, parameters); + + var derivativeHistoryProvider = derivativeHistoryProviders.DeserializeList().IsNullOrEmpty() + ? GetDefaultHistoryProvider() + : derivativeHistoryProviders == underlyingHistoryProviders + ? underlyingHistoryProvider + : CreateHistoryProvider(derivativeHistoryProviders, parameters); + + return (underlyingHistoryProvider, derivativeHistoryProvider); + } + + /// + /// Creates and initializes a history provider manager for the given history providers, + /// or for the "history-provider" config if none are given. + /// + private static HistoryProviderManager CreateHistoryProvider(string historyProviders, HistoryProviderInitializeParameters parameters) + { + var historyProviderManager = new HistoryProviderManager(); + if (string.IsNullOrEmpty(historyProviders)) + { + historyProviderManager.Initialize(parameters); + return historyProviderManager; + } + + // The history provider manager reads the history providers to wrap from the "history-provider" config, + // so we temporarily override it while initializing this instance + var originalHistoryProviders = Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider"); + Config.Set("history-provider", historyProviders); + try + { + historyProviderManager.Initialize(parameters); + } + finally + { + Config.Set("history-provider", originalHistoryProviders); + } + + return historyProviderManager; + } /// /// Validate and extract command line args and configuration options. diff --git a/Lean.DataSource.FuturesUniverseGenerator/FuturesUniverseGenerator.cs b/Lean.DataSource.FuturesUniverseGenerator/FuturesUniverseGenerator.cs index 854b88f..ffd5e58 100644 --- a/Lean.DataSource.FuturesUniverseGenerator/FuturesUniverseGenerator.cs +++ b/Lean.DataSource.FuturesUniverseGenerator/FuturesUniverseGenerator.cs @@ -36,11 +36,30 @@ public class FuturesUniverseGenerator : DerivativeUniverseGenerator.DerivativeUn /// Path to the output folder /// The data provider to use /// The data cache provider to use - /// The history provider to use + /// The history provider to use for both the underlying and the derivatives public FuturesUniverseGenerator(DateTime processingDate, string market, string dataFolderRoot, string outputFolderRoot, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, IHistoryProvider historyProvider) + : this(processingDate, market, dataFolderRoot, outputFolderRoot, dataProvider, dataCacheProvider, + historyProvider, historyProvider) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The processing date + /// Market of data to process + /// Path to the data folder + /// Path to the output folder + /// The data provider to use + /// The data cache provider to use + /// The history provider to use for the underlying security + /// The history provider to use for the future contracts + public FuturesUniverseGenerator(DateTime processingDate, string market, string dataFolderRoot, string outputFolderRoot, + IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, + IHistoryProvider underlyingHistoryProvider, IHistoryProvider derivativeHistoryProvider) : base(processingDate, SecurityType.Future, market, dataFolderRoot, outputFolderRoot, dataProvider, - dataCacheProvider, historyProvider) + dataCacheProvider, underlyingHistoryProvider, derivativeHistoryProvider) { } diff --git a/Lean.DataSource.FuturesUniverseGenerator/Program.cs b/Lean.DataSource.FuturesUniverseGenerator/Program.cs index 320baa6..e041e56 100644 --- a/Lean.DataSource.FuturesUniverseGenerator/Program.cs +++ b/Lean.DataSource.FuturesUniverseGenerator/Program.cs @@ -38,10 +38,10 @@ public static void Main(string[] args) protected override DerivativeUniverseGenerator.DerivativeUniverseGenerator GetUniverseGenerator(SecurityType securityType, string market, string dataFolderRoot, string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, - HistoryProviderManager historyProvider) + HistoryProviderManager underlyingHistoryProvider, HistoryProviderManager derivativeHistoryProvider) { return new FuturesUniverseGenerator(processingDate, market, dataFolderRoot, outputFolderRoot, dataProvider, - dataCacheProvider, historyProvider); + dataCacheProvider, underlyingHistoryProvider, derivativeHistoryProvider); } } } diff --git a/Lean.DataSource.FuturesUniverseGenerator/config.json b/Lean.DataSource.FuturesUniverseGenerator/config.json index d0b385a..df086ad 100644 --- a/Lean.DataSource.FuturesUniverseGenerator/config.json +++ b/Lean.DataSource.FuturesUniverseGenerator/config.json @@ -1,5 +1,8 @@ { "data-folder": "../../../Data/", "history-provider": [ "SubscriptionDataReaderHistoryProvider" ], - "universe-generation-symbols": [] + "universe-generation-underlying-history-provider": "", + "universe-generation-derivative-history-provider": "", + "universe-generation-symbols": [], + "universe-generation-backup-files": false } \ No newline at end of file diff --git a/Lean.DataSource.OptionsUniverseGenerator/OptionsUniverseGenerator.cs b/Lean.DataSource.OptionsUniverseGenerator/OptionsUniverseGenerator.cs index b08c25c..8d92bda 100644 --- a/Lean.DataSource.OptionsUniverseGenerator/OptionsUniverseGenerator.cs +++ b/Lean.DataSource.OptionsUniverseGenerator/OptionsUniverseGenerator.cs @@ -42,10 +42,31 @@ public class OptionsUniverseGenerator : DerivativeUniverseGenerator.DerivativeUn /// Path to the output folder /// The data provider to use /// The data cache provider to use - /// The history provider to use + /// The history provider to use for both the underlying and the derivatives public OptionsUniverseGenerator(DateTime processingDate, SecurityType securityType, string market, string dataFolderRoot, string outputFolderRoot, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, IHistoryProvider historyProvider) - : base(processingDate, securityType, market, dataFolderRoot, outputFolderRoot, dataProvider, dataCacheProvider, historyProvider) + : this(processingDate, securityType, market, dataFolderRoot, outputFolderRoot, dataProvider, dataCacheProvider, + historyProvider, historyProvider) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The processing date + /// Option security type to process + /// Market of data to process + /// Path to the data folder + /// Path to the output folder + /// The data provider to use + /// The data cache provider to use + /// The history provider to use for the underlying security + /// The history provider to use for the option contracts + public OptionsUniverseGenerator(DateTime processingDate, SecurityType securityType, string market, string dataFolderRoot, + string outputFolderRoot, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, + IHistoryProvider underlyingHistoryProvider, IHistoryProvider derivativeHistoryProvider) + : base(processingDate, securityType, market, dataFolderRoot, outputFolderRoot, dataProvider, dataCacheProvider, + underlyingHistoryProvider, derivativeHistoryProvider) { if (!_supportedSecurityTypes.Contains(securityType)) { diff --git a/Lean.DataSource.OptionsUniverseGenerator/Program.cs b/Lean.DataSource.OptionsUniverseGenerator/Program.cs index b40ea10..26ffdff 100644 --- a/Lean.DataSource.OptionsUniverseGenerator/Program.cs +++ b/Lean.DataSource.OptionsUniverseGenerator/Program.cs @@ -39,10 +39,10 @@ public static void Main(string[] args) protected override DerivativeUniverseGenerator.DerivativeUniverseGenerator GetUniverseGenerator(SecurityType securityType, string market, string dataFolderRoot, string outputFolderRoot, DateTime processingDate, IDataProvider dataProvider, IDataCacheProvider dataCacheProvider, - HistoryProviderManager historyProvider) + HistoryProviderManager underlyingHistoryProvider, HistoryProviderManager derivativeHistoryProvider) { return new OptionsUniverseGenerator(processingDate, securityType, market, dataFolderRoot, outputFolderRoot, - dataProvider, dataCacheProvider, historyProvider); + dataProvider, dataCacheProvider, underlyingHistoryProvider, derivativeHistoryProvider); } } } diff --git a/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/DerivativeUniverseGeneratorTests.cs b/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/DerivativeUniverseGeneratorTests.cs new file mode 100644 index 0000000..35bd8ca --- /dev/null +++ b/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/DerivativeUniverseGeneratorTests.cs @@ -0,0 +1,197 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NodaTime; +using NUnit.Framework; +using QuantConnect.Configuration; +using QuantConnect.Data; +using QuantConnect.Interfaces; +using QuantConnect.Securities; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using DerivativeUniverseGeneratorBase = QuantConnect.DataSource.DerivativeUniverseGenerator.DerivativeUniverseGenerator; +using BaseDerivativeUniverseFileEntry = QuantConnect.DataSource.DerivativeUniverseGenerator.BaseDerivativeUniverseFileEntry; +using IDerivativeUniverseFileEntry = QuantConnect.DataSource.DerivativeUniverseGenerator.IDerivativeUniverseFileEntry; + +namespace QuantConnect.DataSource.DerivativeUniverseGeneratorTests +{ + [TestFixture] + public class DerivativeUniverseGeneratorTests + { + private string _outputFolder; + + [SetUp] + public void SetUp() + { + _outputFolder = Path.Combine(Path.GetTempPath(), $"universe-generator-tests-{Guid.NewGuid():N}"); + } + + [TearDown] + public void TearDown() + { + Config.Set("universe-generation-backup-files", "false"); + if (Directory.Exists(_outputFolder)) + { + Directory.Delete(_outputFolder, true); + } + } + + [TestCase(false)] + [TestCase(true)] + public void GeneratesUniverseFilesAsBackupFilesWhenConfigured(bool generateBackupFiles) + { + Config.Set("universe-generation-backup-files", generateBackupFiles ? "true" : "false"); + + var processingDate = new DateTime(2026, 08, 10); + var generator = new TestDerivativeUniverseGenerator(processingDate, SecurityType.Option, Market.USA, _outputFolder); + var underlying = new Symbol(SecurityIdentifier.GenerateEquity("SPY", Market.USA, mapSymbol: false), "SPY"); + var canonicalSymbol = Symbol.CreateCanonicalOption(underlying); + + var universeFileName = generator.GetUniverseFileName(canonicalSymbol); + + var expectedFileName = generateBackupFiles ? $"{processingDate:yyyyMMdd}.csv.backup" : $"{processingDate:yyyyMMdd}.csv"; + Assert.AreEqual(expectedFileName, Path.GetFileName(universeFileName)); + } + + [Test] + public void UnderlyingAndDerivativeHistoryRequestsShareASingleProviderByDefault() + { + var historyProvider = new RecordingHistoryProvider(); + var generator = new TestDerivativeUniverseGenerator(new DateTime(2026, 08, 10), SecurityType.Option, Market.USA, + _outputFolder, historyProvider); + + var (underlying, contract) = GenerateHistoryForBothLegs(generator); + + var underlyingRequests = historyProvider.Requests.Where(x => x.Symbol == underlying).ToList(); + var derivativeRequests = historyProvider.Requests.Where(x => x.Symbol == contract).ToList(); + Assert.AreEqual(1, underlyingRequests.Count); + Assert.AreEqual(3, derivativeRequests.Count); + Assert.AreEqual(historyProvider.Requests.Count, underlyingRequests.Count + derivativeRequests.Count); + } + + [Test] + public void UnderlyingAndDerivativeHistoryRequestsAreRoutedToTheirOwnProviders() + { + var underlyingHistoryProvider = new RecordingHistoryProvider(); + var derivativeHistoryProvider = new RecordingHistoryProvider(); + var generator = new TestDerivativeUniverseGenerator(new DateTime(2026, 08, 10), SecurityType.Option, Market.USA, + _outputFolder, underlyingHistoryProvider, derivativeHistoryProvider); + + var (underlying, contract) = GenerateHistoryForBothLegs(generator); + + Assert.AreEqual(1, underlyingHistoryProvider.Requests.Count); + Assert.IsTrue(underlyingHistoryProvider.Requests.All(x => x.Symbol == underlying)); + + Assert.AreEqual(3, derivativeHistoryProvider.Requests.Count); + Assert.IsTrue(derivativeHistoryProvider.Requests.All(x => x.Symbol == contract)); + } + + /// + /// Runs the underlying entry generation and the derivative entries generation for a single SPY option contract, + /// so the requests each history provider received can be asserted on. + /// + private static (Symbol Underlying, Symbol Contract) GenerateHistoryForBothLegs(TestDerivativeUniverseGenerator generator) + { + var underlying = new Symbol(SecurityIdentifier.GenerateEquity("SPY", Market.USA, mapSymbol: false), "SPY"); + var contract = Symbol.CreateOption(underlying, Market.USA, OptionStyle.American, OptionRight.Call, 400m, new DateTime(2026, 08, 21)); + + var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + var underlyingMarketHoursEntry = marketHoursDatabase.GetEntry(underlying.ID.Market, underlying, underlying.SecurityType); + var derivativeMarketHoursEntry = marketHoursDatabase.GetEntry(contract.ID.Market, contract, contract.SecurityType); + + using var writer = new StreamWriter(Stream.Null); + generator.TryGenerateAndWriteUnderlyingLine(underlying, underlyingMarketHoursEntry, writer); + generator.GenerateDerivativeEntries(contract.Canonical, new List { contract }, derivativeMarketHoursEntry).ToList(); + + return (underlying, contract); + } + + private class TestDerivativeUniverseGenerator : DerivativeUniverseGeneratorBase + { + public TestDerivativeUniverseGenerator(DateTime processingDate, SecurityType securityType, string market, string outputFolderRoot, + IHistoryProvider historyProvider = null) + : base(processingDate, securityType, market, outputFolderRoot, outputFolderRoot, null, null, historyProvider) + { + } + + public TestDerivativeUniverseGenerator(DateTime processingDate, SecurityType securityType, string market, string outputFolderRoot, + IHistoryProvider underlyingHistoryProvider, IHistoryProvider derivativeHistoryProvider) + : base(processingDate, securityType, market, outputFolderRoot, outputFolderRoot, null, null, + underlyingHistoryProvider, derivativeHistoryProvider) + { + } + + public new string GetUniverseFileName(Symbol canonicalSymbol) + { + return base.GetUniverseFileName(canonicalSymbol); + } + + public bool TryGenerateAndWriteUnderlyingLine(Symbol underlyingSymbol, MarketHoursDatabase.Entry marketHoursEntry, + StreamWriter writer) + { + return base.TryGenerateAndWriteUnderlyingLine(underlyingSymbol, marketHoursEntry, writer, out _, out _); + } + + public IEnumerable GenerateDerivativeEntries(Symbol canonicalSymbol, List symbols, + MarketHoursDatabase.Entry marketHoursEntry) + { + return base.GenerateDerivativeEntries(canonicalSymbol, symbols, marketHoursEntry, null, null); + } + + protected override Dictionary> FilterSymbols(Dictionary> symbols, + HashSet symbolsToProcess) + { + return symbols; + } + + protected override IDerivativeUniverseFileEntry CreateUniverseEntry(Symbol symbol) + { + return new BaseDerivativeUniverseFileEntry(symbol); + } + + protected override bool NeedsUnderlyingData() + { + return false; + } + } + + private class RecordingHistoryProvider : HistoryProviderBase + { + /// + /// The history requests this provider has received, across all calls. + /// + public List Requests { get; } = new(); + + public override int DataPointCount => 0; + + public override void Initialize(HistoryProviderInitializeParameters parameters) + { + } + + public override IEnumerable GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) + { + lock (Requests) + { + Requests.AddRange(requests); + } + + return Enumerable.Empty(); + } + } + } +} diff --git a/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/TestSetup.cs b/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/TestSetup.cs new file mode 100644 index 0000000..66e8b15 --- /dev/null +++ b/QuantConnect.DataSource.DerivativeUniverseGeneratorTests/TestSetup.cs @@ -0,0 +1,49 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NUnit.Framework; +using QuantConnect.Configuration; +using QuantConnect.Interfaces; +using QuantConnect.Util; + +namespace QuantConnect.DataSource.DerivativeUniverseGeneratorTests +{ + /// + /// Assembly-level test setup + /// + [SetUpFixture] + public class TestSetup + { + /// + /// Registers the map file and factor file providers in the composer before any test runs. + /// Lean resolves them with (e.g. for symbol mapping + /// and for corporate events), which does not compose new instances, + /// so without this registration those code paths would fail with null providers. + /// + [OneTimeSetUp] + public void SetUp() + { + var dataProvider = Composer.Instance.GetExportedValueByTypeName( + Config.Get("data-provider", "DefaultDataProvider")); + var mapFileProvider = Composer.Instance.GetExportedValueByTypeName( + Config.Get("map-file-provider", "LocalDiskMapFileProvider")); + mapFileProvider.Initialize(dataProvider); + var factorFileProvider = Composer.Instance.GetExportedValueByTypeName( + Config.Get("factor-file-provider", "LocalDiskFactorFileProvider")); + factorFileProvider.Initialize(mapFileProvider, dataProvider); + } + } +}