From 31d18f37629b26971d69b1d48f8a635baa786b7a Mon Sep 17 00:00:00 2001 From: Joseph Scorsone Date: Mon, 29 Dec 2025 14:34:13 -0500 Subject: [PATCH 01/18] Improvements from previous PR. Adhere to QC code conventions better. More robust code. Refactored and properly passing tests. --- .../DataBentoDataDownloaderTests.cs | 221 +++++----- .../DataBentoDataProviderHistoryTests.cs | 170 +++----- .../DataBentoRawLiveClientTests.cs | 227 +++------- ...tConnect.DataSource.DataBento.Tests.csproj | 4 +- QuantConnect.DataBento.Tests/TestSetup.cs | 17 - QuantConnect.DataBento.Tests/config.json | 2 +- .../DataBentoDataDownloader.cs | 332 ++++++--------- .../DataBentoDataProvider.cs | 383 +++++++---------- .../DataBentoHistoryProivder.cs | 85 +--- .../DataBentoRawLiveClient.cs | 402 ++++++++---------- .../DataBentoSymbolMapper.cs | 174 ++++++++ .../QuantConnect.DataSource.DataBento.csproj | 4 + models/DataBentoTypes.cs | 112 +++++ 13 files changed, 1012 insertions(+), 1121 deletions(-) create mode 100644 QuantConnect.DataBento/DataBentoSymbolMapper.cs create mode 100644 models/DataBentoTypes.cs diff --git a/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs index d9d8071..a5063df 100644 --- a/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs @@ -17,11 +17,13 @@ using System; using System.Linq; using NUnit.Framework; +using QuantConnect.Configuration; using QuantConnect.Data; using QuantConnect.Data.Market; using QuantConnect.Lean.DataSource.DataBento; using QuantConnect.Logging; -using QuantConnect.Configuration; +using QuantConnect.Securities; +using QuantConnect.Util; namespace QuantConnect.Lean.DataSource.DataBento.Tests { @@ -29,12 +31,20 @@ namespace QuantConnect.Lean.DataSource.DataBento.Tests public class DataBentoDataDownloaderTests { private DataBentoDataDownloader _downloader; - private readonly string _apiKey = Config.Get("databento-api-key"); + private MarketHoursDatabase _marketHoursDatabase; + protected readonly string ApiKey = Config.Get("databento-api-key"); + + private static Symbol CreateEsFuture() + { + var expiration = new DateTime(2026, 3, 20); + return Symbol.CreateFuture("ES", Market.CME, expiration); + } [SetUp] public void SetUp() { - _downloader = new DataBentoDataDownloader(_apiKey); + _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + _downloader = new DataBentoDataDownloader(ApiKey, _marketHoursDatabase); } [TearDown] @@ -43,141 +53,154 @@ public void TearDown() _downloader?.Dispose(); } - [Test] - [TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Daily, TickType.Trade)] - [TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Hour, TickType.Trade)] - [TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Minute, TickType.Trade)] - [TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Second, TickType.Trade)] - [TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Tick, TickType.Trade)] - [Explicit("This test requires a configured DataBento API key")] - public void DownloadsHistoricalData(string ticker, SecurityType securityType, string market, Resolution resolution, TickType tickType) + [TestCase(Resolution.Daily)] + [TestCase(Resolution.Hour)] + [TestCase(Resolution.Minute)] + [TestCase(Resolution.Second)] + [TestCase(Resolution.Tick)] + public void DownloadsTradeDataForLeanFuture(Resolution resolution) { - var symbol = Symbol.Create(ticker, securityType, market); - var startTime = new DateTime(2024, 1, 15); - var endTime = new DateTime(2024, 1, 16); - var param = new DataDownloaderGetParameters(symbol, resolution, startTime, endTime, tickType); + var symbol = CreateEsFuture(); + var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; - var downloadResponse = _downloader.Get(param).ToList(); + var startUtc = new DateTime(2024, 5, 1, 0, 0, 0, DateTimeKind.Utc); + var endUtc = new DateTime(2024, 5, 2, 0, 0, 0, DateTimeKind.Utc); - Log.Trace($"Downloaded {downloadResponse.Count} data points for {symbol} at {resolution} resolution"); + if (resolution == Resolution.Tick) + { + startUtc = new DateTime(2024, 5, 1, 9, 30, 0, DateTimeKind.Utc); + endUtc = startUtc.AddMinutes(15); + } - Assert.IsTrue(downloadResponse.Any(), "Expected to download at least one data point"); + var parameters = new DataDownloaderGetParameters( + symbol, + resolution, + startUtc, + endUtc, + TickType.Trade + ); - foreach (var data in downloadResponse) + var data = _downloader.Get(parameters).ToList(); + + Log.Trace($"Downloaded {data.Count} trade points for {symbol} @ {resolution}"); + + Assert.IsNotEmpty(data); + + var startExchange = startUtc.ConvertFromUtc(exchangeTimeZone); + var endExchange = endUtc.ConvertFromUtc(exchangeTimeZone); + + foreach (var point in data) { - Assert.IsNotNull(data, "Data point should not be null"); - Assert.AreEqual(symbol, data.Symbol, "Symbol should match requested symbol"); - Assert.IsTrue(data.Time >= startTime && data.Time <= endTime, "Data time should be within requested range"); + Assert.AreEqual(symbol, point.Symbol); + Assert.That(point.Time, Is.InRange(startExchange, endExchange)); - if (data is TradeBar tradeBar) + switch (point) { - Assert.Greater(tradeBar.Close, 0, "Close price should be positive"); - Assert.GreaterOrEqual(tradeBar.Volume, 0, "Volume should be non-negative"); - Assert.Greater(tradeBar.High, 0, "High price should be positive"); - Assert.Greater(tradeBar.Low, 0, "Low price should be positive"); - Assert.Greater(tradeBar.Open, 0, "Open price should be positive"); - Assert.GreaterOrEqual(tradeBar.High, tradeBar.Low, "High should be >= Low"); - Assert.GreaterOrEqual(tradeBar.High, tradeBar.Open, "High should be >= Open"); - Assert.GreaterOrEqual(tradeBar.High, tradeBar.Close, "High should be >= Close"); - Assert.LessOrEqual(tradeBar.Low, tradeBar.Open, "Low should be <= Open"); - Assert.LessOrEqual(tradeBar.Low, tradeBar.Close, "Low should be <= Close"); - } - else if (data is QuoteBar quoteBar) - { - Assert.Greater(quoteBar.Close, 0, "Quote close price should be positive"); - if (quoteBar.Bid != null) - { - Assert.Greater(quoteBar.Bid.Close, 0, "Bid price should be positive"); - } - if (quoteBar.Ask != null) - { - Assert.Greater(quoteBar.Ask.Close, 0, "Ask price should be positive"); - } - } - else if (data is Tick tick) - { - Assert.Greater(tick.Value, 0, "Tick value should be positive"); - Assert.GreaterOrEqual(tick.Quantity, 0, "Tick quantity should be non-negative"); + case TradeBar bar: + Assert.Greater(bar.Open, 0); + Assert.Greater(bar.High, 0); + Assert.Greater(bar.Low, 0); + Assert.Greater(bar.Close, 0); + Assert.GreaterOrEqual(bar.Volume, 0); + Assert.GreaterOrEqual(bar.High, bar.Low); + break; + + case Tick tick: + Assert.Greater(tick.Value, 0); + Assert.GreaterOrEqual(tick.Quantity, 0); + break; + + default: + Assert.Fail($"Unexpected data type {point.GetType()}"); + break; } } } [Test] - [TestCase("ZNM3", SecurityType.Future, Market.CME, Resolution.Daily, TickType.Trade)] - [TestCase("ZNM3", SecurityType.Future, Market.CME, Resolution.Hour, TickType.Trade)] - [Explicit("This test requires a configured DataBento API key")] - public void DownloadsFuturesHistoricalData(string ticker, SecurityType securityType, string market, Resolution resolution, TickType tickType) + public void DownloadsQuoteTicksForLeanFuture() { - var symbol = Symbol.Create(ticker, securityType, market); - var startTime = new DateTime(2024, 1, 15); - var endTime = new DateTime(2024, 1, 16); - var param = new DataDownloaderGetParameters(symbol, resolution, startTime, endTime, tickType); - - var downloadResponse = _downloader.Get(param).ToList(); + var symbol = CreateEsFuture(); + var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; - Log.Trace($"Downloaded {downloadResponse.Count} data points for futures {symbol}"); + var startUtc = new DateTime(2024, 5, 1, 9, 30, 0, DateTimeKind.Utc); + var endUtc = startUtc.AddMinutes(15); - Assert.IsTrue(downloadResponse.Any(), "Expected to download futures data"); + var parameters = new DataDownloaderGetParameters( + symbol, + Resolution.Tick, + startUtc, + endUtc, + TickType.Quote + ); - foreach (var data in downloadResponse) - { - Assert.AreEqual(symbol, data.Symbol, "Symbol should match requested futures symbol"); - Assert.Greater(data.Value, 0, "Data value should be positive"); - } - } - - [Test] - [TestCase("ESM3", SecurityType.Future, Market.CME, Resolution.Tick, TickType.Quote)] - [Explicit("This test requires a configured DataBento API key and advanced subscription")] - public void DownloadsQuoteData(string ticker, SecurityType securityType, string market, Resolution resolution, TickType tickType) - { - var symbol = Symbol.Create(ticker, securityType, market); - var startTime = new DateTime(2024, 1, 15, 9, 30, 0); - var endTime = new DateTime(2024, 1, 15, 9, 45, 0); - var param = new DataDownloaderGetParameters(symbol, resolution, startTime, endTime, tickType); + var data = _downloader.Get(parameters).ToList(); - var downloadResponse = _downloader.Get(param).ToList(); + Log.Trace($"Downloaded {data.Count} quote ticks for {symbol}"); - Log.Trace($"Downloaded {downloadResponse.Count} quote data points for {symbol}"); + Assert.IsNotEmpty(data); - Assert.IsTrue(downloadResponse.Any(), "Expected to download quote data"); + var startExchange = startUtc.ConvertFromUtc(exchangeTimeZone); + var endExchange = endUtc.ConvertFromUtc(exchangeTimeZone); - foreach (var data in downloadResponse) + foreach (var point in data) { - Assert.AreEqual(symbol, data.Symbol, "Symbol should match requested symbol"); - if (data is QuoteBar quoteBar) + Assert.AreEqual(symbol, point.Symbol); + Assert.That(point.Time, Is.InRange(startExchange, endExchange)); + + if (point is Tick tick) + { + Assert.AreEqual(TickType.Quote, tick.TickType); + Assert.IsTrue( + tick.BidPrice > 0 || tick.AskPrice > 0, + "Quote tick must have bid or ask" + ); + } + else if (point is QuoteBar bar) { - Assert.IsTrue(quoteBar.Bid != null || quoteBar.Ask != null, "Quote should have bid or ask data"); + Assert.IsTrue(bar.Bid != null || bar.Ask != null); } } } [Test] - [Explicit("This test requires a configured DataBento API key")] public void DataIsSortedByTime() { - var symbol = Symbol.Create("ESM3", SecurityType.Future, Market.CME); - var startTime = new DateTime(2024, 1, 15); - var endTime = new DateTime(2024, 1, 16); - var param = new DataDownloaderGetParameters(symbol, Resolution.Minute, startTime, endTime, TickType.Trade); + var symbol = CreateEsFuture(); - var downloadResponse = _downloader.Get(param).ToList(); + var startUtc = new DateTime(2024, 5, 1, 0, 0, 0, DateTimeKind.Utc); + var endUtc = new DateTime(2024, 5, 2, 0, 0, 0, DateTimeKind.Utc); - Assert.IsTrue(downloadResponse.Any(), "Expected to download data for time sorting test"); + var parameters = new DataDownloaderGetParameters( + symbol, + Resolution.Minute, + startUtc, + endUtc, + TickType.Trade + ); - for (int i = 1; i < downloadResponse.Count; i++) + var data = _downloader.Get(parameters).ToList(); + + Assert.IsNotEmpty(data); + + for (int i = 1; i < data.Count; i++) { - Assert.GreaterOrEqual(downloadResponse[i].Time, downloadResponse[i - 1].Time, - $"Data should be sorted by time. Item {i} time {downloadResponse[i].Time} should be >= item {i - 1} time {downloadResponse[i - 1].Time}"); + Assert.GreaterOrEqual( + data[i].Time, + data[i - 1].Time, + $"Data not sorted at index {i}" + ); } } [Test] - public void DisposesCorrectly() + public void DisposeIsIdempotent() { - var downloader = new DataBentoDataDownloader(); - Assert.DoesNotThrow(() => downloader.Dispose(), "Dispose should not throw"); - Assert.DoesNotThrow(() => downloader.Dispose(), "Multiple dispose calls should not throw"); + var downloader = new DataBentoDataDownloader(ApiKey, + MarketHoursDatabase.FromDataFolder()); + + Assert.DoesNotThrow(downloader.Dispose); + Assert.DoesNotThrow(downloader.Dispose); } } } diff --git a/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs index 2da5b8f..94c78f8 100644 --- a/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs @@ -32,12 +32,19 @@ namespace QuantConnect.Lean.DataSource.DataBento.Tests public class DataBentoDataProviderHistoryTests { private DataBentoProvider _historyDataProvider; - private readonly string _apiKey = Config.Get("databento-api-key"); + private MarketHoursDatabase _marketHoursDatabase; + protected readonly string ApiKey = Config.Get("databento-api-key"); + + private static Symbol CreateEsFuture() + { + var expiration = new DateTime(2026, 3, 20); + return Symbol.CreateFuture("ES", Market.CME, expiration); + } [SetUp] public void SetUp() { - _historyDataProvider = new DataBentoProvider(_apiKey); + _historyDataProvider = new DataBentoProvider(); } [TearDown] @@ -50,150 +57,99 @@ internal static IEnumerable TestParameters { get { + var es = CreateEsFuture(); - // DataBento futures - var esMini = Symbol.Create("ESM3", SecurityType.Future, Market.CME); - var znNote = Symbol.Create("ZNM3", SecurityType.Future, Market.CME); - var gcGold = Symbol.Create("GCM3", SecurityType.Future, Market.CME); - - // test cases for supported futures - yield return new TestCaseData(esMini, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(5), false) - .SetDescription("Valid ES futures - Daily resolution, 5 days period") - .SetCategory("Valid"); - - yield return new TestCaseData(esMini, Resolution.Hour, TickType.Trade, TimeSpan.FromDays(2), false) - .SetDescription("Valid ES futures - Hour resolution, 2 days period") - .SetCategory("Valid"); - - yield return new TestCaseData(esMini, Resolution.Minute, TickType.Trade, TimeSpan.FromHours(4), false) - .SetDescription("Valid ES futures - Minute resolution, 4 hours period") + yield return new TestCaseData(es, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(5), false) + .SetDescription("ES futures daily trade history") .SetCategory("Valid"); - yield return new TestCaseData(znNote, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(3), false) - .SetDescription("Valid ZN futures - Daily resolution, 3 days period") + yield return new TestCaseData(es, Resolution.Hour, TickType.Trade, TimeSpan.FromDays(2), false) + .SetDescription("ES futures hourly trade history") .SetCategory("Valid"); - yield return new TestCaseData(gcGold, Resolution.Hour, TickType.Trade, TimeSpan.FromDays(1), false) - .SetDescription("Valid GC futures - Hour resolution, 1 day period") + yield return new TestCaseData(es, Resolution.Minute, TickType.Trade, TimeSpan.FromHours(4), false) + .SetDescription("ES futures minute trade history") .SetCategory("Valid"); - // Test cases for quote data (may require advanced subscription) - yield return new TestCaseData(esMini, Resolution.Tick, TickType.Quote, TimeSpan.FromMinutes(15), false) - .SetDescription("ES futures quote data - Tick resolution") + yield return new TestCaseData(es, Resolution.Tick, TickType.Quote, TimeSpan.FromMinutes(15), false) + .SetDescription("ES futures quote ticks") .SetCategory("Quote"); - - // Unsupported security types - var equity = Symbol.Create("SPY", SecurityType.Equity, Market.USA); - var option = Symbol.Create("SPY", SecurityType.Option, Market.USA); - - yield return new TestCaseData(equity, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(5), true) - .SetDescription("Invalid - Equity not supported by DataBento") - .SetCategory("Invalid"); - - yield return new TestCaseData(option, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(5), true) - .SetDescription("Invalid - Option not supported by DataBento") - .SetCategory("Invalid"); } } [Test, TestCaseSource(nameof(TestParameters))] - [Explicit("This test requires a configured DataBento API key")] public void GetsHistory(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period, bool expectsNoData) { var request = GetHistoryRequest(resolution, tickType, symbol, period); - try + var history = _historyDataProvider.GetHistory(request); + + if (expectsNoData) { - var slices = _historyDataProvider.GetHistory(request)?.Select(data => new Slice(data.Time, new[] { data }, data.Time.ConvertToUtc(request.DataTimeZone))).ToList(); + Assert.IsTrue(history == null || !history.Any(), + $"Expected no data for unsupported symbol: {symbol}"); + return; + } - if (expectsNoData) - { - Assert.IsTrue(slices == null || !slices.Any(), - $"Expected no data for unsupported symbol/security type: {symbol}"); - } - else + Assert.IsNotNull(history); + var data = history.ToList(); + Assert.IsNotEmpty(data); + + Log.Trace($"Received {data.Count} data points for {symbol} @ {resolution}"); + + foreach (var point in data.Take(5)) + { + Assert.AreEqual(symbol, point.Symbol); + + if (point is TradeBar bar) { - Assert.IsNotNull(slices, "Expected to receive history data"); - - if (slices.Any()) - { - Log.Trace($"Received {slices.Count} slices for {symbol} at {resolution} resolution"); - - foreach (var slice in slices.Take(5)) // Check first 5 slices - { - Assert.IsNotNull(slice, "Slice should not be null"); - Assert.IsTrue(slice.Time >= request.StartTimeUtc && slice.Time <= request.EndTimeUtc, - "Slice time should be within requested range"); - - if (slice.Bars.ContainsKey(symbol)) - { - var bar = slice.Bars[symbol]; - Assert.Greater(bar.Close, 0, "Bar close price should be positive"); - Assert.GreaterOrEqual(bar.Volume, 0, "Bar volume should be non-negative"); - } - } - } + Assert.Greater(bar.Close, 0); + Assert.GreaterOrEqual(bar.Volume, 0); } - } - catch (Exception ex) - { - Log.Error($"Error getting history for {symbol}: {ex.Message}"); - if (!expectsNoData) + if (point is Tick tick && tickType == TickType.Quote) { - throw; + Assert.IsTrue(tick.BidPrice > 0 || tick.AskPrice > 0); } } } [Test] - [Explicit("This test requires a configured DataBento API key")] public void GetHistoryWithMultipleSymbols() { - var symbol1 = Symbol.Create("ESM3", SecurityType.Future, Market.CME); - var symbol2 = Symbol.Create("ZNM3", SecurityType.Future, Market.CME); - - var request1 = GetHistoryRequest(Resolution.Daily, TickType.Trade, symbol1, TimeSpan.FromDays(3)); - var request2 = GetHistoryRequest(Resolution.Daily, TickType.Trade, symbol2, TimeSpan.FromDays(3)); - - var history1 = _historyDataProvider.GetHistory(request1); - var history2 = _historyDataProvider.GetHistory(request2); + var es = CreateEsFuture(); - var allData = new List(); - if (history1 != null) allData.AddRange(history1); - if (history2 != null) allData.AddRange(history2); + var request = GetHistoryRequest(Resolution.Daily, TickType.Trade, es, TimeSpan.FromDays(3)); - // timezone from the first request - var slices = allData.GroupBy(d => d.Time) - .Select(g => new Slice(g.Key, g.ToList(), g.Key.ConvertToUtc(request1.DataTimeZone))) - .ToList(); + var history = _historyDataProvider.GetHistory(request)?.ToList(); - Assert.IsNotNull(slices, "Expected to receive history data for multiple symbols"); - - if (slices.Any()) - { - Log.Trace($"Received {slices.Count} slices for multiple symbols"); - - var hasSymbol1Data = slices.Any(s => s.Bars.ContainsKey(symbol1)); - var hasSymbol2Data = slices.Any(s => s.Bars.ContainsKey(symbol2)); - - Assert.IsTrue(hasSymbol1Data || hasSymbol2Data, - "Expected data for at least one of the requested symbols"); - } + Assert.IsTrue( + history != null && history.Any(), + "Expected history for ES" + ); } - internal static HistoryRequest GetHistoryRequest(Resolution resolution, TickType tickType, Symbol symbol, TimeSpan period) + internal static HistoryRequest GetHistoryRequest( + Resolution resolution, + TickType tickType, + Symbol symbol, + TimeSpan period) { - var utcNow = DateTime.UtcNow; + var endUtc = new DateTime(2024, 5, 10, 0, 0, 0, DateTimeKind.Utc); + var startUtc = endUtc - period; + var dataType = LeanData.GetDataType(resolution, tickType); var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); - var exchangeHours = marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType); - var dataTimeZone = marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType); + var exchangeHours = marketHoursDatabase.GetExchangeHours( + symbol.ID.Market, symbol, symbol.SecurityType); + + var dataTimeZone = marketHoursDatabase.GetDataTimeZone( + symbol.ID.Market, symbol, symbol.SecurityType); return new HistoryRequest( - startTimeUtc: utcNow.Add(-period), - endTimeUtc: utcNow, + startTimeUtc: startUtc, + endTimeUtc: endUtc, dataType: dataType, symbol: symbol, resolution: resolution, @@ -204,7 +160,7 @@ internal static HistoryRequest GetHistoryRequest(Resolution resolution, TickType isCustomData: false, DataNormalizationMode.Raw, tickType: tickType - ); + ); } } } diff --git a/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs index 7df71a4..22e7f04 100644 --- a/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs @@ -16,26 +16,32 @@ using System; using System.Threading; -using System.Threading.Tasks; using NUnit.Framework; +using QuantConnect.Configuration; using QuantConnect.Data; using QuantConnect.Data.Market; using QuantConnect.Lean.DataSource.DataBento; using QuantConnect.Logging; -using QuantConnect.Configuration; namespace QuantConnect.Lean.DataSource.DataBento.Tests { [TestFixture] - public class DataBentoRawLiveClientTests + public class DataBentoRawLiveClientSyncTests { - private DatabentoRawClient _client; - private readonly string _apiKey = Config.Get("databento-api-key"); + private DataBentoRawLiveClient _client; + protected readonly string ApiKey = Config.Get("databento-api-key"); + + private static Symbol CreateEsFuture() + { + var expiration = new DateTime(2026, 3, 20); + return Symbol.CreateFuture("ES", Market.CME, expiration); + } [SetUp] public void SetUp() { - _client = new DatabentoRawClient(_apiKey); + Log.Trace("DataBentoLiveClientTests: Using API Key: " + ApiKey); + _client = new DataBentoRawLiveClient(ApiKey); } [TearDown] @@ -45,208 +51,97 @@ public void TearDown() } [Test] - [Explicit("This test requires a configured DataBento API key and live connection")] - public async Task ConnectsToGateway() + public void Connects() { - if (string.IsNullOrEmpty(_apiKey)) - { - Assert.Ignore("DataBento API key not configured"); - return; - } - - var connected = await _client.ConnectAsync(); + var connected = _client.Connect(); - Assert.IsTrue(connected, "Should successfully connect to DataBento gateway"); - Assert.IsTrue(_client.IsConnected, "IsConnected should return true after successful connection"); + Assert.IsTrue(connected); + Assert.IsTrue(_client.IsConnected); - Log.Trace("Successfully connected to DataBento gateway"); + Log.Trace("Connected successfully"); } [Test] - [Explicit("This test requires a configured DataBento API key and live connection")] - public async Task SubscribesToSymbol() + public void SubscribesToLeanFutureSymbol() { - if (string.IsNullOrEmpty(_apiKey)) - { - Assert.Ignore("DataBento API key not configured"); - return; - } - - var connected = await _client.ConnectAsync(); - Assert.IsTrue(connected, "Must be connected to test subscription"); + Assert.IsTrue(_client.Connect()); - var symbol = Symbol.Create("ESM3", SecurityType.Future, Market.CME); - var subscribed = _client.Subscribe(symbol, Resolution.Minute, TickType.Trade); + var symbol = CreateEsFuture(); - Assert.IsTrue(subscribed, "Should successfully subscribe to symbol"); + Assert.IsTrue(_client.Subscribe(symbol, TickType.Trade)); + Assert.IsTrue(_client.StartSession()); - Log.Trace($"Successfully subscribed to {symbol}"); + Thread.Sleep(1000); - // Wait a moment to ensure subscription is active - await Task.Delay(2000); - - var unsubscribed = _client.Unsubscribe(symbol); - Assert.IsTrue(unsubscribed, "Should successfully unsubscribe from symbol"); - - Log.Trace($"Successfully unsubscribed from {symbol}"); + Assert.IsTrue(_client.Unsubscribe(symbol)); } [Test] - [Explicit("This test requires a configured DataBento API key and live connection")] - public async Task ReceivesLiveData() + public void ReceivesTradeOrQuoteTicks() { - if (string.IsNullOrEmpty(_apiKey)) - { - Assert.Ignore("DataBento API key not configured"); - return; - } - - var dataReceived = false; - var dataReceivedEvent = new ManualResetEventSlim(false); - BaseData receivedData = null; + var receivedEvent = new ManualResetEventSlim(false); + BaseData received = null; - _client.DataReceived += (sender, data) => + _client.DataReceived += (_, data) => { - receivedData = data; - dataReceived = true; - dataReceivedEvent.Set(); - Log.Trace($"Received data: {data}"); + received = data; + receivedEvent.Set(); }; - var connected = await _client.ConnectAsync(); - Assert.IsTrue(connected, "Must be connected to test data reception"); + Assert.IsTrue(_client.Connect()); - var symbol = Symbol.Create("ESM3", SecurityType.Future, Market.CME); - var subscribed = _client.Subscribe(symbol, Resolution.Tick, TickType.Trade); - Assert.IsTrue(subscribed, "Must be subscribed to receive data"); + var symbol = CreateEsFuture(); - // Wait for data with timeout - var dataReceiptTimeout = TimeSpan.FromMinutes(2); - var receivedWithinTimeout = dataReceivedEvent.Wait(dataReceiptTimeout); + Assert.IsTrue(_client.Subscribe(symbol, TickType.Trade)); + Assert.IsTrue(_client.StartSession()); - if (receivedWithinTimeout) - { - Assert.IsTrue(dataReceived, "Should have received data"); - Assert.IsNotNull(receivedData, "Received data should not be null"); - Assert.AreEqual(symbol, receivedData.Symbol, "Received data symbol should match subscription"); - Assert.Greater(receivedData.Value, 0, "Received data value should be positive"); + var gotData = receivedEvent.Wait(TimeSpan.FromMinutes(2)); - Log.Trace($"Successfully received live data: {receivedData}"); - } - else + if (!gotData) { - Log.Trace("No data received within timeout period - this may be expected during non-market hours"); + Assert.Inconclusive("No data received (likely outside market hours)"); + return; } - _client.Unsubscribe(symbol); - } + Assert.NotNull(received); + Assert.AreEqual(symbol, received.Symbol); - [Test] - [Explicit("This test requires a configured DataBento API key and live connection")] - public async Task HandlesConnectionEvents() - { - if (string.IsNullOrEmpty(_apiKey)) + if (received is Tick tick) { - Assert.Ignore("DataBento API key not configured"); - return; + Assert.Greater(tick.Time, DateTime.MinValue); + Assert.Greater(tick.Value, 0); } - - var connectionStatusChanged = false; - var connectionStatusEvent = new ManualResetEventSlim(false); - - _client.ConnectionStatusChanged += (sender, isConnected) => + else if (received is TradeBar bar) { - connectionStatusChanged = true; - connectionStatusEvent.Set(); - Log.Trace($"Connection status changed: {isConnected}"); - }; - - var connected = await _client.ConnectAsync(); - Assert.IsTrue(connected, "Should connect successfully"); - - // Connection status event should fire on connect - var eventFiredWithinTimeout = connectionStatusEvent.Wait(TimeSpan.FromSeconds(10)); - Assert.IsTrue(eventFiredWithinTimeout || connectionStatusChanged, - "Connection status changed event should fire"); - - _client.Disconnect(); - Assert.IsFalse(_client.IsConnected, "Should be disconnected after calling Disconnect()"); - } - - [Test] - public void HandlesInvalidApiKey() - { - var invalidClient = new DatabentoRawClient("invalid-api-key"); - - // Connection with invalid API key should fail gracefully - Assert.DoesNotThrowAsync(async () => + Assert.Greater(bar.Close, 0); + } + else { - var connected = await invalidClient.ConnectAsync(); - Assert.IsFalse(connected, "Connection should fail with invalid API key"); - }); - - invalidClient.Dispose(); + Assert.Fail($"Unexpected data type: {received.GetType()}"); + } } [Test] - public void DisposesCorrectly() + public void DisposeIsIdempotent() { - var client = new DatabentoRawClient(_apiKey); - Assert.DoesNotThrow(() => client.Dispose(), "Dispose should not throw"); - Assert.DoesNotThrow(() => client.Dispose(), "Multiple dispose calls should not throw"); + var client = new DataBentoRawLiveClient(ApiKey); + Assert.DoesNotThrow(client.Dispose); + Assert.DoesNotThrow(client.Dispose); } [Test] - public void SymbolMappingWorksCorrectly() + public void SymbolMappingDoesNotThrow() { - // Test that futures are mapped correctly to DataBento format - var esFuture = Symbol.Create("ESM3", SecurityType.Future, Market.CME); + Assert.IsTrue(_client.Connect()); - // Since the mapping method is private, we test indirectly through subscription - Assert.DoesNotThrowAsync(async () => - { - if (!string.IsNullOrEmpty(_apiKey)) - { - var connected = await _client.ConnectAsync(); - if (connected) - { - _client.Subscribe(esFuture, Resolution.Minute, TickType.Trade); - _client.Unsubscribe(esFuture); - } - } - }); - } - - [Test] - public void SchemaResolutionMappingWorksCorrectly() - { - // Test that resolution mappings work correctly - var symbol = Symbol.Create("ESM3", SecurityType.Future, Market.CME); + var symbol = CreateEsFuture(); - Assert.DoesNotThrowAsync(async () => + Assert.DoesNotThrow(() => { - if (!string.IsNullOrEmpty(_apiKey)) - { - var connected = await _client.ConnectAsync(); - if (connected) - { - // Test different resolutions - _client.Subscribe(symbol, Resolution.Tick, TickType.Trade); - _client.Unsubscribe(symbol); - - _client.Subscribe(symbol, Resolution.Second, TickType.Trade); - _client.Unsubscribe(symbol); - - _client.Subscribe(symbol, Resolution.Minute, TickType.Trade); - _client.Unsubscribe(symbol); - - _client.Subscribe(symbol, Resolution.Hour, TickType.Trade); - _client.Unsubscribe(symbol); - - _client.Subscribe(symbol, Resolution.Daily, TickType.Trade); - _client.Unsubscribe(symbol); - } - } + _client.Subscribe(symbol, TickType.Trade); + _client.StartSession(); + Thread.Sleep(500); + _client.Unsubscribe(symbol); }); } } diff --git a/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj b/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj index d2d7ef1..f5fce66 100644 --- a/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj +++ b/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj @@ -15,7 +15,6 @@ - @@ -24,10 +23,9 @@ - - + PreserveNewest diff --git a/QuantConnect.DataBento.Tests/TestSetup.cs b/QuantConnect.DataBento.Tests/TestSetup.cs index b0e07c1..39cb9de 100644 --- a/QuantConnect.DataBento.Tests/TestSetup.cs +++ b/QuantConnect.DataBento.Tests/TestSetup.cs @@ -62,22 +62,5 @@ private static void ReloadConfiguration() // resets the version among other things Globals.Reset(); } - - private static void SetUp() - { - Log.LogHandler = new CompositeLogHandler(); - Log.Trace("TestSetup(): starting..."); - ReloadConfiguration(); - Log.DebuggingEnabled = Config.GetBool("debug-mode"); - } - - private static TestCaseData[] TestParameters - { - get - { - SetUp(); - return new [] { new TestCaseData() }; - } - } } } diff --git a/QuantConnect.DataBento.Tests/config.json b/QuantConnect.DataBento.Tests/config.json index ff861ca..7256aba 100644 --- a/QuantConnect.DataBento.Tests/config.json +++ b/QuantConnect.DataBento.Tests/config.json @@ -1,5 +1,5 @@ { - "data-folder":"../../../../../Lean/Data/", + "data-folder":"../../../../../Data/", "job-user-id": "0", "api-access-token": "", diff --git a/QuantConnect.DataBento/DataBentoDataDownloader.cs b/QuantConnect.DataBento/DataBentoDataDownloader.cs index 2a44e55..841d070 100644 --- a/QuantConnect.DataBento/DataBentoDataDownloader.cs +++ b/QuantConnect.DataBento/DataBentoDataDownloader.cs @@ -15,19 +15,19 @@ */ using System; +using System.Collections.Generic; +using System.Globalization; +using NodaTime; using System.IO; -using System.Text; +using System.Linq; using System.Net.Http; -using System.Net.Http.Headers; -using System.Globalization; -using System.Collections.Generic; +using System.Text; using CsvHelper; -using CsvHelper.Configuration.Attributes; +using QuantConnect.Configuration; using QuantConnect.Data; using QuantConnect.Data.Market; +using QuantConnect.Lean.DataSource.DataBento.Models; using QuantConnect.Util; -using QuantConnect.Configuration; -using QuantConnect.Interfaces; using QuantConnect.Securities; namespace QuantConnect.Lean.DataSource.DataBento @@ -38,32 +38,28 @@ namespace QuantConnect.Lean.DataSource.DataBento /// public class DataBentoDataDownloader : IDataDownloader, IDisposable { - private readonly HttpClient _httpClient; + private readonly HttpClient _httpClient = new(); private readonly string _apiKey; - private const decimal PriceScaleFactor = 1e-9m; + private readonly DataBentoSymbolMapper _symbolMapper; + private readonly MarketHoursDatabase _marketHoursDatabase; + private readonly Dictionary _symbolExchangeTimeZones = new(); /// /// Initializes a new instance of the /// - public DataBentoDataDownloader(string apiKey) + /// The DataBento API key. + public DataBentoDataDownloader(string apiKey, MarketHoursDatabase marketHoursDatabase) { + _marketHoursDatabase = marketHoursDatabase; _apiKey = apiKey; - _httpClient = new HttpClient(); - - // Set up HTTP Basic Authentication - var credentials = Convert.ToBase64String(Encoding.ASCII.GetBytes($"{_apiKey}:")); - _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", credentials); - } - - public DataBentoDataDownloader() - : this(Config.Get("databento-api-key")) - { + _httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{_apiKey}:"))); + _symbolMapper = new DataBentoSymbolMapper(); } /// /// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC). /// - /// Parameters for the historical data request + /// Parameters for the historical data request /// Enumerable of base data for this symbol /// public IEnumerable Get(DataDownloaderGetParameters parameters) @@ -72,25 +68,29 @@ public IEnumerable Get(DataDownloaderGetParameters parameters) var resolution = parameters.Resolution; var tickType = parameters.TickType; - var dataset = "GLBX.MDP3"; // hard coded for now. Later on can add equities and options with different mapping + /// + /// Dataset for CME Globex futures + /// https://databento.com/docs/venues-and-datasets has more information on datasets through DataBento + /// + const string dataset = "GLBX.MDP3"; // hard coded for now. Later on can add equities and options with different mapping var schema = GetSchema(resolution, tickType); - var dbSymbol = MapSymbolToDataBento(symbol); + var databentoSymbol = _symbolMapper.GetBrokerageSymbol(symbol); // prepare body for Raw HTTP request - var body = new StringBuilder(); - body.Append($"dataset={dataset}"); - body.Append($"&symbols={dbSymbol}"); - body.Append($"&schema={schema}"); - body.Append($"&start={parameters.StartUtc:yyyy-MM-ddTHH:mm}"); - body.Append($"&end={parameters.EndUtc:yyyy-MM-ddTHH:mm}"); - body.Append("&stype_in=parent"); - body.Append("&encoding=csv"); - - var request = new HttpRequestMessage( - HttpMethod.Post, + var body = new StringBuilder() + .Append($"dataset={dataset}") + .Append($"&symbols={databentoSymbol}") + .Append($"&schema={schema}") + .Append($"&start={parameters.StartUtc:yyyy-MM-ddTHH:mm}") + .Append($"&end={parameters.EndUtc:yyyy-MM-ddTHH:mm}") + .Append("&stype_in=parent") + .Append("&encoding=csv") + .ToString(); + + using var request = new HttpRequestMessage(HttpMethod.Post, "https://hist.databento.com/v0/timeseries.get_range") { - Content = new StringContent(body.ToString(), Encoding.UTF8, "application/x-www-form-urlencoded") + Content = new StringContent(body, Encoding.UTF8, "application/x-www-form-urlencoded") }; // send the request with the get range url @@ -99,85 +99,70 @@ public IEnumerable Get(DataDownloaderGetParameters parameters) // Add error handling to see the actual error message if (!response.IsSuccessStatusCode) { - var errorContent = response.Content.ReadAsStringAsync().Result; + var errorContent = response.Content.ReadAsStringAsync().SynchronouslyAwaitTaskResult(); throw new HttpRequestException($"DataBento API error ({response.StatusCode}): {errorContent}"); } - response.EnsureSuccessStatusCode(); - - using var stream = response.Content.ReadAsStream(); - using var reader = new StreamReader(stream); - using var csv = new CsvReader(reader, CultureInfo.InvariantCulture); + using var csv = new CsvReader( + new StreamReader(response.Content.ReadAsStream()), + CultureInfo.InvariantCulture + ); - if (tickType == TickType.Trade) + return (tickType, resolution) switch { - if (resolution == Resolution.Tick) - { - // For tick data, use the trades schema which returns individual trades - foreach (var record in csv.GetRecords()) - { - yield return new Tick - { - Time = record.Timestamp, - Symbol = symbol, - Value = record.Price, - Quantity = record.Size - }; - } - } - else - { - // For aggregated data, use the ohlcv schema which returns bars - foreach (var record in csv.GetRecords()) - { - yield return new TradeBar - { - Symbol = symbol, - Time = record.Timestamp, - Open = record.Open, - High = record.High, - Low = record.Low, - Close = record.Close, - Volume = record.Volume - }; - } - } - } - else if (tickType == TickType.Quote) - { - foreach (var record in csv.GetRecords()) - { - var bidPrice = record.BidPrice * PriceScaleFactor; - var askPrice = record.AskPrice * PriceScaleFactor; - - if (resolution == Resolution.Tick) - { - yield return new Tick + (TickType.Trade, Resolution.Tick) => + csv.ForEach(dt => + new Tick( + GetTickTime(symbol, dt.Timestamp), + symbol, + string.Empty, + string.Empty, + dt.Size, + dt.Price + ) + ), + + (TickType.Trade, _) => + csv.ForEach(bar => + new TradeBar( + GetTickTime(symbol, bar.Timestamp), + symbol, + bar.Open, + bar.High, + bar.Low, + bar.Close, + bar.Volume + ) + ), + + (TickType.Quote, Resolution.Tick) => + csv.ForEach(q => + new Tick( + GetTickTime(symbol, q.Timestamp), + symbol, + bidPrice: q.BidPrice, + askPrice: q.AskPrice, + bidSize: q.BidSize, + askSize: q.AskSize + ) { - Time = record.Timestamp, - Symbol = symbol, - AskPrice = askPrice, - BidPrice = bidPrice, - AskSize = record.AskSize, - BidSize = record.BidSize, TickType = TickType.Quote - }; - } - else - { - var bidBar = new Bar(bidPrice, bidPrice, bidPrice, bidPrice); - var askBar = new Bar(askPrice, askPrice, askPrice, askPrice); - yield return new QuoteBar( - record.Timestamp, + } + ), + + (TickType.Quote, _) => + csv.ForEach(q => + new QuoteBar( + GetTickTime(symbol, q.Timestamp), symbol, - bidBar, - record.BidSize, - askBar, - record.AskSize - ); - } - } - } + new Bar(q.BidPrice, q.BidPrice, q.BidPrice, q.BidPrice), q.BidSize, + new Bar(q.AskPrice, q.AskPrice, q.AskPrice, q.AskPrice), q.AskSize + ) + ), + + _ => throw new NotSupportedException( + $"Unsupported tickType={tickType} resolution={resolution}") + }; } /// @@ -191,113 +176,60 @@ public void Dispose() /// /// Pick Databento schema from Lean resolution/ticktype /// - private string GetSchema(Resolution resolution, TickType tickType) + private static string GetSchema(Resolution resolution, TickType tickType) { - if (tickType == TickType.Trade) + return (tickType, resolution) switch { - if (resolution == Resolution.Tick) - return "trades"; - if (resolution == Resolution.Second) - return "ohlcv-1s"; - if (resolution == Resolution.Minute) - return "ohlcv-1m"; - if (resolution == Resolution.Hour) - return "ohlcv-1h"; - if (resolution == Resolution.Daily) - return "ohlcv-1d"; - } - else if (tickType == TickType.Quote) - { - // top of book - if (resolution == Resolution.Tick || resolution == Resolution.Second || resolution == Resolution.Minute || resolution == Resolution.Hour || resolution == Resolution.Daily) - return "mbp-1"; - } + (TickType.Trade, Resolution.Tick) => "mbp-1", + (TickType.Trade, Resolution.Second) => "ohlcv-1s", + (TickType.Trade, Resolution.Minute) => "ohlcv-1m", + (TickType.Trade, Resolution.Hour) => "ohlcv-1h", + (TickType.Trade, Resolution.Daily) => "ohlcv-1d", - throw new NotSupportedException($"Unsupported resolution {resolution} / {tickType}"); + (TickType.Quote, _) => "mbp-1", + + _ => throw new NotSupportedException( + $"Unsupported resolution {resolution} / {tickType}" + ) + }; } /// - /// Maps a LEAN symbol to DataBento symbol format + /// Converts the given UTC time into the symbol security exchange time zone /// - private string MapSymbolToDataBento(Symbol symbol) + private DateTime GetTickTime(Symbol symbol, DateTime utcTime) { - if (symbol.SecurityType == SecurityType.Future) + DateTimeZone exchangeTimeZone; + lock (_symbolExchangeTimeZones) { - // For DataBento, use the root symbol with .FUT suffix for parent subscription - // ES19Z25 -> ES.FUT - var value = symbol.Value; - - // Extract root by removing digits and month codes - var root = new string(value.TakeWhile(c => !char.IsDigit(c)).ToArray()); + if (!_symbolExchangeTimeZones.TryGetValue(symbol, out exchangeTimeZone)) + { + // read the exchange time zone from market-hours-database + if (_marketHoursDatabase.TryGetEntry(symbol.ID.Market, symbol, symbol.SecurityType, out var entry)) + { + exchangeTimeZone = entry.ExchangeHours.TimeZone; + } + // If there is no entry for the given Symbol, default to New York + else + { + exchangeTimeZone = TimeZones.NewYork; + } - return $"{root}.FUT"; + _symbolExchangeTimeZones.Add(symbol, exchangeTimeZone); + } } - return symbol.Value; - } - - /// Class for parsing trade data from Databento - /// Really used as a map from the http request to then get it in QC data structures - private class DatabentoBar - { - [Name("ts_event")] - public long TimestampNanos { get; set; } - - public DateTime Timestamp => DateTimeOffset.FromUnixTimeSeconds(TimestampNanos / 1_000_000_000) - .AddTicks((TimestampNanos % 1_000_000_000) / 100).UtcDateTime; - - [Name("open")] - public decimal Open { get; set; } - - [Name("high")] - public decimal High { get; set; } - - [Name("low")] - public decimal Low { get; set; } - - [Name("close")] - public decimal Close { get; set; } - - [Name("volume")] - public decimal Volume { get; set; } - } - - private class DatabentoTrade - { - [Name("ts_event")] - public long TimestampNanos { get; set; } - - public DateTime Timestamp => DateTimeOffset.FromUnixTimeSeconds(TimestampNanos / 1_000_000_000) - .AddTicks((TimestampNanos % 1_000_000_000) / 100).UtcDateTime; - - [Name("price")] - public long PriceRaw { get; set; } - - public decimal Price => PriceRaw * PriceScaleFactor; - - [Name("size")] - public int Size { get; set; } + return utcTime.ConvertFromUtc(exchangeTimeZone); } + } +} - private class DatabentoQuote - { - [Name("ts_event")] - public long TimestampNanos { get; set; } - - public DateTime Timestamp => DateTimeOffset.FromUnixTimeSeconds(TimestampNanos / 1_000_000_000) - .AddTicks((TimestampNanos % 1_000_000_000) / 100).UtcDateTime; - - [Name("bid_px_00")] - public long BidPrice { get; set; } - - [Name("bid_sz_00")] - public int BidSize { get; set; } - - [Name("ask_px_00")] - public long AskPrice { get; set; } - - [Name("ask_sz_00")] - public int AskSize { get; set; } - } +public static class CsvReaderExtensions +{ + public static IEnumerable ForEach( + this CsvReader csv, + Func map) + { + return csv.GetRecords().Select(map).ToList(); } } diff --git a/QuantConnect.DataBento/DataBentoDataProvider.cs b/QuantConnect.DataBento/DataBentoDataProvider.cs index 8cf6015..aa9dd32 100644 --- a/QuantConnect.DataBento/DataBentoDataProvider.cs +++ b/QuantConnect.DataBento/DataBentoDataProvider.cs @@ -14,14 +14,11 @@ * */ -using System; -using System.Linq; using NodaTime; using QuantConnect.Data; using QuantConnect.Data.Market; using QuantConnect.Util; using QuantConnect.Interfaces; -using System.Collections.Generic; using QuantConnect.Configuration; using QuantConnect.Logging; using QuantConnect.Packets; @@ -31,23 +28,24 @@ namespace QuantConnect.Lean.DataSource.DataBento { /// - /// Implementation of Custom Data Provider + /// A data Provider for DataBento that provides live market data and historical data. + /// Handles Subscribing, Unsubscribing, and fetching historical data from DataBento. + /// It will handle if a symbol is subscribable and will log errors if it is not. /// - public class DataBentoProvider : IDataQueueHandler + public partial class DataBentoProvider : IDataQueueHandler { private readonly IDataAggregator _dataAggregator = Composer.Instance.GetExportedValueByTypeName( Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false); - private EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager = null!; - private readonly List _activeSubscriptionConfigs = new(); - private readonly ConcurrentDictionary _subscriptionConfigs = new(); - private DatabentoRawClient _client = null!; - private readonly string _apiKey; + private EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager; + private DataBentoRawLiveClient _client; private readonly DataBentoDataDownloader _dataDownloader; private bool _potentialUnsupportedResolutionMessageLogged; private bool _sessionStarted = false; - private readonly object _sessionLock = new object(); + private readonly object _sessionLock = new(); private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); private readonly ConcurrentDictionary _symbolExchangeTimeZones = new(); + private bool _initialized; + private bool _unsupportedTickTypeMessagedLogged; /// /// Returns true if we're currently connected to the Data Provider @@ -59,109 +57,120 @@ public class DataBentoProvider : IDataQueueHandler /// public DataBentoProvider() { - _apiKey = Config.Get("databento-api-key"); - if (string.IsNullOrEmpty(_apiKey)) - { - throw new ArgumentException("DataBento API key is required. Set 'databento-api-key' in configuration."); - } - - _dataDownloader = new DataBentoDataDownloader(_apiKey); - Initialize(); - } - - /// - /// Initializes a new instance of the DataBentoProvider with custom API key - /// - /// DataBento API key - public DataBentoProvider(string apiKey) - { - _apiKey = apiKey ?? throw new ArgumentNullException(nameof(apiKey)); - _dataDownloader = new DataBentoDataDownloader(_apiKey); - Initialize(); + var apiKey = Config.Get("databento-api-key"); + _dataDownloader = new DataBentoDataDownloader(apiKey, _marketHoursDatabase); + Initialize(apiKey); } /// /// Common initialization logic + /// DataBento API key from config file retrieved on constructor /// - private void Initialize() + private void Initialize(string apiKey) { - Log.Trace("DataBentoProvider.Initialize(): Starting initialization"); - _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager(); - _subscriptionManager.SubscribeImpl = (symbols, tickType) => + Log.Debug("DataBentoProvider.Initialize(): Starting initialization"); + _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager() { - Log.Trace($"DataBentoProvider.SubscribeImpl(): Received subscription request for {symbols.Count()} symbols, TickType={tickType}"); - foreach (var symbol in symbols) + SubscribeImpl = (symbols, tickType) => { - Log.Trace($"DataBentoProvider.SubscribeImpl(): Processing symbol {symbol}"); - if (!_subscriptionConfigs.TryGetValue(symbol, out var config)) - { - Log.Error($"DataBentoProvider.SubscribeImpl(): No subscription config found for {symbol}"); - return false; - } - if (_client?.IsConnected != true) - { - Log.Error($"DataBentoProvider.SubscribeImpl(): Client is not connected. Cannot subscribe to {symbol}"); - return false; - } + return SubscriptionLogic(symbols, tickType); + }, + UnsubscribeImpl = (symbols, tickType) => + { + return UnsubscribeLogic(symbols, tickType); + } + }; + + // Initialize the live client + _client = new DataBentoRawLiveClient(apiKey); + _client.DataReceived += OnDataReceived; - var resolution = config.Resolution > Resolution.Tick ? Resolution.Tick : config.Resolution; - if (!_client.Subscribe(config.Symbol, resolution, config.TickType)) + // Connect to live gateway + Log.Debug("DataBentoProvider.Initialize(): Attempting connection to DataBento live gateway"); + var cancellationTokenSource = new CancellationTokenSource(); + Task.Factory.StartNew(() => + { + try + { + var connected = _client.Connect(); + Log.Debug($"DataBentoProvider.Initialize(): Connect() returned {connected}"); + + if (connected) { - Log.Error($"Failed to subscribe to {config.Symbol}"); - return false; + Log.Debug("DataBentoProvider.Initialize(): Successfully connected to DataBento live gateway"); } - - lock (_sessionLock) + else { - if (!_sessionStarted) - _sessionStarted = _client.StartSession(); + Log.Error("DataBentoProvider.Initialize(): Failed to connect to DataBento live gateway"); } } + catch (Exception ex) + { + Log.Error($"DataBentoProvider.Initialize(): Exception during Connect(): {ex.Message}\n{ex.StackTrace}"); + } + }, + cancellationTokenSource.Token, + TaskCreationOptions.LongRunning, + TaskScheduler.Default); + _initialized = true; - return true; - }; + Log.Debug("DataBentoProvider.Initialize(): Initialization complete"); + } - _subscriptionManager.UnsubscribeImpl = (symbols, tickType) => + /// + /// Logic to unsubscribe from the specified symbols + /// + public bool UnsubscribeLogic(IEnumerable symbols, TickType tickType) + { + foreach (var symbol in symbols) { - foreach (var symbol in symbols) + Log.Debug($"DataBentoProvider.UnsubscribeImpl(): Processing symbol {symbol}"); + if (_client?.IsConnected != true) { - Log.Trace($"DataBentoProvider.UnsubscribeImpl(): Processing symbol {symbol}"); - if (_client?.IsConnected != true) - { - throw new InvalidOperationException($"DataBentoProvider.UnsubscribeImpl(): Client is not connected. Cannot unsubscribe from {symbol}"); - } - - _client.Unsubscribe(symbol); + throw new InvalidOperationException($"DataBentoProvider.UnsubscribeImpl(): Client is not connected. Cannot unsubscribe from {symbol}"); } - return true; - }; + _client.Unsubscribe(symbol); + } - // Initialize the live client - Log.Trace("DataBentoProvider.Initialize(): Creating DatabentoRawClient"); - _client = new DatabentoRawClient(_apiKey); - _client.DataReceived += OnDataReceived; - _client.ConnectionStatusChanged += OnConnectionStatusChanged; + return true; + } - // Connect to live gateway - Log.Trace("DataBentoProvider.Initialize(): Attempting connection to DataBento live gateway"); - Task.Run(() => + /// + /// Logic to subscribe to the specified symbols + /// + public bool SubscriptionLogic(IEnumerable symbols, TickType tickType) + { + if (_client?.IsConnected != true) { - var connected = _client.Connect(); - Log.Trace($"DataBentoProvider.Initialize(): Connect() returned {connected}"); + Log.Error("DataBentoProvider.SubscriptionLogic(): Client is not connected. Cannot subscribe to symbols"); + return false; + } - if (connected) - { - Log.Trace("DataBentoProvider.Initialize(): Successfully connected to DataBento live gateway"); - } - else + foreach (var symbol in symbols) + { + if (!CanSubscribe(symbol)) { - Log.Error("DataBentoProvider.Initialize(): Failed to connect to DataBento live gateway"); + Log.Error($"DataBentoProvider.SubscriptionLogic(): Unsupported subscription: {symbol}"); + return false; } - }); + _client.Subscribe(symbol, tickType); + } - Log.Trace("DataBentoProvider.Initialize(): Initialization complete"); + return true; + } + + /// + /// Checks if this Data provider supports the specified symbol + /// + /// The symbol + /// returns true if Data Provider supports the specified symbol; otherwise false + private bool CanSubscribe(Symbol symbol) + { + return !symbol.Value.Contains("universe", StringComparison.InvariantCultureIgnoreCase) && + !symbol.IsCanonical() && + IsSecurityTypeSupported(symbol.SecurityType); } /// @@ -172,17 +181,22 @@ private void Initialize() /// The new enumerator for this subscription request public IEnumerator? Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler) { - Log.Trace($"DataBentoProvider.Subscribe(): Received subscription request for {dataConfig.Symbol}, Resolution={dataConfig.Resolution}, TickType={dataConfig.TickType}"); - if (!CanSubscribe(dataConfig)) + if (!IsSupported(dataConfig.SecurityType, dataConfig.Type, dataConfig.TickType, dataConfig.Resolution)) { - Log.Error($"DataBentoProvider.Subscribe(): Cannot subscribe to {dataConfig.Symbol} with Resolution={dataConfig.Resolution}, TickType={dataConfig.TickType}"); return null; } - _subscriptionConfigs[dataConfig.Symbol] = dataConfig; + lock (_sessionLock) + { + if (!_sessionStarted) + { + Log.Debug("DataBentoProvider.SubscriptionLogic(): Starting session"); + _sessionStarted = _client.StartSession(); + } + } + var enumerator = _dataAggregator.Add(dataConfig, newDataAvailableHandler); _subscriptionManager.Subscribe(dataConfig); - _activeSubscriptionConfigs.Add(dataConfig); return enumerator; } @@ -193,15 +207,8 @@ private void Initialize() /// Subscription config to be removed public void Unsubscribe(SubscriptionDataConfig dataConfig) { - Log.Trace($"DataBentoProvider.Unsubscribe(): Received unsubscription request for {dataConfig.Symbol}, Resolution={dataConfig.Resolution}, TickType={dataConfig.TickType}"); - _subscriptionConfigs.TryRemove(dataConfig.Symbol, out _); + Log.Debug($"DataBentoProvider.Unsubscribe(): Received unsubscription request for {dataConfig.Symbol}, Resolution={dataConfig.Resolution}, TickType={dataConfig.TickType}"); _subscriptionManager.Unsubscribe(dataConfig); - var toRemove = _activeSubscriptionConfigs.FirstOrDefault(c => c.Symbol == dataConfig.Symbol && c.TickType == dataConfig.TickType); - if (toRemove != null) - { - Log.Trace($"DataBentoProvider.Unsubscribe(): Removing active subscription for {dataConfig.Symbol}, Resolution={dataConfig.Resolution}, TickType={dataConfig.TickType}"); - _activeSubscriptionConfigs.Remove(toRemove); - } _dataAggregator.Remove(dataConfig); } @@ -211,7 +218,10 @@ public void Unsubscribe(SubscriptionDataConfig dataConfig) /// Job we're subscribing for public void SetJob(LiveNodePacket job) { - // No action required for DataBento since the job details are not used in the subscription process. + if (_initialized) + { + return; + } } /// @@ -221,62 +231,8 @@ public void Dispose() { _dataAggregator?.DisposeSafely(); _subscriptionManager?.DisposeSafely(); - _client?.Dispose(); - _dataDownloader?.Dispose(); - } - - /// - /// Gets the history for the requested security - /// - /// The historical data request - /// An enumerable of BaseData points - public IEnumerable? GetHistory(Data.HistoryRequest request) - { - Log.Trace($"DataBentoProvider.GetHistory(): Received history request for {request.Symbol}, Resolution={request.Resolution}, TickType={request.TickType}"); - if (!CanSubscribe(request.Symbol)) - { - Log.Error($"DataBentoProvider.GetHistory(): Cannot provide history for {request.Symbol} with Resolution={request.Resolution}, TickType={request.TickType}"); - return null; - } - - try - { - // Use the data downloader to get historical data - var parameters = new DataDownloaderGetParameters( - request.Symbol, - request.Resolution, - request.StartTimeUtc, - request.EndTimeUtc, - request.TickType); - - return _dataDownloader.Get(parameters); - } - catch (Exception ex) - { - Log.Error($"DataBentoProvider.GetHistory(): Failed to get history for {request.Symbol}: {ex.Message}"); - return null; - } - } - - /// - /// Checks if this Data provider supports the specified symbol - /// - /// The symbol - /// returns true if Data Provider supports the specified symbol; otherwise false - private bool CanSubscribe(Symbol symbol) - { - return !symbol.Value.Contains("universe", StringComparison.InvariantCultureIgnoreCase) && - !symbol.IsCanonical() && - IsSecurityTypeSupported(symbol.SecurityType); - } - - /// - /// Determines whether or not the specified config can be subscribed to - /// - private bool CanSubscribe(SubscriptionDataConfig config) - { - return CanSubscribe(config.Symbol) && - IsSupported(config.SecurityType, config.Type, config.TickType, config.Resolution); + _client?.DisposeSafely(); + _dataDownloader?.DisposeSafely(); } /// @@ -295,12 +251,6 @@ private bool IsSecurityTypeSupported(SecurityType securityType) /// private bool IsSupported(SecurityType securityType, Type dataType, TickType tickType, Resolution resolution) { - // Check supported security types - if (!IsSecurityTypeSupported(securityType)) - { - throw new NotSupportedException($"Unsupported security type: {securityType}"); - } - // Check supported data types if (dataType != typeof(TradeBar) && dataType != typeof(QuoteBar) && @@ -328,47 +278,66 @@ private bool IsSupported(SecurityType securityType, Type dataType, TickType tick /// private DateTime GetTickTime(Symbol symbol, DateTime utcTime) { - var exchangeTimeZone = _symbolExchangeTimeZones.GetOrAdd(symbol, sym => + DateTimeZone exchangeTimeZone; + lock (_symbolExchangeTimeZones) { - if (_marketHoursDatabase.TryGetEntry(sym.ID.Market, sym, sym.SecurityType, out var entry)) + if (!_symbolExchangeTimeZones.TryGetValue(symbol, out exchangeTimeZone)) { - return entry.ExchangeHours.TimeZone; + // read the exchange time zone from market-hours-database + if (_marketHoursDatabase.TryGetEntry(symbol.ID.Market, symbol, symbol.SecurityType, out var entry)) + { + exchangeTimeZone = entry.ExchangeHours.TimeZone; + } + // If there is no entry for the given Symbol, default to New York + else + { + exchangeTimeZone = TimeZones.NewYork; + } + + _symbolExchangeTimeZones[symbol] = exchangeTimeZone; } - // Futures default to Chicago - return TimeZones.Chicago; - }); + } return utcTime.ConvertFromUtc(exchangeTimeZone); } - // + /// /// Handles data received from the live client /// - private void OnDataReceived(object? sender, BaseData data) + private void OnDataReceived(object _, BaseData data) { try { - if (data is Tick tick) + switch (data) { - tick.Time = GetTickTime(tick.Symbol, tick.Time); - _dataAggregator.Update(tick); - - Log.Trace($"DataBentoProvider.OnDataReceived(): Updated tick - Symbol: {tick.Symbol}, " + - $"TickType: {tick.TickType}, Price: {tick.Value}, Quantity: {tick.Quantity}"); - } - else if (data is TradeBar tradeBar) - { - tradeBar.Time = GetTickTime(tradeBar.Symbol, tradeBar.Time); - tradeBar.EndTime = GetTickTime(tradeBar.Symbol, tradeBar.EndTime); - _dataAggregator.Update(tradeBar); + case Tick tick: + tick.Time = GetTickTime(tick.Symbol, tick.Time); + lock (_dataAggregator) + { + _dataAggregator.Update(tick); + } + // Log.Trace($"DataBentoProvider.OnDataReceived(): Updated tick - Symbol: {tick.Symbol}, " + + // $"TickType: {tick.TickType}, Price: {tick.Value}, Quantity: {tick.Quantity}"); + break; + + case TradeBar tradeBar: + tradeBar.Time = GetTickTime(tradeBar.Symbol, tradeBar.Time); + tradeBar.EndTime = GetTickTime(tradeBar.Symbol, tradeBar.EndTime); + lock (_dataAggregator) + { + _dataAggregator.Update(tradeBar); + } + // Log.Trace($"DataBentoProvider.OnDataReceived(): Updated TradeBar - Symbol: {tradeBar.Symbol}, " + + // $"O:{tradeBar.Open} H:{tradeBar.High} L:{tradeBar.Low} C:{tradeBar.Close} V:{tradeBar.Volume}"); + break; - Log.Trace($"DataBentoProvider.OnDataReceived(): Updated TradeBar - Symbol: {tradeBar.Symbol}, " + - $"O:{tradeBar.Open} H:{tradeBar.High} L:{tradeBar.Low} C:{tradeBar.Close} V:{tradeBar.Volume}"); - } - else - { - data.Time = GetTickTime(data.Symbol, data.Time); - _dataAggregator.Update(data); + default: + data.Time = GetTickTime(data.Symbol, data.Time); + lock (_dataAggregator) + { + _dataAggregator.Update(data); + } + break; } } catch (Exception ex) @@ -376,41 +345,5 @@ private void OnDataReceived(object? sender, BaseData data) Log.Error($"DataBentoProvider.OnDataReceived(): Error updating data aggregator: {ex.Message}\n{ex.StackTrace}"); } } - - /// - /// Handles connection status changes from the live client - /// - private void OnConnectionStatusChanged(object? sender, bool isConnected) - { - Log.Trace($"DataBentoProvider.OnConnectionStatusChanged(): Connection status changed to: {isConnected}"); - - if (isConnected) - { - // Reset session flag on reconnection - lock (_sessionLock) - { - _sessionStarted = false; - } - - // Resubscribe to all active subscriptions - foreach (var config in _activeSubscriptionConfigs) - { - _client.Subscribe(config.Symbol, config.Resolution, config.TickType); - } - - // Start session after resubscribing - if (_activeSubscriptionConfigs.Any()) - { - lock (_sessionLock) - { - if (!_sessionStarted) - { - Log.Trace("DataBentoProvider.OnConnectionStatusChanged(): Starting session after reconnection"); - _sessionStarted = _client.StartSession(); - } - } - } - } - } } } diff --git a/QuantConnect.DataBento/DataBentoHistoryProivder.cs b/QuantConnect.DataBento/DataBentoHistoryProivder.cs index a93b50e..eb23154 100644 --- a/QuantConnect.DataBento/DataBentoHistoryProivder.cs +++ b/QuantConnect.DataBento/DataBentoHistoryProivder.cs @@ -14,7 +14,6 @@ * */ -using System; using NodaTime; using QuantConnect.Data; using QuantConnect.Data.Market; @@ -22,30 +21,25 @@ using QuantConnect.Lean.Engine.HistoricalData; using QuantConnect.Logging; using QuantConnect.Util; -using QuantConnect.Lean.DataSource.DataBento; using QuantConnect.Interfaces; -using System.Collections.Generic; -using QuantConnect.Configuration; using QuantConnect.Securities; using QuantConnect.Data.Consolidators; namespace QuantConnect.Lean.DataSource.DataBento { /// - /// DataBento implementation of + /// Impleements a history provider for DataBento historical data. + /// Uses consolidators to produce the requested resolution when necessary. /// - public partial class DataBentoHistoryProvider : SynchronizingHistoryProvider + public partial class DataBentoProvider : SynchronizingHistoryProvider { private int _dataPointCount; - private DataBentoDataDownloader _dataDownloader; + + /// + /// Indicates whether a error for an invalid start time has been fired, where the start time is greater than or equal to the end time in UTC. + /// private volatile bool _invalidStartTimeErrorFired; - private volatile bool _invalidTickTypeAndResolutionErrorFired; - private volatile bool _unsupportedTickTypeMessagedLogged; - private MarketHoursDatabase _marketHoursDatabase; - private bool _unsupportedSecurityTypeMessageLogged; - private bool _unsupportedDataTypeMessageLogged; - private bool _potentialUnsupportedResolutionMessageLogged; - + /// /// Gets the total number of data points emitted by this history provider /// @@ -57,8 +51,6 @@ public partial class DataBentoHistoryProvider : SynchronizingHistoryProvider /// The initialization parameters public override void Initialize(HistoryProviderInitializeParameters parameters) { - _dataDownloader = new DataBentoDataDownloader(); - _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); } /// @@ -101,8 +93,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) /// An enumerable of BaseData points public IEnumerable? GetHistory(HistoryRequest request) { - if (request.Symbol.IsCanonical() || - !IsSupported(request.Symbol.SecurityType, request.DataType, request.TickType, request.Resolution)) + if (!CanSubscribe(request.Symbol)) { // It is Logged in IsSupported(...) return null; @@ -113,7 +104,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) if (!_unsupportedTickTypeMessagedLogged) { _unsupportedTickTypeMessagedLogged = true; - Log.Trace($"DataBentoHistoryProvider.GetHistory(): Unsupported tick type: {TickType.OpenInterest}"); + Log.Trace($"DataBentoProvider.GetHistory(): Unsupported tick type: {TickType.OpenInterest}"); } return null; } @@ -123,7 +114,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) if (!_invalidStartTimeErrorFired) { _invalidStartTimeErrorFired = true; - Log.Error($"{nameof(DataBentoHistoryProvider)}.{nameof(GetHistory)}:InvalidDateRange. The history request start date must precede the end date, no history returned"); + Log.Error($"{nameof(DataBentoProvider)}.{nameof(GetHistory)}:InvalidDateRange. The history request start date must precede the end date, no history returned"); } return null; } @@ -230,59 +221,5 @@ private IEnumerable GetQuotes(HistoryRequest request) var parameters = new DataDownloaderGetParameters(request.Symbol, Resolution.Tick, request.StartTimeUtc, request.EndTimeUtc, request.TickType); return _dataDownloader.Get(parameters); } - - /// - /// Checks if the security type is supported - /// - /// Security type to check - /// True if supported - private bool IsSecurityTypeSupported(SecurityType securityType) - { - // DataBento primarily supports futures, but also has equity and option coverage - return securityType == SecurityType.Future; - } - - /// - /// Determines if the specified subscription is supported - /// - private bool IsSupported(SecurityType securityType, Type dataType, TickType tickType, Resolution resolution) - { - // Check supported security types - if (!IsSecurityTypeSupported(securityType)) - { - if (!_unsupportedSecurityTypeMessageLogged) - { - _unsupportedSecurityTypeMessageLogged = true; - Log.Trace($"DataBentoDataProvider.IsSupported(): Unsupported security type: {securityType}"); - } - return false; - } - - // Check supported data types - if (dataType != typeof(TradeBar) && - dataType != typeof(QuoteBar) && - dataType != typeof(Tick) && - dataType != typeof(OpenInterest)) - { - if (!_unsupportedDataTypeMessageLogged) - { - _unsupportedDataTypeMessageLogged = true; - Log.Trace($"DataBentoDataProvider.IsSupported(): Unsupported data type: {dataType}"); - } - return false; - } - - // Warn about potential limitations for tick data - // I'm mimicing polygon implementation with this - if (!_potentialUnsupportedResolutionMessageLogged) - { - _potentialUnsupportedResolutionMessageLogged = true; - Log.Trace("DataBentoDataProvider.IsSupported(): " + - $"Subscription for {securityType}-{dataType}-{tickType}-{resolution} will be attempted. " + - $"An Advanced DataBento subscription plan is required to stream tick data."); - } - - return true; - } } } diff --git a/QuantConnect.DataBento/DataBentoRawLiveClient.cs b/QuantConnect.DataBento/DataBentoRawLiveClient.cs index 2e7bb97..2dab1d2 100644 --- a/QuantConnect.DataBento/DataBentoRawLiveClient.cs +++ b/QuantConnect.DataBento/DataBentoRawLiveClient.cs @@ -14,14 +14,12 @@ * */ -using System; -using System.IO; using System.Text; using System.Net.Sockets; using System.Security.Cryptography; using System.Collections.Concurrent; using System.Text.Json; -using System.Linq; +using System.Threading.Tasks; using QuantConnect.Data; using QuantConnect.Data.Market; using QuantConnect.Logging; @@ -31,33 +29,44 @@ namespace QuantConnect.Lean.DataSource.DataBento /// /// DataBento Raw TCP client for live streaming data /// - public class DatabentoRawClient : IDisposable + public class DataBentoRawLiveClient : IDisposable { + /// + /// The DataBento API key for authentication + /// private readonly string _apiKey; - private readonly string _gateway; + /// + /// The DataBento live gateway address to receive data from + /// + private const string _gateway = "glbx-mdp3.lsg.databento.com:13000"; + /// + /// The dataset to subscribe to + /// private readonly string _dataset; - private TcpClient? _tcpClient; + private readonly TcpClient? _tcpClient; + private readonly string _host; + private readonly int _port; private NetworkStream? _stream; - private StreamReader? _reader; - private StreamWriter? _writer; - private CancellationTokenSource _cancellationTokenSource; + private StreamReader _reader; + private StreamWriter _writer; + private readonly CancellationTokenSource _cancellationTokenSource; private readonly ConcurrentDictionary _subscriptions; private readonly object _connectionLock = new object(); private bool _isConnected; private bool _disposed; private const decimal PriceScaleFactor = 1e-9m; private readonly ConcurrentDictionary _instrumentIdToSymbol = new ConcurrentDictionary(); - private readonly ConcurrentDictionary _lastTicks = new ConcurrentDictionary(); + private readonly DataBentoSymbolMapper _symbolMapper; /// /// Event fired when new data is received /// - public event EventHandler? DataReceived; + public event EventHandler DataReceived; /// /// Event fired when connection status changes /// - public event EventHandler? ConnectionStatusChanged; + public event EventHandler ConnectionStatusChanged; /// /// Gets whether the client is currently connected @@ -65,15 +74,21 @@ public class DatabentoRawClient : IDisposable public bool IsConnected => _isConnected && _tcpClient?.Connected == true; /// - /// Initializes a new instance of the DatabentoRawClient + /// Initializes a new instance of the DataBentoRawLiveClient + /// The DataBento API key. /// - public DatabentoRawClient(string apiKey, string gateway = "glbx-mdp3.lsg.databento.com:13000", string dataset = "GLBX.MDP3") + public DataBentoRawLiveClient(string apiKey, string dataset = "GLBX.MDP3") { _apiKey = apiKey ?? throw new ArgumentNullException(nameof(apiKey)); - _gateway = gateway ?? throw new ArgumentNullException(nameof(gateway)); _dataset = dataset; + _tcpClient = new TcpClient(); _subscriptions = new ConcurrentDictionary(); _cancellationTokenSource = new CancellationTokenSource(); + _symbolMapper = new DataBentoSymbolMapper(); + + var parts = _gateway.Split(':'); + _host = parts[0]; + _port = parts.Length > 1 ? int.Parse(parts[1]) : 13000; } /// @@ -81,20 +96,15 @@ public DatabentoRawClient(string apiKey, string gateway = "glbx-mdp3.lsg.databen /// public bool Connect() { - Log.Trace("DatabentoRawClient.Connect(): Connecting to DataBento live gateway"); - if (_isConnected || _disposed) + Log.Trace("DataBentoRawLiveClient.Connect(): Connecting to DataBento live gateway"); + if (_isConnected) { return _isConnected; } try { - var parts = _gateway.Split(':'); - var host = parts[0]; - var port = parts.Length > 1 ? int.Parse(parts[1]) : 13000; - - _tcpClient = new TcpClient(); - _tcpClient.Connect(host, port); + _tcpClient.Connect(_host, _port); _stream = _tcpClient.GetStream(); _reader = new StreamReader(_stream, Encoding.ASCII); _writer = new StreamWriter(_stream, Encoding.ASCII) { AutoFlush = true }; @@ -106,15 +116,15 @@ public bool Connect() ConnectionStatusChanged?.Invoke(this, true); // Start message processing - ProcessMessages(); + Task.Run(ProcessMessages, _cancellationTokenSource.Token); - Log.Trace("DatabentoRawClient.Connect(): Connected and authenticated to DataBento live gateway"); + Log.Trace("DataBentoRawLiveClient.Connect(): Connected and authenticated to DataBento live gateway"); return true; } } catch (Exception ex) { - Log.Error($"DatabentoRawClient.Connect(): Failed to connect: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient.Connect(): Failed to connect: {ex.Message}"); Disconnect(); } @@ -126,124 +136,97 @@ public bool Connect() /// private bool Authenticate() { - if (_reader == null || _writer == null) - return false; - try { // Read greeting and challenge - string? versionLine = _reader.ReadLine(); - string? cramLine = _reader.ReadLine(); + var versionLine = _reader.ReadLine(); + var cramLine = _reader.ReadLine(); if (string.IsNullOrEmpty(versionLine) || string.IsNullOrEmpty(cramLine)) { - Log.Error("DatabentoRawClient.Authenticate(): Failed to receive greeting or challenge"); + Log.Error("DataBentoRawLiveClient.Authenticate(): Failed to receive greeting or challenge"); return false; } - Log.Trace($"DatabentoRawClient.Authenticate(): Version: {versionLine}"); - Log.Trace($"DatabentoRawClient.Authenticate(): Challenge: {cramLine}"); - // Parse challenge - string[] cramParts = cramLine.Split('='); + var cramParts = cramLine.Split('='); if (cramParts.Length != 2 || cramParts[0] != "cram") { - Log.Error("DatabentoRawClient.Authenticate(): Invalid challenge format"); - return false; - } - string cram = cramParts[1].Trim(); - - // Compute auth hash - string concat = $"{cram}|{_apiKey}"; - string hashHex = ComputeSHA256(concat); - string bucketId = _apiKey.Length >= 5 ? _apiKey.Substring(_apiKey.Length - 5) : _apiKey; - string authString = $"{hashHex}-{bucketId}"; - - // Send auth message - string authMsg = $"auth={authString}|dataset={_dataset}|encoding=json|ts_out=0"; - Log.Trace($"DatabentoRawClient.Authenticate(): Sending auth"); - _writer.WriteLine(authMsg); - - // Read auth response - string? authResp = _reader.ReadLine(); - if (string.IsNullOrEmpty(authResp)) - { - Log.Error("DatabentoRawClient.Authenticate(): No authentication response received"); + Log.Error("DataBentoRawLiveClient.Authenticate(): Invalid challenge format"); return false; } + var cram = cramParts[1].Trim(); - Log.Trace($"DatabentoRawClient.Authenticate(): Auth response: {authResp}"); - + // Auth + _writer.WriteLine($"auth={GetAuthStringFromCram(cram)}|dataset={_dataset}|encoding=json|ts_out=0"); + var authResp = _reader.ReadLine(); if (!authResp.Contains("success=1")) { - Log.Error($"DatabentoRawClient.Authenticate(): Authentication failed: {authResp}"); + Log.Error($"DataBentoRawLiveClient.Authenticate(): Authentication failed: {authResp}"); return false; } - Log.Trace("DatabentoRawClient.Authenticate(): Authentication successful"); + Log.Trace("DataBentoRawLiveClient.Authenticate(): Authentication successful"); return true; } catch (Exception ex) { - Log.Error($"DatabentoRawClient.Authenticate(): Authentication failed: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient.Authenticate(): Authentication failed: {ex.Message}"); return false; } } - private static string ComputeSHA256(string input) + + /// + /// Handles the DataBento authentication string from a CRAM challenge + /// + /// The CRAM challenge string + /// The auth string to send to the server + private string GetAuthStringFromCram(string cram) { - using var sha = SHA256.Create(); - byte[] hash = sha.ComputeHash(Encoding.UTF8.GetBytes(input)); - var sb = new StringBuilder(); - foreach (byte b in hash) - { - sb.Append(b.ToString("x2")); - } - return sb.ToString(); + if (string.IsNullOrWhiteSpace(cram)) + throw new ArgumentException("CRAM challenge cannot be null or empty", nameof(cram)); + + string concat = $"{cram}|{_apiKey}"; + string hashHex = ComputeSHA256(concat); + string bucketId = _apiKey.Substring(_apiKey.Length - 5); + + return $"{hashHex}-{bucketId}"; } /// /// Subscribes to live data for a symbol /// - public bool Subscribe(Symbol symbol, Resolution resolution, TickType tickType) + public bool Subscribe(Symbol symbol, TickType tickType) { - if (!IsConnected || _writer == null) + if (!IsConnected) { - Log.Error("DatabentoRawClient.Subscribe(): Not connected to gateway"); + Log.Error("DataBentoRawLiveClient.Subscribe(): Not connected to gateway"); return false; } try { // Get the databento symbol form LEAN symbol - // Get schema from the resolution - var databentoSymbol = MapSymbolToDataBento(symbol); - var schema = GetSchema(resolution, tickType); + var databentoSymbol = _symbolMapper.GetBrokerageSymbol(symbol); + var schema = "mbp-1"; + var resolution = Resolution.Tick; // subscribe var subscribeMessage = $"schema={schema}|stype_in=parent|symbols={databentoSymbol}"; - Log.Trace($"DatabentoRawClient.Subscribe(): Subscribing with message: {subscribeMessage}"); + Log.Debug($"DataBentoRawLiveClient.Subscribe(): Subscribing with message: {subscribeMessage}"); // Send subscribe message _writer.WriteLine(subscribeMessage); // Store subscription _subscriptions.TryAdd(symbol, (resolution, tickType)); - Log.Trace($"DatabentoRawClient.Subscribe(): Subscribed to {symbol} ({databentoSymbol}) at {resolution} resolution for {tickType}"); - - // If subscribing to quote ticks, also subscribe to trade ticks - if (tickType == TickType.Quote && resolution == Resolution.Tick) - { - var tradeSchema = GetSchema(resolution, TickType.Trade); - var tradeSubscribeMessage = $"schema={tradeSchema}|stype_in=parent|symbols={databentoSymbol}"; - Log.Trace($"DatabentoRawClient.Subscribe(): Also subscribing to trades with message: {tradeSubscribeMessage}"); - _writer.WriteLine(tradeSubscribeMessage); - } + Log.Debug($"DataBentoRawLiveClient.Subscribe(): Subscribed to {symbol} ({databentoSymbol}) at {resolution} resolution for {tickType}"); return true; } catch (Exception ex) { - Log.Error($"DatabentoRawClient.Subscribe(): Failed to subscribe to {symbol}: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient.Subscribe(): Failed to subscribe to {symbol}: {ex.Message}"); return false; } } @@ -253,21 +236,21 @@ public bool Subscribe(Symbol symbol, Resolution resolution, TickType tickType) /// public bool StartSession() { - if (!IsConnected || _writer == null) + if (!IsConnected) { - Log.Error("DatabentoRawClient.StartSession(): Not connected"); + Log.Error("DataBentoRawLiveClient.StartSession(): Not connected"); return false; } try { - Log.Trace("DatabentoRawClient.StartSession(): Starting session"); + Log.Trace("DataBentoRawLiveClient.StartSession(): Starting session"); _writer.WriteLine("start_session=1"); return true; } catch (Exception ex) { - Log.Error($"DatabentoRawClient.StartSession(): Failed to start session: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient.StartSession(): Failed to start session: {ex.Message}"); return false; } } @@ -281,13 +264,13 @@ public bool Unsubscribe(Symbol symbol) { if (_subscriptions.TryRemove(symbol, out _)) { - Log.Trace($"DatabentoRawClient.Unsubscribe(): Unsubscribed from {symbol}"); + Log.Debug($"DataBentoRawLiveClient.Unsubscribe(): Unsubscribed from {symbol}"); } return true; } catch (Exception ex) { - Log.Error($"DatabentoRawClient.Unsubscribe(): Failed to unsubscribe from {symbol}: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient.Unsubscribe(): Failed to unsubscribe from {symbol}: {ex.Message}"); return false; } } @@ -297,33 +280,17 @@ public bool Unsubscribe(Symbol symbol) /// private void ProcessMessages() { - Log.Trace("DatabentoRawClient.ProcessMessages(): Starting message processing"); - if (_reader == null) - { - Log.Error("DatabentoRawClient.ProcessMessages(): No reader available"); - return; - } - - var messageCount = 0; + Log.Debug("DataBentoRawLiveClient.ProcessMessages(): Starting message processing"); try { while (!_cancellationTokenSource.IsCancellationRequested && IsConnected) { var line = _reader.ReadLine(); - if (line == null) - { - Log.Trace("DatabentoRawClient.ProcessMessages(): Connection closed by server"); - break; - } - if (string.IsNullOrWhiteSpace(line)) - continue; - - messageCount++; - if (messageCount <= 50 || messageCount % 100 == 0) { - Log.Trace($"DatabentoRawClient.ProcessMessages(): Message #{messageCount}: {line.Substring(0, Math.Min(150, line.Length))}..."); + Log.Trace("DataBentoRawLiveClient.ProcessMessages(): Line is null or empty. Issue receiving data."); + break; } ProcessSingleMessage(line); @@ -331,19 +298,18 @@ private void ProcessMessages() } catch (OperationCanceledException) { - Log.Trace("DatabentoRawClient.ProcessMessages(): Message processing cancelled"); + Log.Trace("DataBentoRawLiveClient.ProcessMessages(): Message processing cancelled"); } catch (IOException ex) when (ex.InnerException is SocketException) { - Log.Trace($"DatabentoRawClient.ProcessMessages(): Socket exception: {ex.Message}"); + Log.Trace($"DataBentoRawLiveClient.ProcessMessages(): Socket exception: {ex.Message}"); } catch (Exception ex) { - Log.Error($"DatabentoRawClient.ProcessMessages(): Error processing messages: {ex.Message}\n{ex.StackTrace}"); + Log.Error($"DataBentoRawLiveClient.ProcessMessages(): Error processing messages: {ex.Message}\n{ex.StackTrace}"); } finally { - Log.Trace($"DatabentoRawClient.ProcessMessages(): Exiting. Total messages processed: {messageCount}"); Disconnect(); } } @@ -365,57 +331,71 @@ private void ProcessSingleMessage(string message) { var rtype = rtypeElement.GetInt32(); - if (rtype == 23) - { - if (root.TryGetProperty("msg", out var msgElement)) - { - Log.Trace($"DatabentoRawClient: System message: {msgElement.GetString()}"); - } - return; - } - else if (rtype == 22) + switch (rtype) { - // Symbol mapping message - if (root.TryGetProperty("stype_in_symbol", out var inSymbol) && - root.TryGetProperty("stype_out_symbol", out var outSymbol) && - headerElement.TryGetProperty("instrument_id", out var instId)) - { - var instrumentId = instId.GetInt64(); - var outSymbolStr = outSymbol.GetString(); - - Log.Trace($"DatabentoRawClient: Symbol mapping: {inSymbol.GetString()} -> {outSymbolStr} (instrument_id: {instrumentId})"); - - // Find the subscription that matches this symbol - foreach (var kvp in _subscriptions) + case 23: + // System message + if (root.TryGetProperty("msg", out var msgElement)) { - var leanSymbol = kvp.Key; + Log.Debug($"DataBentoRawLiveClient: System message: {msgElement.GetString()}"); + } + return; + + case 22: + // Symbol mapping message + if (root.TryGetProperty("stype_in_symbol", out var inSymbol) && + root.TryGetProperty("stype_out_symbol", out var outSymbol) && + headerElement.TryGetProperty("instrument_id", out var instId)) + { + var instrumentId = instId.GetInt64(); + var outSymbolStr = outSymbol.GetString(); + + Log.Debug($"DataBentoRawLiveClient: Symbol mapping: {inSymbol.GetString()} -> {outSymbolStr} (instrument_id: {instrumentId})"); + if (outSymbolStr != null) { - _instrumentIdToSymbol[instrumentId] = leanSymbol; - Log.Trace($"DatabentoRawClient: Mapped instrument_id {instrumentId} to {leanSymbol}"); - break; + // Let's find the subscribed symbol to get the market and security type + var inSymbolStr = inSymbol.GetString(); + var subscription = _subscriptions.Keys.FirstOrDefault(s => _symbolMapper.GetBrokerageSymbol(s) == inSymbolStr); + if (subscription != null) + { + if (subscription.SecurityType == SecurityType.Future) + { + var leanSymbol = _symbolMapper.GetLeanSymbolForFuture(outSymbolStr); + if (leanSymbol == null) + { + Log.Trace($"DataBentoRawLiveClient: Future spreads are not supported: {outSymbolStr}. Skipping mapping."); + return; + } + _instrumentIdToSymbol[instrumentId] = leanSymbol; + Log.Debug($"DataBentoRawLiveClient: Mapped instrument_id {instrumentId} to {leanSymbol}"); + } + } } } - } - return; - } - else if (rtype == 1) - { - // MBP-1 (Market By Price) - Quote ticks - HandleMBPMessage(root, headerElement); - return; - } - else if (rtype == 0) - { - // Trade messages - Trade ticks - HandleTradeTickMessage(root, headerElement); - return; - } - else if (rtype == 32 || rtype == 33 || rtype == 34 || rtype == 35) - { - // OHLCV bar messages - HandleOHLCVMessage(root, headerElement); - return; + return; + + case 1: + // MBP-1 (Market By Price) + HandleMBPMessage(root, headerElement); + return; + + case 0: + // Trade messages + HandleTradeTickMessage(root, headerElement); + return; + + case 32: + case 33: + case 34: + case 35: + // OHLCV bar messages + HandleOHLCVMessage(root, headerElement); + return; + + default: + Log.Error($"DataBentoRawLiveClient: Unknown rtype {rtype} in message"); + return; } } } @@ -423,16 +403,16 @@ private void ProcessSingleMessage(string message) // Handle other message types if needed if (root.TryGetProperty("error", out var errorElement)) { - Log.Error($"DatabentoRawClient: Server error: {errorElement.GetString()}"); + Log.Error($"DataBentoRawLiveClient: Server error: {errorElement.GetString()}"); } } catch (JsonException ex) { - Log.Error($"DatabentoRawClient.ProcessSingleMessage(): JSON parse error: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient.ProcessSingleMessage(): JSON parse error: {ex.Message}"); } catch (Exception ex) { - Log.Error($"DatabentoRawClient.ProcessSingleMessage(): Error: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient.ProcessSingleMessage(): Error: {ex.Message}"); } } @@ -458,7 +438,7 @@ private void HandleOHLCVMessage(JsonElement root, JsonElement header) if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) { - Log.Trace($"DatabentoRawClient: No mapping for instrument_id {instrumentId} in OHLCV message."); + Log.Debug($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in OHLCV message."); return; } @@ -511,13 +491,13 @@ private void HandleOHLCVMessage(JsonElement root, JsonElement header) period ); - Log.Trace($"DatabentoRawClient: OHLCV bar: {matchedSymbol} O={open} H={high} L={low} C={close} V={volume} at {timestamp}"); + // Log.Trace($"DataBentoRawLiveClient: OHLCV bar: {matchedSymbol} O={open} H={high} L={low} C={close} V={volume} at {timestamp}"); DataReceived?.Invoke(this, tradeBar); } } catch (Exception ex) { - Log.Error($"DatabentoRawClient.HandleOHLCVMessage(): Error: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient.HandleOHLCVMessage(): Error: {ex.Message}"); } } @@ -543,7 +523,7 @@ private void HandleMBPMessage(JsonElement root, JsonElement header) if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) { - Log.Trace($"DatabentoRawClient: No mapping for instrument_id {instrumentId} in MBP message."); + Log.Trace($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in MBP message."); return; } @@ -582,13 +562,13 @@ private void HandleMBPMessage(JsonElement root, JsonElement header) // QuantConnect convention: Quote ticks should have zero Price and Quantity quoteTick.Quantity = 0; - Log.Trace($"DatabentoRawClient: Quote tick: {matchedSymbol} Bid={quoteTick.BidPrice}x{quoteTick.BidSize} Ask={quoteTick.AskPrice}x{quoteTick.AskSize}"); + // Log.Trace($"DataBentoRawLiveClient: Quote tick: {matchedSymbol} Bid={quoteTick.BidPrice}x{quoteTick.BidSize} Ask={quoteTick.AskPrice}x{quoteTick.AskSize}"); DataReceived?.Invoke(this, quoteTick); } } catch (Exception ex) { - Log.Error($"DatabentoRawClient.HandleMBPMessage(): Error: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient.HandleMBPMessage(): Error: {ex.Message}"); } } @@ -614,7 +594,7 @@ private void HandleTradeTickMessage(JsonElement root, JsonElement header) if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) { - Log.Trace($"DatabentoRawClient: No mapping for instrument_id {instrumentId} in trade message."); + Log.Trace($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in trade message."); return; } @@ -639,66 +619,14 @@ private void HandleTradeTickMessage(JsonElement root, JsonElement header) AskSize = 0 }; - Log.Trace($"DatabentoRawClient: Trade tick: {matchedSymbol} Price={price} Quantity={size}"); + // Log.Trace($"DataBentoRawLiveClient: Trade tick: {matchedSymbol} Price={price} Quantity={size}"); DataReceived?.Invoke(this, tradeTick); } } catch (Exception ex) { - Log.Error($"DatabentoRawClient.HandleTradeTickMessage(): Error: {ex.Message}"); - } - } - - /// - /// Maps a LEAN symbol to DataBento symbol format - /// - private string MapSymbolToDataBento(Symbol symbol) - { - if (symbol.SecurityType == SecurityType.Future) - { - // For DataBento, use the root symbol with .FUT suffix for parent subscription - // ES19Z25 -> ES.FUT - var value = symbol.Value; - - // Extract root by removing digits and month codes - var root = new string(value.TakeWhile(c => !char.IsDigit(c)).ToArray()); - - return $"{root}.FUT"; + Log.Error($"DataBentoRawLiveClient.HandleTradeTickMessage(): Error: {ex.Message}"); } - - return symbol.Value; - } - - /// - /// Pick Databento schema from Lean resolution/ticktype - /// - private string GetSchema(Resolution resolution, TickType tickType) - { - if (tickType == TickType.Trade) - { - if (resolution == Resolution.Tick) - return "trades"; - if (resolution == Resolution.Second) - return "ohlcv-1s"; - if (resolution == Resolution.Minute) - return "ohlcv-1m"; - if (resolution == Resolution.Hour) - return "ohlcv-1h"; - if (resolution == Resolution.Daily) - return "ohlcv-1d"; - } - else if (tickType == TickType.Quote) - { - // top of book - if (resolution == Resolution.Tick || resolution == Resolution.Second || resolution == Resolution.Minute || resolution == Resolution.Hour || resolution == Resolution.Daily) - return "mbp-1"; - } - else if (tickType == TickType.OpenInterest) - { - return "statistics"; - } - - throw new NotSupportedException($"Unsupported resolution {resolution} / {tickType}"); } /// @@ -723,11 +651,11 @@ public void Disconnect() } catch (Exception ex) { - Log.Trace($"DatabentoRawClient.Disconnect(): Error during disconnect: {ex.Message}"); + Log.Trace($"DataBentoRawLiveClient.Disconnect(): Error during disconnect: {ex.Message}"); } ConnectionStatusChanged?.Invoke(this, false); - Log.Trace("DatabentoRawClient.Disconnect(): Disconnected from DataBento gateway"); + Log.Trace("DataBentoRawLiveClient.Disconnect(): Disconnected from DataBento gateway"); } } @@ -748,5 +676,21 @@ public void Dispose() _stream?.Dispose(); _tcpClient?.Dispose(); } + + /// + /// Computes the SHA-256 hash of the input string + /// + private static string ComputeSHA256(string input) + { + using var sha = SHA256.Create(); + var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(input)); + var sb = new StringBuilder(); + foreach (byte b in hash) + { + sb.Append(b.ToString("x2")); + } + return sb.ToString(); + } + } } diff --git a/QuantConnect.DataBento/DataBentoSymbolMapper.cs b/QuantConnect.DataBento/DataBentoSymbolMapper.cs new file mode 100644 index 0000000..3c6b8b3 --- /dev/null +++ b/QuantConnect.DataBento/DataBentoSymbolMapper.cs @@ -0,0 +1,174 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using QuantConnect; +using QuantConnect.Brokerages; +using System.Globalization; + +namespace QuantConnect.Lean.DataSource.DataBento +{ + /// + /// Provides the mapping between Lean symbols and DataBento symbols. + /// + public class DataBentoSymbolMapper : ISymbolMapper + { + private readonly Dictionary _leanSymbolsCache = new(); + private readonly Dictionary _brokerageSymbolsCache = new(); + private readonly object _locker = new(); + + /// + /// Converts a Lean symbol instance to a brokerage symbol + /// + /// A Lean symbol instance + /// The brokerage symbol + public string GetBrokerageSymbol(Symbol symbol) + { + if (symbol == null || string.IsNullOrWhiteSpace(symbol.Value)) + { + throw new ArgumentException($"Invalid symbol: {(symbol == null ? "null" : symbol.ToString())}"); + } + + return GetBrokerageSymbol(symbol, false); + } + + /// + /// Converts a Lean symbol instance to a brokerage symbol with updating of cached symbol collection + /// + /// + /// + /// + /// + public string GetBrokerageSymbol(Symbol symbol, bool isUpdateCachedSymbol) + { + lock (_locker) + { + if (!_brokerageSymbolsCache.TryGetValue(symbol, out var brokerageSymbol) || isUpdateCachedSymbol) + { + switch (symbol.SecurityType) + { + case SecurityType.Future: + brokerageSymbol = $"{symbol.ID.Symbol}.FUT"; + break; + + case SecurityType.Equity: + brokerageSymbol = symbol.Value; + break; + + default: + throw new Exception($"DataBentoSymbolMapper.GetBrokerageSymbol(): unsupported security type: {symbol.SecurityType}"); + } + + // Lean-to-DataBento symbol conversion is accurate, so we can cache it both ways + _brokerageSymbolsCache[symbol] = brokerageSymbol; + _leanSymbolsCache[brokerageSymbol] = symbol; + } + + return brokerageSymbol; + } + } + + /// + /// Converts a brokerage symbol to a Lean symbol instance + /// + /// The brokerage symbol + /// The security type + /// The market + /// Expiration date of the security(if applicable) + /// A new Lean Symbol instance + public Symbol GetLeanSymbol(string brokerageSymbol, SecurityType securityType, string market, + DateTime expirationDate = new DateTime(), decimal strike = 0, OptionRight optionRight = 0) + { + if (string.IsNullOrWhiteSpace(brokerageSymbol)) + { + throw new ArgumentException("Invalid symbol: " + brokerageSymbol); + } + + lock (_locker) + { + if (!_leanSymbolsCache.TryGetValue(brokerageSymbol, out var leanSymbol)) + { + switch (securityType) + { + case SecurityType.Future: + leanSymbol = Symbol.CreateFuture(brokerageSymbol, market, expirationDate); + break; + + default: + throw new Exception($"DataBentoSymbolMapper.GetLeanSymbol(): unsupported security type: {securityType}"); + } + + _leanSymbolsCache[brokerageSymbol] = leanSymbol; + _brokerageSymbolsCache[leanSymbol] = brokerageSymbol; + } + + return leanSymbol; + } + } + + /// + /// Gets the Lean symbol for the specified DataBento symbol + /// + /// The databento symbol + /// The corresponding Lean symbol + public Symbol GetLeanSymbol(string databentoSymbol) + { + lock (_locker) + { + if (!_leanSymbolsCache.TryGetValue(databentoSymbol, out var symbol)) + { + symbol = GetLeanSymbol(databentoSymbol, SecurityType.Equity, Market.USA); + } + + return symbol; + } + } + + /// + /// Converts a brokerage future symbol to a Lean symbol instance + /// + /// The brokerage symbol + /// A new Lean Symbol instance + public Symbol GetLeanSymbolForFuture(string brokerageSymbol) + { + if (string.IsNullOrWhiteSpace(brokerageSymbol)) + { + throw new ArgumentException("Invalid symbol: " + brokerageSymbol); + } + + // ignore futures spreads + if (brokerageSymbol.Contains("-")) + { + return null; + } + + lock (_locker) + { + if (!_leanSymbolsCache.TryGetValue(brokerageSymbol, out var leanSymbol)) + { + leanSymbol = SymbolRepresentation.ParseFutureSymbol(brokerageSymbol); + + if (leanSymbol == null) + { + throw new ArgumentException("Invalid future symbol: " + brokerageSymbol); + } + + _leanSymbolsCache[brokerageSymbol] = leanSymbol; + } + + return leanSymbol; + } + } + } +} diff --git a/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj b/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj index 718002c..249ad0a 100644 --- a/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj +++ b/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj @@ -39,4 +39,8 @@ + + + + diff --git a/models/DataBentoTypes.cs b/models/DataBentoTypes.cs new file mode 100644 index 0000000..b7fa58c --- /dev/null +++ b/models/DataBentoTypes.cs @@ -0,0 +1,112 @@ +using System; +using CsvHelper.Configuration.Attributes; +using QuantConnect; + +namespace QuantConnect.Lean.DataSource.DataBento.Models +{ + /// + /// Provides a constant for scaling price values from DataBento. + /// + public static class PriceScaling + { + /// + /// price scale factor is needed to find the true price from the message + /// Due to compression each "1 unit corresponds to 1e-9, i.e. 1/1,000,000,000 or 0.000000001" + /// https://databento.com/docs/api-reference-live/basics/schemas-and-conventions?historical=raw&live=raw&reference=raw + /// + public const decimal PriceScaleFactor = 1e-9m; + } + + /// + /// Represents a single bar of historical data from DataBento. + /// This class is used to map CSV data from HTTP requests into a structured format. + /// + public class DatabentoBar + { + [Name("ts_event")] + public long TimestampNanos { get; set; } + + public DateTime Timestamp => Time.UnixNanosecondTimeStampToDateTime(TimestampNanos); + + [Name("open")] + public long RawOpen { get; set; } + + [Name("high")] + public long RawHigh { get; set; } + + [Name("low")] + public long RawLow { get; set; } + + + + [Name("close")] + public long RawClose { get; set; } + + [Ignore] + public decimal Open => RawOpen == long.MaxValue ? 0m : RawOpen * PriceScaling.PriceScaleFactor; + + [Ignore] + public decimal High => RawHigh == long.MaxValue ? 0m : RawHigh * PriceScaling.PriceScaleFactor; + + [Ignore] + public decimal Low => RawLow == long.MaxValue ? 0m : RawLow * PriceScaling.PriceScaleFactor; + + [Ignore] + public decimal Close => RawClose == long.MaxValue ? 0m : RawClose * PriceScaling.PriceScaleFactor; + + [Name("volume")] + public long RawVolume { get; set; } + + [Ignore] + public decimal Volume => RawVolume == long.MaxValue ? 0m : RawVolume; + } + + /// + /// Represents a single trade event from DataBento. + /// + public class DatabentoTrade + { + [Name("ts_event")] + public long TimestampNanos { get; set; } + + public DateTime Timestamp => Time.UnixNanosecondTimeStampToDateTime(TimestampNanos); + + [Name("price")] + public long RawPrice { get; set; } + + [Ignore] + public decimal Price => RawPrice == long.MaxValue ? 0m : RawPrice * PriceScaling.PriceScaleFactor; + + [Name("size")] + public int Size { get; set; } + } + + /// + /// Represents a single quote from DataBento. + /// + public class DatabentoQuote + { + [Name("ts_event")] + public long TimestampNanos { get; set; } + + public DateTime Timestamp => Time.UnixNanosecondTimeStampToDateTime(TimestampNanos); + + [Name("bid_px_00")] + public long RawBidPrice { get; set; } + + [Ignore] + public decimal BidPrice => RawBidPrice == long.MaxValue ? 0m : RawBidPrice * PriceScaling.PriceScaleFactor; + + [Name("bid_sz_00")] + public int BidSize { get; set; } + + [Name("ask_px_00")] + public long RawAskPrice { get; set; } + + [Ignore] + public decimal AskPrice => RawAskPrice == long.MaxValue ? 0m : RawAskPrice * PriceScaling.PriceScaleFactor; + + [Name("ask_sz_00")] + public int AskSize { get; set; } + } +} From 68fd7f6f471dfe293d086bb63bb01a952993bddd Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 20 Jan 2026 11:09:12 +0200 Subject: [PATCH 02/18] update: .net 10 --- .../QuantConnect.DataSource.DataBento.Tests.csproj | 2 +- QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj b/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj index f5fce66..82c36da 100644 --- a/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj +++ b/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj @@ -1,6 +1,6 @@ - net9.0 + net10.0 QuantConnect.DataLibrary.Tests diff --git a/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj b/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj index 249ad0a..0c83d24 100644 --- a/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj +++ b/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj @@ -3,7 +3,7 @@ Release AnyCPU - net9.0 + net10.0 QuantConnect.Lean.DataSource.DataBento QuantConnect.Lean.DataSource.DataBento QuantConnect.Lean.DataSource.DataBento From c23c1851289e8e01ea6ed239c5109ea61ca7ccb0 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 20 Jan 2026 11:31:49 +0200 Subject: [PATCH 03/18] feat: setup project configurations --- ...tConnect.DataSource.DataBento.Tests.csproj | 65 ++++++++++--------- .../Models}/DataBentoTypes.cs | 0 .../QuantConnect.DataSource.DataBento.csproj | 9 +-- 3 files changed, 37 insertions(+), 37 deletions(-) rename {models => QuantConnect.DataBento/Models}/DataBentoTypes.cs (100%) diff --git a/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj b/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj index 82c36da..b4c0ccd 100644 --- a/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj +++ b/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj @@ -1,32 +1,37 @@ - - net10.0 - QuantConnect.DataLibrary.Tests - - - - - - - - - all - - - - - - - - - - - - - - - - PreserveNewest - - + + Release + AnyCPU + net10.0 + false + UnitTest + bin\$(Configuration)\ + QuantConnect.Lean.DataSource.DataBento.Tests + QuantConnect.Lean.DataSource.DataBento.Tests + QuantConnect.Lean.DataSource.DataBento.Tests + QuantConnect.Lean.DataSource.DataBento.Tests + false + + + + + + + + + all + + + + + + + + + + + + PreserveNewest + + diff --git a/models/DataBentoTypes.cs b/QuantConnect.DataBento/Models/DataBentoTypes.cs similarity index 100% rename from models/DataBentoTypes.cs rename to QuantConnect.DataBento/Models/DataBentoTypes.cs diff --git a/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj b/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj index 0c83d24..ada272c 100644 --- a/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj +++ b/QuantConnect.DataBento/QuantConnect.DataSource.DataBento.csproj @@ -1,4 +1,3 @@ - Release @@ -29,18 +28,14 @@ bin\Release\ - + - - - - From f660594837b0fc72cd5746cfe1b09d9cb48e3986 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 20 Jan 2026 11:36:17 +0200 Subject: [PATCH 04/18] remove: demonstration file --- Demonstration.cs | 94 ------------------- ...tConnect.DataSource.DataBento.Tests.csproj | 3 - 2 files changed, 97 deletions(-) delete mode 100644 Demonstration.cs diff --git a/Demonstration.cs b/Demonstration.cs deleted file mode 100644 index f98b935..0000000 --- a/Demonstration.cs +++ /dev/null @@ -1,94 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using QuantConnect.Algorithm; -using QuantConnect.Data.Market; -using QuantConnect.Interfaces; -using QuantConnect; -using QuantConnect.Data; -using QuantConnect.Securities.Future; -using QuantConnect.Util; -using System; -using System.Linq; - -namespace QuantConnect.Algorithm.CSharp -{ - public class DatabentoFuturesTestAlgorithm : QCAlgorithm - { - private Future _es; - - public override void Initialize() - { - Log("Algorithm Initialize"); - - SetStartDate(2025, 10, 1); - SetEndDate(2025, 10, 16); - SetCash(100000); - - var exp = new DateTime(2025, 12, 19); - var symbol = QuantConnect.Symbol.CreateFuture("ES", Market.CME, exp); - //_es = AddFutureContract(symbol, Resolution.Tick, true, 1, true); - _es = AddFutureContract(symbol, Resolution.Second, true, 1, true); - Log($"_es: {_es}"); - - var history = History(_es.Symbol, 10, Resolution.Minute).ToList(); - - Log($"History returned {history.Count} bars"); - - foreach (var bar in history) - { - Log($"History Bar: {bar.Time} - O:{bar.Open} H:{bar.High} L:{bar.Low} C:{bar.Close} V:{bar.Volume}"); - } - - } - - public override void OnData(Slice slice) - { - if (!slice.HasData) - { - Log("Slice has no data"); - return; - } - - Log($"OnData: Slice has {slice.Count} data points"); - - // For Tick resolution, check Ticks collection - if (slice.Ticks.ContainsKey(_es.Symbol)) - { - var ticks = slice.Ticks[_es.Symbol]; - Log($"Received {ticks.Count} ticks for {_es.Symbol}"); - - foreach (var tick in ticks) - { - if (tick.TickType == TickType.Trade) - { - Log($"Trade Tick - Price: {tick.Price}, Quantity: {tick.Quantity}, Time: {tick.Time}"); - } - else if (tick.TickType == TickType.Quote) - { - Log($"Quote Tick - Bid: {tick.BidPrice}x{tick.BidSize}, Ask: {tick.AskPrice}x{tick.AskSize}, Time: {tick.Time}"); - } - } - } - - // Access OHLCV bars - foreach (var bar in slice.Bars.Values) - { - Log($"OHLCV BAR: {bar.Symbol.Value} - O: {bar.Open}, H: {bar.High}, L: {bar.Low}, C: {bar.Close}, V: {bar.Volume}"); - } - } - } -} diff --git a/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj b/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj index b4c0ccd..c58ec16 100644 --- a/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj +++ b/QuantConnect.DataBento.Tests/QuantConnect.DataSource.DataBento.Tests.csproj @@ -12,9 +12,6 @@ QuantConnect.Lean.DataSource.DataBento.Tests false - - - From f85939dd148305e82a7c5ce8616314aac96bf12c Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 20 Jan 2026 11:40:55 +0200 Subject: [PATCH 05/18] test:refactor: config file --- QuantConnect.DataBento.Tests/config.json | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/QuantConnect.DataBento.Tests/config.json b/QuantConnect.DataBento.Tests/config.json index 7256aba..2464c3e 100644 --- a/QuantConnect.DataBento.Tests/config.json +++ b/QuantConnect.DataBento.Tests/config.json @@ -1,9 +1,7 @@ { - "data-folder":"../../../../../Data/", - - "job-user-id": "0", + "data-folder": "../../../../Lean/Data/", + "job-user-id": "", "api-access-token": "", "job-organization-id": "", - - "databento-api-key":"" + "databento-api-key": "" } \ No newline at end of file From 991d8bf46d495ba17d663c369ef7392b9b288230 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 22 Jan 2026 01:00:36 +0200 Subject: [PATCH 06/18] refactor: HistoryProvider, DataDownloader, SymbolMapper --- Lean.DataSource.DataBento.sln | 8 +- .../DataBentoDataDownloaderTests.cs | 48 +--- .../DataBentoDataProviderHistoryTests.cs | 191 +++++-------- .../DataBentoHistoricalApiClientTests.cs | 95 +++++++ .../DataBentoJsonConverterTests.cs | 135 +++++++++ .../DataBentoRawLiveClientTests.cs | 2 +- .../DataBentoSymbolMapperTests.cs.cs | 56 ++++ QuantConnect.DataBento.Tests/TestSetup.cs | 2 +- .../Api/HistoricalAPIClient.cs | 168 ++++++++++++ .../DataBentoDataDownloader.cs | 256 ++++-------------- .../DataBentoDataProvider.cs | 46 ++-- .../DataBentoHistoryProivder.cs | 158 ++++++----- .../DataBentoRawLiveClient.cs | 2 +- .../DataBentoSymbolMapper.cs | 120 +------- QuantConnect.DataBento/Extensions.cs | 35 +++ .../Models/DataBentoTypes.cs | 112 -------- .../Models/Enums/StatisticType.cs | 117 ++++++++ QuantConnect.DataBento/Models/Header.cs | 48 ++++ .../Models/LevelOneBookLevel.cs | 49 ++++ QuantConnect.DataBento/Models/LevelOneData.cs | 69 +++++ .../Models/MarketDataRecord.cs | 30 ++ QuantConnect.DataBento/Models/OhlcvBar.cs | 48 ++++ .../Models/StatisticsData.cs | 31 +++ .../Serialization/JsonSettings.cs | 34 +++ .../SnakeCaseContractResolver.cs | 39 +++ 25 files changed, 1234 insertions(+), 665 deletions(-) create mode 100644 QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs create mode 100644 QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs create mode 100644 QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs create mode 100644 QuantConnect.DataBento/Api/HistoricalAPIClient.cs create mode 100644 QuantConnect.DataBento/Extensions.cs delete mode 100644 QuantConnect.DataBento/Models/DataBentoTypes.cs create mode 100644 QuantConnect.DataBento/Models/Enums/StatisticType.cs create mode 100644 QuantConnect.DataBento/Models/Header.cs create mode 100644 QuantConnect.DataBento/Models/LevelOneBookLevel.cs create mode 100644 QuantConnect.DataBento/Models/LevelOneData.cs create mode 100644 QuantConnect.DataBento/Models/MarketDataRecord.cs create mode 100644 QuantConnect.DataBento/Models/OhlcvBar.cs create mode 100644 QuantConnect.DataBento/Models/StatisticsData.cs create mode 100644 QuantConnect.DataBento/Serialization/JsonSettings.cs create mode 100644 QuantConnect.DataBento/Serialization/SnakeCaseContractResolver.cs diff --git a/Lean.DataSource.DataBento.sln b/Lean.DataSource.DataBento.sln index f2f03d6..bfecccc 100644 --- a/Lean.DataSource.DataBento.sln +++ b/Lean.DataSource.DataBento.sln @@ -2,9 +2,9 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.5.2.0 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.DataBento", "QuantConnect.DataBento\QuantConnect.DataSource.DataBento.csproj", "{367AEEDC-F0B3-7F47-539D-10E5EC242C2A}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.DataSource.DataBento", "QuantConnect.DataBento\QuantConnect.DataSource.DataBento.csproj", "{367AEEDC-F0B3-7F47-539D-10E5EC242C2A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.DataBento.Tests", "QuantConnect.DataBento.Tests\QuantConnect.DataSource.DataBento.Tests.csproj", "{9CF47860-2CEA-F379-09D8-9AEF27965D12}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QuantConnect.DataSource.DataBento.Tests", "QuantConnect.DataBento.Tests\QuantConnect.DataSource.DataBento.Tests.csproj", "{9CF47860-2CEA-F379-09D8-9AEF27965D12}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -16,10 +16,6 @@ Global {367AEEDC-F0B3-7F47-539D-10E5EC242C2A}.Debug|Any CPU.Build.0 = Debug|Any CPU {367AEEDC-F0B3-7F47-539D-10E5EC242C2A}.Release|Any CPU.ActiveCfg = Release|Any CPU {367AEEDC-F0B3-7F47-539D-10E5EC242C2A}.Release|Any CPU.Build.0 = Release|Any CPU - {4B379C8F-16CE-1972-73E3-C14F6410D428}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4B379C8F-16CE-1972-73E3-C14F6410D428}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4B379C8F-16CE-1972-73E3-C14F6410D428}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4B379C8F-16CE-1972-73E3-C14F6410D428}.Release|Any CPU.Build.0 = Release|Any CPU {9CF47860-2CEA-F379-09D8-9AEF27965D12}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {9CF47860-2CEA-F379-09D8-9AEF27965D12}.Debug|Any CPU.Build.0 = Debug|Any CPU {9CF47860-2CEA-F379-09D8-9AEF27965D12}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs index a5063df..7525823 100644 --- a/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,13 +17,10 @@ using System; using System.Linq; using NUnit.Framework; -using QuantConnect.Configuration; -using QuantConnect.Data; -using QuantConnect.Data.Market; -using QuantConnect.Lean.DataSource.DataBento; +using QuantConnect.Util; using QuantConnect.Logging; using QuantConnect.Securities; -using QuantConnect.Util; +using QuantConnect.Data.Market; namespace QuantConnect.Lean.DataSource.DataBento.Tests { @@ -31,20 +28,13 @@ namespace QuantConnect.Lean.DataSource.DataBento.Tests public class DataBentoDataDownloaderTests { private DataBentoDataDownloader _downloader; - private MarketHoursDatabase _marketHoursDatabase; - protected readonly string ApiKey = Config.Get("databento-api-key"); - private static Symbol CreateEsFuture() - { - var expiration = new DateTime(2026, 3, 20); - return Symbol.CreateFuture("ES", Market.CME, expiration); - } + private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); [SetUp] public void SetUp() { - _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); - _downloader = new DataBentoDataDownloader(ApiKey, _marketHoursDatabase); + _downloader = new DataBentoDataDownloader(); } [TearDown] @@ -60,15 +50,15 @@ public void TearDown() [TestCase(Resolution.Tick)] public void DownloadsTradeDataForLeanFuture(Resolution resolution) { - var symbol = CreateEsFuture(); + var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; - var startUtc = new DateTime(2024, 5, 1, 0, 0, 0, DateTimeKind.Utc); - var endUtc = new DateTime(2024, 5, 2, 0, 0, 0, DateTimeKind.Utc); + var startUtc = new DateTime(2026, 1, 18, 0, 0, 0, DateTimeKind.Utc); + var endUtc = new DateTime(2026, 1, 20, 0, 0, 0, DateTimeKind.Utc); if (resolution == Resolution.Tick) { - startUtc = new DateTime(2024, 5, 1, 9, 30, 0, DateTimeKind.Utc); + startUtc = new DateTime(2026, 1, 21, 9, 30, 0, DateTimeKind.Utc); endUtc = startUtc.AddMinutes(15); } @@ -120,10 +110,10 @@ public void DownloadsTradeDataForLeanFuture(Resolution resolution) [Test] public void DownloadsQuoteTicksForLeanFuture() { - var symbol = CreateEsFuture(); + var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; - var startUtc = new DateTime(2024, 5, 1, 9, 30, 0, DateTimeKind.Utc); + var startUtc = new DateTime(2026, 1, 20, 9, 30, 0, DateTimeKind.Utc); var endUtc = startUtc.AddMinutes(15); var parameters = new DataDownloaderGetParameters( @@ -166,10 +156,10 @@ public void DownloadsQuoteTicksForLeanFuture() [Test] public void DataIsSortedByTime() { - var symbol = CreateEsFuture(); + var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); - var startUtc = new DateTime(2024, 5, 1, 0, 0, 0, DateTimeKind.Utc); - var endUtc = new DateTime(2024, 5, 2, 0, 0, 0, DateTimeKind.Utc); + var startUtc = new DateTime(2026, 1, 20, 0, 0, 0, DateTimeKind.Utc); + var endUtc = new DateTime(2024, 1, 21, 0, 0, 0, DateTimeKind.Utc); var parameters = new DataDownloaderGetParameters( symbol, @@ -192,15 +182,5 @@ public void DataIsSortedByTime() ); } } - - [Test] - public void DisposeIsIdempotent() - { - var downloader = new DataBentoDataDownloader(ApiKey, - MarketHoursDatabase.FromDataFolder()); - - Assert.DoesNotThrow(downloader.Dispose); - Assert.DoesNotThrow(downloader.Dispose); - } } } diff --git a/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs index 94c78f8..f0d2e2f 100644 --- a/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,148 +19,101 @@ using NUnit.Framework; using QuantConnect.Data; using QuantConnect.Util; -using QuantConnect.Lean.DataSource.DataBento; using QuantConnect.Securities; using System.Collections.Generic; using QuantConnect.Logging; using QuantConnect.Data.Market; -using QuantConnect.Configuration; -namespace QuantConnect.Lean.DataSource.DataBento.Tests +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoDataProviderHistoryTests { - [TestFixture] - public class DataBentoDataProviderHistoryTests + private DataBentoProvider _historyDataProvider; + + [SetUp] + public void SetUp() { - private DataBentoProvider _historyDataProvider; - private MarketHoursDatabase _marketHoursDatabase; - protected readonly string ApiKey = Config.Get("databento-api-key"); + _historyDataProvider = new DataBentoProvider(); + } - private static Symbol CreateEsFuture() - { - var expiration = new DateTime(2026, 3, 20); - return Symbol.CreateFuture("ES", Market.CME, expiration); - } + [TearDown] + public void TearDown() + { + _historyDataProvider?.Dispose(); + } - [SetUp] - public void SetUp() + internal static IEnumerable TestParameters + { + get { - _historyDataProvider = new DataBentoProvider(); - } + var es = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); - [TearDown] - public void TearDown() - { - _historyDataProvider?.Dispose(); + yield return new TestCaseData(es, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(5), false); + yield return new TestCaseData(es, Resolution.Hour, TickType.Trade, TimeSpan.FromDays(2), false); + yield return new TestCaseData(es, Resolution.Minute, TickType.Trade, TimeSpan.FromHours(4), false); + yield return new TestCaseData(es, Resolution.Second, TickType.Trade, TimeSpan.FromHours(4), false); + yield return new TestCaseData(es, Resolution.Tick, TickType.Quote, TimeSpan.FromMinutes(15), true); } + } - internal static IEnumerable TestParameters - { - get - { - var es = CreateEsFuture(); - - yield return new TestCaseData(es, Resolution.Daily, TickType.Trade, TimeSpan.FromDays(5), false) - .SetDescription("ES futures daily trade history") - .SetCategory("Valid"); + [Test, TestCaseSource(nameof(TestParameters))] + public void GetsHistory(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period, bool expectsNoData) + { + var request = GetHistoryRequest(resolution, tickType, symbol, period); - yield return new TestCaseData(es, Resolution.Hour, TickType.Trade, TimeSpan.FromDays(2), false) - .SetDescription("ES futures hourly trade history") - .SetCategory("Valid"); + var history = _historyDataProvider.GetHistory(request); - yield return new TestCaseData(es, Resolution.Minute, TickType.Trade, TimeSpan.FromHours(4), false) - .SetDescription("ES futures minute trade history") - .SetCategory("Valid"); + Assert.IsNotNull(history); - yield return new TestCaseData(es, Resolution.Tick, TickType.Quote, TimeSpan.FromMinutes(15), false) - .SetDescription("ES futures quote ticks") - .SetCategory("Quote"); - } - } - - [Test, TestCaseSource(nameof(TestParameters))] - public void GetsHistory(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period, bool expectsNoData) + foreach (var point in history) { - var request = GetHistoryRequest(resolution, tickType, symbol, period); + Assert.AreEqual(symbol, point.Symbol); - var history = _historyDataProvider.GetHistory(request); - - if (expectsNoData) + if (point is TradeBar bar) { - Assert.IsTrue(history == null || !history.Any(), - $"Expected no data for unsupported symbol: {symbol}"); - return; + Assert.Greater(bar.Close, 0); + Assert.GreaterOrEqual(bar.Volume, 0); } - Assert.IsNotNull(history); - var data = history.ToList(); - Assert.IsNotEmpty(data); - - Log.Trace($"Received {data.Count} data points for {symbol} @ {resolution}"); - - foreach (var point in data.Take(5)) + if (point is Tick tick && tickType == TickType.Quote) { - Assert.AreEqual(symbol, point.Symbol); - - if (point is TradeBar bar) - { - Assert.Greater(bar.Close, 0); - Assert.GreaterOrEqual(bar.Volume, 0); - } - - if (point is Tick tick && tickType == TickType.Quote) - { - Assert.IsTrue(tick.BidPrice > 0 || tick.AskPrice > 0); - } + Assert.IsTrue(tick.BidPrice > 0 || tick.AskPrice > 0); } } + } - [Test] - public void GetHistoryWithMultipleSymbols() - { - var es = CreateEsFuture(); - - var request = GetHistoryRequest(Resolution.Daily, TickType.Trade, es, TimeSpan.FromDays(3)); - - var history = _historyDataProvider.GetHistory(request)?.ToList(); - - Assert.IsTrue( - history != null && history.Any(), - "Expected history for ES" - ); - } - - internal static HistoryRequest GetHistoryRequest( - Resolution resolution, - TickType tickType, - Symbol symbol, - TimeSpan period) - { - var endUtc = new DateTime(2024, 5, 10, 0, 0, 0, DateTimeKind.Utc); - var startUtc = endUtc - period; - - var dataType = LeanData.GetDataType(resolution, tickType); - var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); - - var exchangeHours = marketHoursDatabase.GetExchangeHours( - symbol.ID.Market, symbol, symbol.SecurityType); - - var dataTimeZone = marketHoursDatabase.GetDataTimeZone( - symbol.ID.Market, symbol, symbol.SecurityType); - - return new HistoryRequest( - startTimeUtc: startUtc, - endTimeUtc: endUtc, - dataType: dataType, - symbol: symbol, - resolution: resolution, - exchangeHours: exchangeHours, - dataTimeZone: dataTimeZone, - fillForwardResolution: resolution, - includeExtendedMarketHours: true, - isCustomData: false, - DataNormalizationMode.Raw, - tickType: tickType - ); - } + private static HistoryRequest GetHistoryRequest( + Resolution resolution, + TickType tickType, + Symbol symbol, + TimeSpan period) + { + var endUtc = new DateTime(2026, 1, 22); + var startUtc = endUtc - period; + + var dataType = LeanData.GetDataType(resolution, tickType); + var marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + + var exchangeHours = marketHoursDatabase.GetExchangeHours( + symbol.ID.Market, symbol, symbol.SecurityType); + + var dataTimeZone = marketHoursDatabase.GetDataTimeZone( + symbol.ID.Market, symbol, symbol.SecurityType); + + return new HistoryRequest( + startTimeUtc: startUtc, + endTimeUtc: endUtc, + dataType: dataType, + symbol: symbol, + resolution: resolution, + exchangeHours: exchangeHours, + dataTimeZone: dataTimeZone, + fillForwardResolution: resolution, + includeExtendedMarketHours: true, + isCustomData: false, + DataNormalizationMode.Raw, + tickType: tickType + ); } } diff --git a/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs new file mode 100644 index 0000000..7edac6d --- /dev/null +++ b/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs @@ -0,0 +1,95 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using NUnit.Framework; +using QuantConnect.Logging; +using QuantConnect.Configuration; +using QuantConnect.Lean.DataSource.DataBento.Api; + +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoHistoricalApiClientTests +{ + private HistoricalAPIClient _client; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + var apiKey = Config.Get("databento-api-key"); + if (string.IsNullOrEmpty(apiKey)) + { + Assert.Inconclusive("Please set the 'databento-api-key' in your configuration to enable these tests."); + } + + _client = new HistoricalAPIClient(apiKey); + } + + [TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Daily)] + [TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Hour)] + [TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Minute)] + [TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Second)] + //[TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Tick)] + [TestCase("ESH6 C6875", "2026/01/11", "2026/01/20", Resolution.Daily)] + public void CanInitializeHistoricalApiClient(string ticker, DateTime startDate, DateTime endDate, Resolution resolution) + { + var dataCounter = 0; + var previousEndTime = DateTime.MinValue; + foreach (var data in _client.GetHistoricalOhlcvBars(ticker, startDate, endDate, resolution, TickType.Trade)) + { + Assert.IsNotNull(data); + + Assert.Greater(data.Open, 0m); + Assert.Greater(data.High, 0m); + Assert.Greater(data.Low, 0m); + Assert.Greater(data.Close, 0m); + Assert.Greater(data.Volume, 0m); + Assert.AreNotEqual(default(DateTime), data.Header.UtcTime); + + Assert.IsTrue(data.Header.UtcTime > previousEndTime, + $"Bar at {data.Header.UtcTime:o} is not after previous bar at {previousEndTime:o}"); + previousEndTime = data.Header.UtcTime; + + dataCounter++; + } + + Log.Trace($"{nameof(CanInitializeHistoricalApiClient)}: {ticker} | [{startDate} - {endDate}] | {resolution} = {dataCounter} (bars)"); + Assert.Greater(dataCounter, 0); + } + + [TestCase("ESH6 C6875", "2026/01/11", "2026/01/20", Resolution.Daily)] + public void ShouldFetchOpenInterest(string ticker, DateTime startDate, DateTime endDate, Resolution resolution) + { + var dataCounter = 0; + var previousEndTime = DateTime.MinValue; + foreach (var data in _client.GetOpenInterest(ticker, startDate, endDate)) + { + Assert.IsNotNull(data); + + Assert.Greater(data.Quantity, 0m); + Assert.AreNotEqual(default(DateTime), data.Header.UtcTime); + + Assert.IsTrue(data.Header.UtcTime > previousEndTime, + $"Bar at {data.Header.UtcTime:o} is not after previous bar at {previousEndTime:o}"); + previousEndTime = data.Header.UtcTime; + + dataCounter++; + } + + Log.Trace($"{nameof(CanInitializeHistoricalApiClient)}: {ticker} | [{startDate} - {endDate}] | {resolution} = {dataCounter} (bars)"); + Assert.Greater(dataCounter, 0); + } +} diff --git a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs new file mode 100644 index 0000000..74d1126 --- /dev/null +++ b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs @@ -0,0 +1,135 @@ +using Newtonsoft.Json; +using NUnit.Framework; +using QuantConnect.Lean.DataSource.DataBento.Models; +using QuantConnect.Lean.DataSource.DataBento.Models.Enums; + +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoJsonConverterTests +{ + [Test] + public void DeserializeHistoricalOhlcvBar() + { + var json = @"{ + ""hd"": { + ""ts_event"": ""1738281600000000000"", + ""rtype"": 35, + ""publisher_id"": 1, + ""instrument_id"": 42140878 + }, + ""open"": ""6359.000000000"", + ""high"": ""6359.000000000"", + ""low"": ""6355.000000000"", + ""close"": ""6355.000000000"", + ""volume"": ""2"" +}"; + var res = json.DeserializeKebabCase(); + + Assert.IsNotNull(res); + + Assert.AreEqual(1738281600000000000m, res.Header.TsEvent); + Assert.AreEqual(35, res.Header.Rtype); + Assert.AreEqual(1, res.Header.PublisherId); + Assert.AreEqual(42140878, res.Header.InstrumentId); + + Assert.AreEqual(6359m, res.Open); + Assert.AreEqual(6359m, res.High); + Assert.AreEqual(6355m, res.Low); + Assert.AreEqual(6355m, res.Close); + Assert.AreEqual(2L, res.Volume); + } + + [Test] + public void DeserializeHistoricalLevelOneData() + { + var json = @"{ + ""ts_recv"": ""1768137063449660443"", + ""hd"": { + ""ts_event"": ""1768137063107829777"", + ""rtype"": 1, + ""publisher_id"": 1, + ""instrument_id"": 42140878 + }, + ""action"": ""A"", + ""side"": ""N"", + ""depth"": 0, + ""price"": ""7004.250000000"", + ""size"": 15, + ""flags"": 128, + ""ts_in_delta"": 17537, + ""sequence"": 811, + ""levels"": [ + { + ""bid_px"": ""7004.000000000"", + ""ask_px"": ""7004.250000000"", + ""bid_sz"": 11, + ""ask_sz"": 15, + ""bid_ct"": 1, + ""ask_ct"": 1 + } + ] +}"; + var res = json.DeserializeKebabCase(); + + Assert.IsNotNull(res); + + Assert.AreEqual(1768137063449660443, res.TsRecv); + + Assert.AreEqual(1768137063107829777, res.Header.TsEvent); + Assert.AreEqual(1, res.Header.Rtype); + Assert.AreEqual(1, res.Header.PublisherId); + Assert.AreEqual(42140878, res.Header.InstrumentId); + + Assert.AreEqual('A', res.Action); + Assert.AreEqual('N', res.Side); + Assert.AreEqual(0, res.Depth); + Assert.AreEqual(7004.25m, res.Price); + Assert.AreEqual(15, res.Size); + Assert.AreEqual(128, res.Flags); + Assert.IsNotNull(res.Levels); + Assert.AreEqual(1, res.Levels.Count); + var level = res.Levels[0]; + Assert.AreEqual(7004.0m, level.BidPx); + Assert.AreEqual(7004.25m, level.AskPx); + Assert.AreEqual(11, level.BidSz); + Assert.AreEqual(15, level.AskSz); + Assert.AreEqual(1, level.BidCt); + Assert.AreEqual(1, level.AskCt); + } + + [Test] + public void DeserializeHistoricalStatisticsData() + { + var json = @"{ + ""ts_recv"": ""1768156232522711477"", + ""hd"": { + ""ts_event"": ""1768156232522476283"", + ""rtype"": 24, + ""publisher_id"": 1, + ""instrument_id"": 42566722 + }, + ""ts_ref"": ""1767916800000000000"", + ""price"": null, + ""quantity"": 470, + ""sequence"": 29232, + ""ts_in_delta"": 12477, + ""stat_type"": 9, + ""channel_id"": 1, + ""update_action"": 1, + ""stat_flags"": 0 +}"; + + var res = json.DeserializeKebabCase(); + + Assert.IsNotNull(res); + + + Assert.AreEqual(1768156232522476283, res.Header.TsEvent); + Assert.AreEqual(24, res.Header.Rtype); + Assert.AreEqual(1, res.Header.PublisherId); + Assert.AreEqual(42566722, res.Header.InstrumentId); + Assert.AreEqual(470m, res.Quantity); + Assert.AreEqual(StatisticType.OpenInterest, res.StatType); + } +} diff --git a/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs index 22e7f04..10536a3 100644 --- a/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs b/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs new file mode 100644 index 0000000..471519a --- /dev/null +++ b/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs @@ -0,0 +1,56 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using NUnit.Framework; +using System; +using System.Collections.Generic; + +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoSymbolMapperTests +{ + /// + /// Provides the mapping between Lean symbols and brokerage specific symbols. + /// + private DataBentoSymbolMapper _symbolMapper; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + _symbolMapper = new DataBentoSymbolMapper(); + } + private static IEnumerable LeanSymbolTestCases + { + get + { + // TSLA - Equity + var es = Symbol.CreateFuture(Securities.Futures.Indices.SP500EMini, Market.CME, new DateTime(2026, 3, 20)); + yield return new TestCaseData(es, "ESH6"); + + } + } + + [Test, TestCaseSource(nameof(LeanSymbolTestCases))] + public void ReturnsCorrectBrokerageSymbol(Symbol symbol, string expectedBrokerageSymbol) + { + var brokerageSymbol = _symbolMapper.GetBrokerageSymbol(symbol); + + Assert.IsNotNull(brokerageSymbol); + Assert.IsNotEmpty(brokerageSymbol); + Assert.AreEqual(expectedBrokerageSymbol, brokerageSymbol); + } +} diff --git a/QuantConnect.DataBento.Tests/TestSetup.cs b/QuantConnect.DataBento.Tests/TestSetup.cs index 39cb9de..023eae0 100644 --- a/QuantConnect.DataBento.Tests/TestSetup.cs +++ b/QuantConnect.DataBento.Tests/TestSetup.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/QuantConnect.DataBento/Api/HistoricalAPIClient.cs b/QuantConnect.DataBento/Api/HistoricalAPIClient.cs new file mode 100644 index 0000000..92dc868 --- /dev/null +++ b/QuantConnect.DataBento/Api/HistoricalAPIClient.cs @@ -0,0 +1,168 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System.Net; +using System.Text; +using QuantConnect.Util; +using QuantConnect.Logging; +using System.Net.Http.Headers; +using QuantConnect.Lean.DataSource.DataBento.Models; + +namespace QuantConnect.Lean.DataSource.DataBento.Api; + +public class HistoricalAPIClient : IDisposable +{ + //private const string + + /// + /// Dataset for CME Globex futures + /// https://databento.com/docs/venues-and-datasets has more information on datasets through DataBento + /// + /// + /// TODO: Hard coded for now. Later on can add equities and options with different mapping + /// + private const string Dataset = "GLBX.MDP3"; + + private readonly HttpClient _httpClient = new() + { + BaseAddress = new Uri("https://hist.databento.com") + }; + + public HistoricalAPIClient(string apiKey) + { + _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + AuthenticationSchemes.Basic.ToString(), + // Basic Auth expects "username:password". Using ":" means API key with an empty password. + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{apiKey}:")) + ); + } + + public IEnumerable GetHistoricalOhlcvBars(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, Resolution resolution, TickType tickType) + { + string schema; + switch (resolution) + { + case Resolution.Second: + schema = "ohlcv-1s"; + break; + case Resolution.Minute: + schema = "ohlcv-1m"; + break; + case Resolution.Hour: + schema = "ohlcv-1h"; + break; + case Resolution.Daily: + schema = "ohlcv-1d"; + break; + default: + throw new ArgumentException($"Unsupported resolution {resolution} for OHLCV data."); + } + + return GetRange(symbol, startDateTimeUtc, endDateTimeUtc, schema); + } + + public IEnumerable GetTickBars(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc) + { + return GetRange(symbol, startDateTimeUtc, endDateTimeUtc, "mbp-1", useLimit: true); + } + + public IEnumerable GetOpenInterest(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc) + { + foreach (var statistics in GetRange(symbol, startDateTimeUtc, endDateTimeUtc, "statistics")) + { + if (statistics.StatType == Models.Enums.StatisticType.OpenInterest) + { + yield return statistics; + } + } + } + + private IEnumerable GetRange(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, string schema, bool useLimit = false) where T : MarketDataRecord + { + var formData = new Dictionary + { + { "dataset", Dataset }, + { "end", Time.DateTimeToUnixTimeStampNanoseconds(endDateTimeUtc).ToStringInvariant() }, + { "symbols", symbol }, + { "schema", schema }, + { "encoding", "json" }, + { "stype_in", "raw_symbol" }, + { "pretty_px", "true" }, + }; + + if (useLimit) + { + formData["limit"] = "10000"; + } + + var start = startDateTimeUtc; + var httpStatusCode = default(HttpStatusCode); + do + { + formData["start"] = Time.DateTimeToUnixTimeStampNanoseconds(start).ToStringInvariant(); + + using var content = new FormUrlEncodedContent(formData); + + using var requestMessage = new HttpRequestMessage(HttpMethod.Post, "/v0/timeseries.get_range") + { + Content = content + }; + + using var response = _httpClient.Send(requestMessage); + + if (response.Headers.TryGetValues("x-warning", out var warnings)) + { + foreach (var warning in warnings) + { + Log.Trace($"{nameof(HistoricalAPIClient)}.{nameof(GetRange)}: {warning}"); + } + } + + using var stream = response.Content.ReadAsStream(); + + if (stream.Length == 0) + { + continue; + } + + using var reader = new StreamReader(stream); + + var line = default(string); + if (response.StatusCode == HttpStatusCode.UnprocessableContent) + { + line = reader.ReadLine(); + Log.Trace($"{nameof(HistoricalAPIClient)}.{nameof(GetRange)}.Response: {line}. " + + $"Request: [{response.RequestMessage?.Method}]({response.RequestMessage?.RequestUri}), " + + $"Payload: {string.Join(", ", formData.Select(kvp => $"{kvp.Key}: {kvp.Value}"))}"); + yield break; + } + + httpStatusCode = response.EnsureSuccessStatusCode().StatusCode; + + var data = default(T); + while ((line = reader.ReadLine()) != null) + { + data = line.DeserializeKebabCase(); + yield return data; + } + start = data.Header.UtcTime.AddTicks(1); + } while (httpStatusCode == HttpStatusCode.PartialContent); + } + + public void Dispose() + { + _httpClient?.DisposeSafely(); + } +} diff --git a/QuantConnect.DataBento/DataBentoDataDownloader.cs b/QuantConnect.DataBento/DataBentoDataDownloader.cs index 841d070..9d78ce2 100644 --- a/QuantConnect.DataBento/DataBentoDataDownloader.cs +++ b/QuantConnect.DataBento/DataBentoDataDownloader.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,222 +14,84 @@ * */ -using System; -using System.Collections.Generic; -using System.Globalization; -using NodaTime; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Text; -using CsvHelper; -using QuantConnect.Configuration; using QuantConnect.Data; -using QuantConnect.Data.Market; -using QuantConnect.Lean.DataSource.DataBento.Models; using QuantConnect.Util; using QuantConnect.Securities; +using QuantConnect.Configuration; + +namespace QuantConnect.Lean.DataSource.DataBento; -namespace QuantConnect.Lean.DataSource.DataBento +/// +/// Data downloader class for pulling data from DataBento +/// +public class DataBentoDataDownloader : IDataDownloader, IDisposable { /// - /// Data downloader for historical data from DataBento's Raw HTTP API - /// Converts DataBento data to Lean data types + /// Provides access to historical market data via the DataBento service. /// - public class DataBentoDataDownloader : IDataDownloader, IDisposable - { - private readonly HttpClient _httpClient = new(); - private readonly string _apiKey; - private readonly DataBentoSymbolMapper _symbolMapper; - private readonly MarketHoursDatabase _marketHoursDatabase; - private readonly Dictionary _symbolExchangeTimeZones = new(); - - /// - /// Initializes a new instance of the - /// - /// The DataBento API key. - public DataBentoDataDownloader(string apiKey, MarketHoursDatabase marketHoursDatabase) - { - _marketHoursDatabase = marketHoursDatabase; - _apiKey = apiKey; - _httpClient.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{_apiKey}:"))); - _symbolMapper = new DataBentoSymbolMapper(); - } - - /// - /// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC). - /// - /// Parameters for the historical data request - /// Enumerable of base data for this symbol - /// - public IEnumerable Get(DataDownloaderGetParameters parameters) - { - var symbol = parameters.Symbol; - var resolution = parameters.Resolution; - var tickType = parameters.TickType; - - /// - /// Dataset for CME Globex futures - /// https://databento.com/docs/venues-and-datasets has more information on datasets through DataBento - /// - const string dataset = "GLBX.MDP3"; // hard coded for now. Later on can add equities and options with different mapping - var schema = GetSchema(resolution, tickType); - var databentoSymbol = _symbolMapper.GetBrokerageSymbol(symbol); + private readonly DataBentoProvider _historyProvider; - // prepare body for Raw HTTP request - var body = new StringBuilder() - .Append($"dataset={dataset}") - .Append($"&symbols={databentoSymbol}") - .Append($"&schema={schema}") - .Append($"&start={parameters.StartUtc:yyyy-MM-ddTHH:mm}") - .Append($"&end={parameters.EndUtc:yyyy-MM-ddTHH:mm}") - .Append("&stype_in=parent") - .Append("&encoding=csv") - .ToString(); - - using var request = new HttpRequestMessage(HttpMethod.Post, - "https://hist.databento.com/v0/timeseries.get_range") - { - Content = new StringContent(body, Encoding.UTF8, "application/x-www-form-urlencoded") - }; - - // send the request with the get range url - var response = _httpClient.Send(request); - - // Add error handling to see the actual error message - if (!response.IsSuccessStatusCode) - { - var errorContent = response.Content.ReadAsStringAsync().SynchronouslyAwaitTaskResult(); - throw new HttpRequestException($"DataBento API error ({response.StatusCode}): {errorContent}"); - } + /// + /// Provides exchange trading hours and market-specific time zone information. + /// + private readonly MarketHoursDatabase _marketHoursDatabase; - using var csv = new CsvReader( - new StreamReader(response.Content.ReadAsStream()), - CultureInfo.InvariantCulture - ); + /// + /// Initializes a new instance of the + /// getting the DataBento API key from the configuration + /// + public DataBentoDataDownloader() + : this(Config.Get("databento-api-key")) + { - return (tickType, resolution) switch - { - (TickType.Trade, Resolution.Tick) => - csv.ForEach(dt => - new Tick( - GetTickTime(symbol, dt.Timestamp), - symbol, - string.Empty, - string.Empty, - dt.Size, - dt.Price - ) - ), + } - (TickType.Trade, _) => - csv.ForEach(bar => - new TradeBar( - GetTickTime(symbol, bar.Timestamp), - symbol, - bar.Open, - bar.High, - bar.Low, - bar.Close, - bar.Volume - ) - ), + /// + /// Initializes a new instance of the + /// + /// The DataBento API key. + public DataBentoDataDownloader(string apiKey) + { + _historyProvider = new DataBentoProvider(apiKey); + _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + } - (TickType.Quote, Resolution.Tick) => - csv.ForEach(q => - new Tick( - GetTickTime(symbol, q.Timestamp), - symbol, - bidPrice: q.BidPrice, - askPrice: q.AskPrice, - bidSize: q.BidSize, - askSize: q.AskSize - ) - { - TickType = TickType.Quote - } - ), + /// + /// Get historical data enumerable for a single symbol, type and resolution given this start and end time (in UTC). + /// + /// Parameters for the historical data request + /// Enumerable of base data for this symbol + public IEnumerable? Get(DataDownloaderGetParameters parameters) + { + var symbol = parameters.Symbol; + var resolution = parameters.Resolution; + var startUtc = parameters.StartUtc; + var endUtc = parameters.EndUtc; + var tickType = parameters.TickType; - (TickType.Quote, _) => - csv.ForEach(q => - new QuoteBar( - GetTickTime(symbol, q.Timestamp), - symbol, - new Bar(q.BidPrice, q.BidPrice, q.BidPrice, q.BidPrice), q.BidSize, - new Bar(q.AskPrice, q.AskPrice, q.AskPrice, q.AskPrice), q.AskSize - ) - ), + var dataType = LeanData.GetDataType(resolution, tickType); + var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType); + var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType); - _ => throw new NotSupportedException( - $"Unsupported tickType={tickType} resolution={resolution}") - }; - } + var historyRequest = new HistoryRequest(startUtc, endUtc, dataType, symbol, resolution, exchangeHours, dataTimeZone, resolution, + true, false, DataNormalizationMode.Raw, tickType); - /// - /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. - /// - public void Dispose() - { - _httpClient?.DisposeSafely(); - } + var historyData = _historyProvider.GetHistory(historyRequest); - /// - /// Pick Databento schema from Lean resolution/ticktype - /// - private static string GetSchema(Resolution resolution, TickType tickType) + if (historyData == null) { - return (tickType, resolution) switch - { - (TickType.Trade, Resolution.Tick) => "mbp-1", - (TickType.Trade, Resolution.Second) => "ohlcv-1s", - (TickType.Trade, Resolution.Minute) => "ohlcv-1m", - (TickType.Trade, Resolution.Hour) => "ohlcv-1h", - (TickType.Trade, Resolution.Daily) => "ohlcv-1d", - - (TickType.Quote, _) => "mbp-1", - - _ => throw new NotSupportedException( - $"Unsupported resolution {resolution} / {tickType}" - ) - }; + return null; } - /// - /// Converts the given UTC time into the symbol security exchange time zone - /// - private DateTime GetTickTime(Symbol symbol, DateTime utcTime) - { - DateTimeZone exchangeTimeZone; - lock (_symbolExchangeTimeZones) - { - if (!_symbolExchangeTimeZones.TryGetValue(symbol, out exchangeTimeZone)) - { - // read the exchange time zone from market-hours-database - if (_marketHoursDatabase.TryGetEntry(symbol.ID.Market, symbol, symbol.SecurityType, out var entry)) - { - exchangeTimeZone = entry.ExchangeHours.TimeZone; - } - // If there is no entry for the given Symbol, default to New York - else - { - exchangeTimeZone = TimeZones.NewYork; - } - - _symbolExchangeTimeZones.Add(symbol, exchangeTimeZone); - } - } - - return utcTime.ConvertFromUtc(exchangeTimeZone); - } + return historyData; } -} -public static class CsvReaderExtensions -{ - public static IEnumerable ForEach( - this CsvReader csv, - Func map) + + /// + /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. + /// + public void Dispose() { - return csv.GetRecords().Select(map).ToList(); + _historyProvider.DisposeSafely(); } -} +} \ No newline at end of file diff --git a/QuantConnect.DataBento/DataBentoDataProvider.cs b/QuantConnect.DataBento/DataBentoDataProvider.cs index aa9dd32..c8d4eed 100644 --- a/QuantConnect.DataBento/DataBentoDataProvider.cs +++ b/QuantConnect.DataBento/DataBentoDataProvider.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,6 +24,7 @@ using QuantConnect.Packets; using QuantConnect.Securities; using System.Collections.Concurrent; +using QuantConnect.Lean.DataSource.DataBento.Api; namespace QuantConnect.Lean.DataSource.DataBento { @@ -34,18 +35,25 @@ namespace QuantConnect.Lean.DataSource.DataBento /// public partial class DataBentoProvider : IDataQueueHandler { + /// + /// Resolves map files to correctly handle current and historical ticker symbols. + /// + private readonly IMapFileProvider _mapFileProvider = Composer.Instance.GetPart(); + + private HistoricalAPIClient _historicalApiClient; + + private readonly DataBentoSymbolMapper _symbolMapper = new DataBentoSymbolMapper(); + private readonly IDataAggregator _dataAggregator = Composer.Instance.GetExportedValueByTypeName( Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false); private EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager; private DataBentoRawLiveClient _client; - private readonly DataBentoDataDownloader _dataDownloader; private bool _potentialUnsupportedResolutionMessageLogged; private bool _sessionStarted = false; private readonly object _sessionLock = new(); private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); private readonly ConcurrentDictionary _symbolExchangeTimeZones = new(); private bool _initialized; - private bool _unsupportedTickTypeMessagedLogged; /// /// Returns true if we're currently connected to the Data Provider @@ -56,9 +64,19 @@ public partial class DataBentoProvider : IDataQueueHandler /// Initializes a new instance of the DataBentoProvider /// public DataBentoProvider() + : this(Config.Get("databento-api-key")) { - var apiKey = Config.Get("databento-api-key"); - _dataDownloader = new DataBentoDataDownloader(apiKey, _marketHoursDatabase); + } + + public DataBentoProvider(string apiKey) + { + if (string.IsNullOrWhiteSpace(apiKey)) + { + // If the API key is not provided, we can't do anything. + // The handler might going to be initialized using a node packet job. + return; + } + Initialize(apiKey); } @@ -112,6 +130,8 @@ private void Initialize(string apiKey) cancellationTokenSource.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default); + + _historicalApiClient = new(apiKey); _initialized = true; Log.Debug("DataBentoProvider.Initialize(): Initialization complete"); @@ -166,11 +186,11 @@ public bool SubscriptionLogic(IEnumerable symbols, TickType tickType) /// /// The symbol /// returns true if Data Provider supports the specified symbol; otherwise false - private bool CanSubscribe(Symbol symbol) + private static bool CanSubscribe(Symbol symbol) { return !symbol.Value.Contains("universe", StringComparison.InvariantCultureIgnoreCase) && !symbol.IsCanonical() && - IsSecurityTypeSupported(symbol.SecurityType); + symbol.SecurityType == SecurityType.Future; } /// @@ -232,18 +252,6 @@ public void Dispose() _dataAggregator?.DisposeSafely(); _subscriptionManager?.DisposeSafely(); _client?.DisposeSafely(); - _dataDownloader?.DisposeSafely(); - } - - /// - /// Checks if the security type is supported - /// - /// Security type to check - /// True if supported - private bool IsSecurityTypeSupported(SecurityType securityType) - { - // DataBento primarily supports futures, but also has equity and option coverage - return securityType == SecurityType.Future; } /// diff --git a/QuantConnect.DataBento/DataBentoHistoryProivder.cs b/QuantConnect.DataBento/DataBentoHistoryProivder.cs index eb23154..b92744e 100644 --- a/QuantConnect.DataBento/DataBentoHistoryProivder.cs +++ b/QuantConnect.DataBento/DataBentoHistoryProivder.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,30 +16,34 @@ using NodaTime; using QuantConnect.Data; -using QuantConnect.Data.Market; -using QuantConnect.Lean.Engine.DataFeeds; -using QuantConnect.Lean.Engine.HistoricalData; -using QuantConnect.Logging; using QuantConnect.Util; -using QuantConnect.Interfaces; +using QuantConnect.Logging; using QuantConnect.Securities; +using QuantConnect.Data.Market; using QuantConnect.Data.Consolidators; +using QuantConnect.Lean.Engine.DataFeeds; +using QuantConnect.Lean.Engine.HistoricalData; namespace QuantConnect.Lean.DataSource.DataBento { /// - /// Impleements a history provider for DataBento historical data. + /// Implements a history provider for DataBento historical data. /// Uses consolidators to produce the requested resolution when necessary. /// public partial class DataBentoProvider : SynchronizingHistoryProvider { - private int _dataPointCount; + private static int _dataPointCount; /// /// Indicates whether a error for an invalid start time has been fired, where the start time is greater than or equal to the end time in UTC. /// private volatile bool _invalidStartTimeErrorFired; + /// + /// Indicates whether the warning for invalid has been fired. + /// + private volatile bool _invalidSecurityTypeWarningFired; + /// /// Gets the total number of data points emitted by this history provider /// @@ -64,11 +68,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) var subscriptions = new List(); foreach (var request in requests) { - var history = GetHistory(request); - if (history == null) - { - continue; - } + var history = request.SplitHistoryRequestWithUpdatedMappedSymbol(_mapFileProvider).SelectMany(x => GetHistory(x) ?? []); var subscription = CreateSubscription(request, history); if (!subscription.MoveNext()) @@ -89,54 +89,92 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) /// /// Gets the history for the requested security /// - /// The historical data request + /// The historical data request /// An enumerable of BaseData points - public IEnumerable? GetHistory(HistoryRequest request) + public IEnumerable? GetHistory(HistoryRequest historyRequest) { - if (!CanSubscribe(request.Symbol)) + if (!CanSubscribe(historyRequest.Symbol)) { - // It is Logged in IsSupported(...) - return null; - } - - if (request.TickType == TickType.OpenInterest) - { - if (!_unsupportedTickTypeMessagedLogged) + if (!_invalidSecurityTypeWarningFired) { - _unsupportedTickTypeMessagedLogged = true; - Log.Trace($"DataBentoProvider.GetHistory(): Unsupported tick type: {TickType.OpenInterest}"); + _invalidSecurityTypeWarningFired = true; + LogTrace(nameof(GetHistory), $"Unsupported SecurityType '{historyRequest.Symbol.SecurityType}' for symbol '{historyRequest.Symbol}'."); } return null; } - if (request.EndTimeUtc < request.StartTimeUtc) + if (historyRequest.EndTimeUtc < historyRequest.StartTimeUtc) { if (!_invalidStartTimeErrorFired) { _invalidStartTimeErrorFired = true; - Log.Error($"{nameof(DataBentoProvider)}.{nameof(GetHistory)}:InvalidDateRange. The history request start date must precede the end date, no history returned"); + Log.Error($"{nameof(DataBentoProvider)}.{nameof(GetHistory)}: Invalid date range: the start date must be earlier than the end date."); } return null; } + var history = default(IEnumerable); + var brokerageSymbol = _symbolMapper.GetBrokerageSymbol(historyRequest.Symbol); + switch (historyRequest.TickType) + { + case TickType.Trade when historyRequest.Resolution == Resolution.Tick: + history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol); + break; + case TickType.Trade: + history = GetAggregatedTradeBars(historyRequest, brokerageSymbol); + break; + case TickType.Quote: + history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol); + break; + case TickType.OpenInterest: + history = GetOpenInterestBars(historyRequest, brokerageSymbol); + break; + default: + throw new ArgumentException(""); + } - // Use the trade aggregates API for resolutions above tick for fastest results - if (request.TickType == TickType.Trade && request.Resolution > Resolution.Tick) + if (history == null) { - var data = GetAggregates(request); + return null; + } + + return FilterHistory(history, historyRequest, historyRequest.StartTimeLocal, historyRequest.EndTimeLocal); + } - if (data == null) + private static IEnumerable FilterHistory(IEnumerable history, HistoryRequest request, DateTime startTimeLocal, DateTime endTimeLocal) + { + // cleaning the data before returning it back to user + foreach (var bar in history) + { + if (bar.Time >= startTimeLocal && bar.EndTime <= endTimeLocal) { - return null; + if (request.ExchangeHours.IsOpen(bar.Time, bar.EndTime, request.IncludeExtendedMarketHours)) + { + Interlocked.Increment(ref _dataPointCount); + yield return bar; + } } + } + } - return data; + private IEnumerable GetOpenInterestBars(HistoryRequest request, string brokerageSymbol) + { + foreach (var oi in _historicalApiClient.GetOpenInterest(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) + { + yield return new OpenInterest(oi.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, oi.Quantity); } + } - return GetHistoryThroughDataConsolidator(request); + private IEnumerable GetAggregatedTradeBars(HistoryRequest request, string brokerageSymbol) + { + var period = request.Resolution.ToTimeSpan(); + foreach (var b in _historicalApiClient.GetHistoricalOhlcvBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc, request.Resolution, request.TickType)) + { + yield return new TradeBar(b.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, b.Open, b.High, b.Low, b.Close, b.Volume, period); + } } - private IEnumerable? GetHistoryThroughDataConsolidator(HistoryRequest request) + private IEnumerable? GetHistoryThroughDataConsolidator(HistoryRequest request, string brokerageSymbol) { IDataConsolidator consolidator; IEnumerable history; @@ -146,14 +184,14 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) consolidator = request.Resolution != Resolution.Tick ? new TickConsolidator(request.Resolution.ToTimeSpan()) : FilteredIdentityDataConsolidator.ForTickType(request.TickType); - history = GetTrades(request); + history = GetTrades(request, brokerageSymbol); } else { consolidator = request.Resolution != Resolution.Tick ? new TickQuoteBarConsolidator(request.Resolution.ToTimeSpan()) : FilteredIdentityDataConsolidator.ForTickType(request.TickType); - history = GetQuotes(request); + history = GetQuotes(request, brokerageSymbol); } BaseData? consolidatedData = null; @@ -168,7 +206,6 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) consolidator.Update(data); if (consolidatedData != null) { - Interlocked.Increment(ref _dataPointCount); yield return consolidatedData; consolidatedData = null; } @@ -179,47 +216,34 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) } /// - /// Gets the trade bars for the specified history request + /// Gets the trade ticks that will potentially be aggregated for the specified history request /// - private IEnumerable GetAggregates(HistoryRequest request) + private IEnumerable GetTrades(HistoryRequest request, string brokerageSymbol) { - var resolutionTimeSpan = request.Resolution.ToTimeSpan(); - foreach (var date in Time.EachDay(request.StartTimeUtc, request.EndTimeUtc)) + foreach (var t in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) { - var start = date; - var end = date + Time.OneDay; - - var parameters = new DataDownloaderGetParameters(request.Symbol, request.Resolution, start, end, request.TickType); - var data = _dataDownloader.Get(parameters); - if (data == null) continue; - - foreach (var bar in data) - { - var tradeBar = (TradeBar)bar; - if (tradeBar.Time >= request.StartTimeUtc && tradeBar.EndTime <= request.EndTimeUtc) - { - yield return tradeBar; - } - } + yield return new Tick(t.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, "", "", t.Size, t.Price); } } /// - /// Gets the trade ticks that will potentially be aggregated for the specified history request + /// Gets the quote ticks that will potentially be aggregated for the specified history request /// - private IEnumerable GetTrades(HistoryRequest request) + private IEnumerable GetQuotes(HistoryRequest request, string brokerageSymbol) { - var parameters = new DataDownloaderGetParameters(request.Symbol, Resolution.Tick, request.StartTimeUtc, request.EndTimeUtc, request.TickType); - return _dataDownloader.Get(parameters); + foreach (var quoteBar in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) + { + var time = quoteBar.Header.UtcTime.ConvertFromUtc(request.DataTimeZone); + foreach (var level in quoteBar.Levels) + { + yield return new Tick(time, request.Symbol, level.BidSz, level.BidPx, level.AskSz, level.AskPx); + } + } } - /// - /// Gets the quote ticks that will potentially be aggregated for the specified history request - /// - private IEnumerable GetQuotes(HistoryRequest request) + private static void LogTrace(string methodName, string message) { - var parameters = new DataDownloaderGetParameters(request.Symbol, Resolution.Tick, request.StartTimeUtc, request.EndTimeUtc, request.TickType); - return _dataDownloader.Get(parameters); + Log.Trace($"{nameof(DataBentoProvider)}.{methodName}: {message}"); } } } diff --git a/QuantConnect.DataBento/DataBentoRawLiveClient.cs b/QuantConnect.DataBento/DataBentoRawLiveClient.cs index 2dab1d2..8d8277c 100644 --- a/QuantConnect.DataBento/DataBentoRawLiveClient.cs +++ b/QuantConnect.DataBento/DataBentoRawLiveClient.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/QuantConnect.DataBento/DataBentoSymbolMapper.cs b/QuantConnect.DataBento/DataBentoSymbolMapper.cs index 3c6b8b3..5c90304 100644 --- a/QuantConnect.DataBento/DataBentoSymbolMapper.cs +++ b/QuantConnect.DataBento/DataBentoSymbolMapper.cs @@ -1,6 +1,6 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,9 +13,7 @@ * limitations under the License. */ -using QuantConnect; using QuantConnect.Brokerages; -using System.Globalization; namespace QuantConnect.Lean.DataSource.DataBento { @@ -24,9 +22,6 @@ namespace QuantConnect.Lean.DataSource.DataBento /// public class DataBentoSymbolMapper : ISymbolMapper { - private readonly Dictionary _leanSymbolsCache = new(); - private readonly Dictionary _brokerageSymbolsCache = new(); - private readonly object _locker = new(); /// /// Converts a Lean symbol instance to a brokerage symbol @@ -35,47 +30,12 @@ public class DataBentoSymbolMapper : ISymbolMapper /// The brokerage symbol public string GetBrokerageSymbol(Symbol symbol) { - if (symbol == null || string.IsNullOrWhiteSpace(symbol.Value)) + switch (symbol.SecurityType) { - throw new ArgumentException($"Invalid symbol: {(symbol == null ? "null" : symbol.ToString())}"); - } - - return GetBrokerageSymbol(symbol, false); - } - - /// - /// Converts a Lean symbol instance to a brokerage symbol with updating of cached symbol collection - /// - /// - /// - /// - /// - public string GetBrokerageSymbol(Symbol symbol, bool isUpdateCachedSymbol) - { - lock (_locker) - { - if (!_brokerageSymbolsCache.TryGetValue(symbol, out var brokerageSymbol) || isUpdateCachedSymbol) - { - switch (symbol.SecurityType) - { - case SecurityType.Future: - brokerageSymbol = $"{symbol.ID.Symbol}.FUT"; - break; - - case SecurityType.Equity: - brokerageSymbol = symbol.Value; - break; - - default: - throw new Exception($"DataBentoSymbolMapper.GetBrokerageSymbol(): unsupported security type: {symbol.SecurityType}"); - } - - // Lean-to-DataBento symbol conversion is accurate, so we can cache it both ways - _brokerageSymbolsCache[symbol] = brokerageSymbol; - _leanSymbolsCache[brokerageSymbol] = symbol; - } - - return brokerageSymbol; + case SecurityType.Future: + return SymbolRepresentation.GenerateFutureTicker(symbol.ID.Symbol, symbol.ID.Date, doubleDigitsYear: false, includeExpirationDate: false); + default: + throw new Exception($"The unsupported security type: {symbol.SecurityType}"); } } @@ -90,48 +50,12 @@ public string GetBrokerageSymbol(Symbol symbol, bool isUpdateCachedSymbol) public Symbol GetLeanSymbol(string brokerageSymbol, SecurityType securityType, string market, DateTime expirationDate = new DateTime(), decimal strike = 0, OptionRight optionRight = 0) { - if (string.IsNullOrWhiteSpace(brokerageSymbol)) - { - throw new ArgumentException("Invalid symbol: " + brokerageSymbol); - } - - lock (_locker) + switch (securityType) { - if (!_leanSymbolsCache.TryGetValue(brokerageSymbol, out var leanSymbol)) - { - switch (securityType) - { - case SecurityType.Future: - leanSymbol = Symbol.CreateFuture(brokerageSymbol, market, expirationDate); - break; - - default: - throw new Exception($"DataBentoSymbolMapper.GetLeanSymbol(): unsupported security type: {securityType}"); - } - - _leanSymbolsCache[brokerageSymbol] = leanSymbol; - _brokerageSymbolsCache[leanSymbol] = brokerageSymbol; - } - - return leanSymbol; - } - } - - /// - /// Gets the Lean symbol for the specified DataBento symbol - /// - /// The databento symbol - /// The corresponding Lean symbol - public Symbol GetLeanSymbol(string databentoSymbol) - { - lock (_locker) - { - if (!_leanSymbolsCache.TryGetValue(databentoSymbol, out var symbol)) - { - symbol = GetLeanSymbol(databentoSymbol, SecurityType.Equity, Market.USA); - } - - return symbol; + case SecurityType.Future: + return Symbol.CreateFuture(brokerageSymbol, market, expirationDate); + default: + throw new Exception($"The unsupported security type: {securityType}"); } } @@ -142,33 +66,13 @@ public Symbol GetLeanSymbol(string databentoSymbol) /// A new Lean Symbol instance public Symbol GetLeanSymbolForFuture(string brokerageSymbol) { - if (string.IsNullOrWhiteSpace(brokerageSymbol)) - { - throw new ArgumentException("Invalid symbol: " + brokerageSymbol); - } - // ignore futures spreads if (brokerageSymbol.Contains("-")) { return null; } - lock (_locker) - { - if (!_leanSymbolsCache.TryGetValue(brokerageSymbol, out var leanSymbol)) - { - leanSymbol = SymbolRepresentation.ParseFutureSymbol(brokerageSymbol); - - if (leanSymbol == null) - { - throw new ArgumentException("Invalid future symbol: " + brokerageSymbol); - } - - _leanSymbolsCache[brokerageSymbol] = leanSymbol; - } - - return leanSymbol; - } + return SymbolRepresentation.ParseFutureSymbol(brokerageSymbol); } } } diff --git a/QuantConnect.DataBento/Extensions.cs b/QuantConnect.DataBento/Extensions.cs new file mode 100644 index 0000000..14e8834 --- /dev/null +++ b/QuantConnect.DataBento/Extensions.cs @@ -0,0 +1,35 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; +using QuantConnect.Lean.DataSource.DataBento.Serialization; + +namespace QuantConnect.Lean.DataSource.DataBento; + +public static class Extensions +{ + /// + /// Deserializes the specified JSON string to an object of type + /// using snake-case property name resolution. + /// + /// The target type of the deserialized object. + /// The JSON string to deserialize. + /// The deserialized object of type . + public static T? DeserializeKebabCase(this string json) + { + return JsonConvert.DeserializeObject(json, JsonSettings.SnakeCase); + } +} diff --git a/QuantConnect.DataBento/Models/DataBentoTypes.cs b/QuantConnect.DataBento/Models/DataBentoTypes.cs deleted file mode 100644 index b7fa58c..0000000 --- a/QuantConnect.DataBento/Models/DataBentoTypes.cs +++ /dev/null @@ -1,112 +0,0 @@ -using System; -using CsvHelper.Configuration.Attributes; -using QuantConnect; - -namespace QuantConnect.Lean.DataSource.DataBento.Models -{ - /// - /// Provides a constant for scaling price values from DataBento. - /// - public static class PriceScaling - { - /// - /// price scale factor is needed to find the true price from the message - /// Due to compression each "1 unit corresponds to 1e-9, i.e. 1/1,000,000,000 or 0.000000001" - /// https://databento.com/docs/api-reference-live/basics/schemas-and-conventions?historical=raw&live=raw&reference=raw - /// - public const decimal PriceScaleFactor = 1e-9m; - } - - /// - /// Represents a single bar of historical data from DataBento. - /// This class is used to map CSV data from HTTP requests into a structured format. - /// - public class DatabentoBar - { - [Name("ts_event")] - public long TimestampNanos { get; set; } - - public DateTime Timestamp => Time.UnixNanosecondTimeStampToDateTime(TimestampNanos); - - [Name("open")] - public long RawOpen { get; set; } - - [Name("high")] - public long RawHigh { get; set; } - - [Name("low")] - public long RawLow { get; set; } - - - - [Name("close")] - public long RawClose { get; set; } - - [Ignore] - public decimal Open => RawOpen == long.MaxValue ? 0m : RawOpen * PriceScaling.PriceScaleFactor; - - [Ignore] - public decimal High => RawHigh == long.MaxValue ? 0m : RawHigh * PriceScaling.PriceScaleFactor; - - [Ignore] - public decimal Low => RawLow == long.MaxValue ? 0m : RawLow * PriceScaling.PriceScaleFactor; - - [Ignore] - public decimal Close => RawClose == long.MaxValue ? 0m : RawClose * PriceScaling.PriceScaleFactor; - - [Name("volume")] - public long RawVolume { get; set; } - - [Ignore] - public decimal Volume => RawVolume == long.MaxValue ? 0m : RawVolume; - } - - /// - /// Represents a single trade event from DataBento. - /// - public class DatabentoTrade - { - [Name("ts_event")] - public long TimestampNanos { get; set; } - - public DateTime Timestamp => Time.UnixNanosecondTimeStampToDateTime(TimestampNanos); - - [Name("price")] - public long RawPrice { get; set; } - - [Ignore] - public decimal Price => RawPrice == long.MaxValue ? 0m : RawPrice * PriceScaling.PriceScaleFactor; - - [Name("size")] - public int Size { get; set; } - } - - /// - /// Represents a single quote from DataBento. - /// - public class DatabentoQuote - { - [Name("ts_event")] - public long TimestampNanos { get; set; } - - public DateTime Timestamp => Time.UnixNanosecondTimeStampToDateTime(TimestampNanos); - - [Name("bid_px_00")] - public long RawBidPrice { get; set; } - - [Ignore] - public decimal BidPrice => RawBidPrice == long.MaxValue ? 0m : RawBidPrice * PriceScaling.PriceScaleFactor; - - [Name("bid_sz_00")] - public int BidSize { get; set; } - - [Name("ask_px_00")] - public long RawAskPrice { get; set; } - - [Ignore] - public decimal AskPrice => RawAskPrice == long.MaxValue ? 0m : RawAskPrice * PriceScaling.PriceScaleFactor; - - [Name("ask_sz_00")] - public int AskSize { get; set; } - } -} diff --git a/QuantConnect.DataBento/Models/Enums/StatisticType.cs b/QuantConnect.DataBento/Models/Enums/StatisticType.cs new file mode 100644 index 0000000..51be1c2 --- /dev/null +++ b/QuantConnect.DataBento/Models/Enums/StatisticType.cs @@ -0,0 +1,117 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Enums; + +/// +/// Identifies the type of statistical market data reported for an instrument. +/// +/// +/// The statistic type defines how price, quantity, stat_flags, +/// and ts_ref should be interpreted for a given statistics record. +/// +public enum StatisticType +{ + /// + /// The price of the first trade of an instrument. + /// + OpeningPrice = 1, + + /// + /// The probable price and quantity of the first trade of an instrument, + /// published during the pre-open phase. + /// + IndicativeOpeningPrice = 2, + + /// + /// The settlement price of an instrument. + /// + /// + /// stat_flags indicate whether the settlement price is final or preliminary, + /// and whether it is actual or theoretical. + /// + SettlementPrice = 3, + + /// + /// The lowest trade price of an instrument during the trading session. + /// + TradingSessionLowPrice = 4, + + /// + /// The highest trade price of an instrument during the trading session. + /// + TradingSessionHighPrice = 5, + + /// + /// The number of contracts cleared for an instrument on the previous trading date. + /// + ClearedVolume = 6, + + /// + /// The lowest offer price for an instrument during the trading session. + /// + LowestOffer = 7, + + /// + /// The highest bid price for an instrument during the trading session. + /// + HighestBid = 8, + + /// + /// The current number of outstanding contracts of an instrument. + /// + OpenInterest = 9, + + /// + /// The volume-weighted average price (VWAP) for a fixing period. + /// + FixingPrice = 10, + + /// + /// The last trade price and quantity during a trading session. + /// + ClosePrice = 11, + + /// + /// The change in price from the previous session's close price + /// to the most recent close price. + /// + NetChange = 12, + + /// + /// The volume-weighted average price (VWAP) during the trading session. + /// + VolumeWeightedAveragePrice = 13, + + /// + /// The implied volatility associated with the settlement price. + /// + Volatility = 14, + + /// + /// The options delta associated with the settlement price. + /// + Delta = 15, + + /// + /// The auction uncrossing price and quantity. + /// + /// + /// Used for auctions that are neither the official opening auction + /// nor the official closing auction. + /// + UncrossingPrice = 16 +} + diff --git a/QuantConnect.DataBento/Models/Header.cs b/QuantConnect.DataBento/Models/Header.cs new file mode 100644 index 0000000..2a2ead1 --- /dev/null +++ b/QuantConnect.DataBento/Models/Header.cs @@ -0,0 +1,48 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models; + +/// +/// Metadata header for a historical market data record. +/// Contains event timing, record type, data source, and instrument identifiers. +/// +public sealed class Header +{ + /// + /// Event timestamp in nanoseconds since Unix epoch (UTC). + /// + public long TsEvent { get; set; } + + /// + /// Record type identifier defining the data schema (e.g. trade, quote, bar). + /// + public int Rtype { get; set; } + + /// + /// DataBento publisher (exchange / data source) identifier. + /// + public int PublisherId { get; set; } + + /// + /// Internal instrument identifier for the symbol. + /// + public long InstrumentId { get; set; } + + /// + /// Event time converted to UTC . + /// + public DateTime UtcTime => Time.UnixNanosecondTimeStampToDateTime(TsEvent); +} diff --git a/QuantConnect.DataBento/Models/LevelOneBookLevel.cs b/QuantConnect.DataBento/Models/LevelOneBookLevel.cs new file mode 100644 index 0000000..3eb51f5 --- /dev/null +++ b/QuantConnect.DataBento/Models/LevelOneBookLevel.cs @@ -0,0 +1,49 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models; + +public sealed class LevelOneBookLevel +{ + /// + /// Bid price at this book level. + /// + public decimal BidPx { get; set; } + + /// + /// Ask price at this book level. + /// + public decimal AskPx { get; set; } + + /// + /// Total bid size at this level. + /// + public int BidSz { get; set; } + + /// + /// Total ask size at this level. + /// + public int AskSz { get; set; } + + /// + /// Number of bid orders at this level. + /// + public int BidCt { get; set; } + + /// + /// Number of ask orders at this level. + /// + public int AskCt { get; set; } +} diff --git a/QuantConnect.DataBento/Models/LevelOneData.cs b/QuantConnect.DataBento/Models/LevelOneData.cs new file mode 100644 index 0000000..a4d3454 --- /dev/null +++ b/QuantConnect.DataBento/Models/LevelOneData.cs @@ -0,0 +1,69 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models; + +/// +/// Represents a level-one market data update containing best bid and ask information. +/// +public sealed class LevelOneData : MarketDataRecord +{ + /// + /// Timestamp when the message was received by the gateway, + /// expressed as nanoseconds since the UNIX epoch. + /// + public long TsRecv { get; set; } + + /// + /// The event type or order book operation. Can be Add, Cancel, Modify, cleaR book, Trade, Fill, or None. + /// + public char Action { get; set; } + + /// + /// Side of the book affected by the update. + /// + public char Side { get; set; } + + /// + /// Book depth level affected by this update. + /// + public int Depth { get; set; } + + /// + /// Price associated with the update. + /// + public decimal Price { get; set; } + + /// + /// The side that initiates the event. + /// + /// + /// Can be: + /// - Ask for a sell order (or sell aggressor in a trade); + /// - Bid for a buy order (or buy aggressor in a trade); + /// - None where no side is specified by the original source. + /// + public int Size { get; set; } + + /// + /// A bit field indicating event end, message characteristics, and data quality. + /// + public int Flags { get; set; } + + /// + /// Snapshot of level-one bid and ask data. + /// + public IReadOnlyList Levels { get; set; } = []; +} \ No newline at end of file diff --git a/QuantConnect.DataBento/Models/MarketDataRecord.cs b/QuantConnect.DataBento/Models/MarketDataRecord.cs new file mode 100644 index 0000000..31501d2 --- /dev/null +++ b/QuantConnect.DataBento/Models/MarketDataRecord.cs @@ -0,0 +1,30 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.Lean.DataSource.DataBento.Models; + +/// +/// Base class for all market data records containing a standard metadata header. +/// +public abstract class MarketDataRecord +{ + /// + /// Gets or sets the standard metadata header for this market data record. + /// + [JsonProperty("hd")] + public required Header Header { get; set; } +} diff --git a/QuantConnect.DataBento/Models/OhlcvBar.cs b/QuantConnect.DataBento/Models/OhlcvBar.cs new file mode 100644 index 0000000..a7ce501 --- /dev/null +++ b/QuantConnect.DataBento/Models/OhlcvBar.cs @@ -0,0 +1,48 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models; + +/// +/// Open-High-Low-Close-Volume (OHLCV) bar representing aggregated market data +/// for a specific instrument and time interval. +/// +public sealed class OhlcvBar : MarketDataRecord +{ + /// + /// Opening price of the bar. + /// + public decimal Open { get; set; } + + /// + /// Highest traded price during the bar interval. + /// + public decimal High { get; set; } + + /// + /// Lowest traded price during the bar interval. + /// + public decimal Low { get; set; } + + /// + /// Closing price of the bar. + /// + public decimal Close { get; set; } + + /// + /// Total traded volume during the bar interval. + /// + public decimal Volume { get; set; } +} diff --git a/QuantConnect.DataBento/Models/StatisticsData.cs b/QuantConnect.DataBento/Models/StatisticsData.cs new file mode 100644 index 0000000..749127e --- /dev/null +++ b/QuantConnect.DataBento/Models/StatisticsData.cs @@ -0,0 +1,31 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using QuantConnect.Lean.DataSource.DataBento.Models.Enums; + +namespace QuantConnect.Lean.DataSource.DataBento.Models; + +public sealed class StatisticsData : MarketDataRecord +{ + /// + /// Quantity or value associated with the statistic. + /// + public decimal Quantity { get; set; } + + /// + /// Type of statistic represented by this record. + /// + public StatisticType StatType { get; set; } +} diff --git a/QuantConnect.DataBento/Serialization/JsonSettings.cs b/QuantConnect.DataBento/Serialization/JsonSettings.cs new file mode 100644 index 0000000..5247d97 --- /dev/null +++ b/QuantConnect.DataBento/Serialization/JsonSettings.cs @@ -0,0 +1,34 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.Lean.DataSource.DataBento.Serialization; + +/// +/// Provides globally accessible instances of +/// preconfigured with custom contract resolvers, such as snake-case formatting. +/// +public static class JsonSettings +{ + /// + /// Gets a reusable instance of that uses + /// for snake-case property name formatting. + /// + public static readonly JsonSerializerSettings SnakeCase = new() + { + ContractResolver = SnakeCaseContractResolver.Instance + }; +} diff --git a/QuantConnect.DataBento/Serialization/SnakeCaseContractResolver.cs b/QuantConnect.DataBento/Serialization/SnakeCaseContractResolver.cs new file mode 100644 index 0000000..cdc2319 --- /dev/null +++ b/QuantConnect.DataBento/Serialization/SnakeCaseContractResolver.cs @@ -0,0 +1,39 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json.Serialization; + +namespace QuantConnect.Lean.DataSource.DataBento.Serialization; + +/// +/// A singleton implementation that applies +/// a to JSON property names. +/// +public sealed class SnakeCaseContractResolver : DefaultContractResolver +{ + /// + /// Gets the singleton instance of the . + /// + public static readonly SnakeCaseContractResolver Instance = new(); + + /// + /// Initializes a new instance of the class + /// with a . + /// + private SnakeCaseContractResolver() + { + NamingStrategy = new SnakeCaseNamingStrategy(); + } +} From cbc4bfeb1e8798ac620c1d0b04ac4999dd7c23be Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 22 Jan 2026 10:47:46 +0200 Subject: [PATCH 07/18] refactor: clean, style, spacing, missed license block --- .../DataBentoDataDownloaderTests.cs | 253 ++-- .../DataBentoDataProviderHistoryTests.cs | 4 +- .../DataBentoJsonConverterTests.cs | 16 +- .../DataBentoRawLiveClientTests.cs | 202 ++- .../DataBentoSymbolMapperTests.cs.cs | 4 +- QuantConnect.DataBento.Tests/TestSetup.cs | 68 +- .../DataBentoDataProvider.cs | 515 ++++---- .../DataBentoHistoryProivder.cs | 343 +++-- .../DataBentoRawLiveClient.cs | 1102 ++++++++--------- .../DataBentoSymbolMapper.cs | 93 +- QuantConnect.DataBento/Extensions.cs | 1 - 11 files changed, 1301 insertions(+), 1300 deletions(-) diff --git a/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs index 7525823..b84893c 100644 --- a/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs @@ -22,165 +22,164 @@ using QuantConnect.Securities; using QuantConnect.Data.Market; -namespace QuantConnect.Lean.DataSource.DataBento.Tests +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoDataDownloaderTests { - [TestFixture] - public class DataBentoDataDownloaderTests + private DataBentoDataDownloader _downloader; + + private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + + [SetUp] + public void SetUp() { - private DataBentoDataDownloader _downloader; + _downloader = new DataBentoDataDownloader(); + } - private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + [TearDown] + public void TearDown() + { + _downloader?.Dispose(); + } - [SetUp] - public void SetUp() - { - _downloader = new DataBentoDataDownloader(); - } + [TestCase(Resolution.Daily)] + [TestCase(Resolution.Hour)] + [TestCase(Resolution.Minute)] + [TestCase(Resolution.Second)] + [TestCase(Resolution.Tick)] + public void DownloadsTradeDataForLeanFuture(Resolution resolution) + { + var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); + var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; - [TearDown] - public void TearDown() - { - _downloader?.Dispose(); - } + var startUtc = new DateTime(2026, 1, 18, 0, 0, 0, DateTimeKind.Utc); + var endUtc = new DateTime(2026, 1, 20, 0, 0, 0, DateTimeKind.Utc); - [TestCase(Resolution.Daily)] - [TestCase(Resolution.Hour)] - [TestCase(Resolution.Minute)] - [TestCase(Resolution.Second)] - [TestCase(Resolution.Tick)] - public void DownloadsTradeDataForLeanFuture(Resolution resolution) + if (resolution == Resolution.Tick) { - var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); - var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; + startUtc = new DateTime(2026, 1, 21, 9, 30, 0, DateTimeKind.Utc); + endUtc = startUtc.AddMinutes(15); + } - var startUtc = new DateTime(2026, 1, 18, 0, 0, 0, DateTimeKind.Utc); - var endUtc = new DateTime(2026, 1, 20, 0, 0, 0, DateTimeKind.Utc); + var parameters = new DataDownloaderGetParameters( + symbol, + resolution, + startUtc, + endUtc, + TickType.Trade + ); - if (resolution == Resolution.Tick) - { - startUtc = new DateTime(2026, 1, 21, 9, 30, 0, DateTimeKind.Utc); - endUtc = startUtc.AddMinutes(15); - } + var data = _downloader.Get(parameters).ToList(); - var parameters = new DataDownloaderGetParameters( - symbol, - resolution, - startUtc, - endUtc, - TickType.Trade - ); + Log.Trace($"Downloaded {data.Count} trade points for {symbol} @ {resolution}"); - var data = _downloader.Get(parameters).ToList(); + Assert.IsNotEmpty(data); - Log.Trace($"Downloaded {data.Count} trade points for {symbol} @ {resolution}"); + var startExchange = startUtc.ConvertFromUtc(exchangeTimeZone); + var endExchange = endUtc.ConvertFromUtc(exchangeTimeZone); - Assert.IsNotEmpty(data); - - var startExchange = startUtc.ConvertFromUtc(exchangeTimeZone); - var endExchange = endUtc.ConvertFromUtc(exchangeTimeZone); + foreach (var point in data) + { + Assert.AreEqual(symbol, point.Symbol); + Assert.That(point.Time, Is.InRange(startExchange, endExchange)); - foreach (var point in data) + switch (point) { - Assert.AreEqual(symbol, point.Symbol); - Assert.That(point.Time, Is.InRange(startExchange, endExchange)); - - switch (point) - { - case TradeBar bar: - Assert.Greater(bar.Open, 0); - Assert.Greater(bar.High, 0); - Assert.Greater(bar.Low, 0); - Assert.Greater(bar.Close, 0); - Assert.GreaterOrEqual(bar.Volume, 0); - Assert.GreaterOrEqual(bar.High, bar.Low); - break; - - case Tick tick: - Assert.Greater(tick.Value, 0); - Assert.GreaterOrEqual(tick.Quantity, 0); - break; - - default: - Assert.Fail($"Unexpected data type {point.GetType()}"); - break; - } + case TradeBar bar: + Assert.Greater(bar.Open, 0); + Assert.Greater(bar.High, 0); + Assert.Greater(bar.Low, 0); + Assert.Greater(bar.Close, 0); + Assert.GreaterOrEqual(bar.Volume, 0); + Assert.GreaterOrEqual(bar.High, bar.Low); + break; + + case Tick tick: + Assert.Greater(tick.Value, 0); + Assert.GreaterOrEqual(tick.Quantity, 0); + break; + + default: + Assert.Fail($"Unexpected data type {point.GetType()}"); + break; } } + } - [Test] - public void DownloadsQuoteTicksForLeanFuture() - { - var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); - var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; - - var startUtc = new DateTime(2026, 1, 20, 9, 30, 0, DateTimeKind.Utc); - var endUtc = startUtc.AddMinutes(15); - - var parameters = new DataDownloaderGetParameters( - symbol, - Resolution.Tick, - startUtc, - endUtc, - TickType.Quote - ); + [Test] + public void DownloadsQuoteTicksForLeanFuture() + { + var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); + var exchangeTimeZone = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType).TimeZone; + + var startUtc = new DateTime(2026, 1, 20, 9, 30, 0, DateTimeKind.Utc); + var endUtc = startUtc.AddMinutes(15); - var data = _downloader.Get(parameters).ToList(); + var parameters = new DataDownloaderGetParameters( + symbol, + Resolution.Tick, + startUtc, + endUtc, + TickType.Quote + ); - Log.Trace($"Downloaded {data.Count} quote ticks for {symbol}"); + var data = _downloader.Get(parameters).ToList(); - Assert.IsNotEmpty(data); + Log.Trace($"Downloaded {data.Count} quote ticks for {symbol}"); - var startExchange = startUtc.ConvertFromUtc(exchangeTimeZone); - var endExchange = endUtc.ConvertFromUtc(exchangeTimeZone); + Assert.IsNotEmpty(data); - foreach (var point in data) + var startExchange = startUtc.ConvertFromUtc(exchangeTimeZone); + var endExchange = endUtc.ConvertFromUtc(exchangeTimeZone); + + foreach (var point in data) + { + Assert.AreEqual(symbol, point.Symbol); + Assert.That(point.Time, Is.InRange(startExchange, endExchange)); + + if (point is Tick tick) + { + Assert.AreEqual(TickType.Quote, tick.TickType); + Assert.IsTrue( + tick.BidPrice > 0 || tick.AskPrice > 0, + "Quote tick must have bid or ask" + ); + } + else if (point is QuoteBar bar) { - Assert.AreEqual(symbol, point.Symbol); - Assert.That(point.Time, Is.InRange(startExchange, endExchange)); - - if (point is Tick tick) - { - Assert.AreEqual(TickType.Quote, tick.TickType); - Assert.IsTrue( - tick.BidPrice > 0 || tick.AskPrice > 0, - "Quote tick must have bid or ask" - ); - } - else if (point is QuoteBar bar) - { - Assert.IsTrue(bar.Bid != null || bar.Ask != null); - } + Assert.IsTrue(bar.Bid != null || bar.Ask != null); } } + } - [Test] - public void DataIsSortedByTime() - { - var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); + [Test] + public void DataIsSortedByTime() + { + var symbol = Symbol.CreateFuture("ES", Market.CME, new DateTime(2026, 3, 20)); - var startUtc = new DateTime(2026, 1, 20, 0, 0, 0, DateTimeKind.Utc); - var endUtc = new DateTime(2024, 1, 21, 0, 0, 0, DateTimeKind.Utc); + var startUtc = new DateTime(2026, 1, 20, 0, 0, 0, DateTimeKind.Utc); + var endUtc = new DateTime(2024, 1, 21, 0, 0, 0, DateTimeKind.Utc); - var parameters = new DataDownloaderGetParameters( - symbol, - Resolution.Minute, - startUtc, - endUtc, - TickType.Trade - ); + var parameters = new DataDownloaderGetParameters( + symbol, + Resolution.Minute, + startUtc, + endUtc, + TickType.Trade + ); - var data = _downloader.Get(parameters).ToList(); + var data = _downloader.Get(parameters).ToList(); - Assert.IsNotEmpty(data); + Assert.IsNotEmpty(data); - for (int i = 1; i < data.Count; i++) - { - Assert.GreaterOrEqual( - data[i].Time, - data[i - 1].Time, - $"Data not sorted at index {i}" - ); - } + for (int i = 1; i < data.Count; i++) + { + Assert.GreaterOrEqual( + data[i].Time, + data[i - 1].Time, + $"Data not sorted at index {i}" + ); } } } diff --git a/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs index f0d2e2f..27a52d3 100644 --- a/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs @@ -15,14 +15,12 @@ */ using System; -using System.Linq; using NUnit.Framework; using QuantConnect.Data; using QuantConnect.Util; using QuantConnect.Securities; -using System.Collections.Generic; -using QuantConnect.Logging; using QuantConnect.Data.Market; +using System.Collections.Generic; namespace QuantConnect.Lean.DataSource.DataBento.Tests; diff --git a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs index 74d1126..9d43db8 100644 --- a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs @@ -1,4 +1,18 @@ -using Newtonsoft.Json; +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + using NUnit.Framework; using QuantConnect.Lean.DataSource.DataBento.Models; using QuantConnect.Lean.DataSource.DataBento.Models.Enums; diff --git a/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs index 10536a3..ec03477 100644 --- a/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs @@ -15,134 +15,132 @@ */ using System; -using System.Threading; using NUnit.Framework; -using QuantConnect.Configuration; +using System.Threading; using QuantConnect.Data; -using QuantConnect.Data.Market; -using QuantConnect.Lean.DataSource.DataBento; using QuantConnect.Logging; +using QuantConnect.Data.Market; +using QuantConnect.Configuration; -namespace QuantConnect.Lean.DataSource.DataBento.Tests +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoRawLiveClientSyncTests { - [TestFixture] - public class DataBentoRawLiveClientSyncTests + private DataBentoRawLiveClient _client; + protected readonly string ApiKey = Config.Get("databento-api-key"); + + private static Symbol CreateEsFuture() { - private DataBentoRawLiveClient _client; - protected readonly string ApiKey = Config.Get("databento-api-key"); + var expiration = new DateTime(2026, 3, 20); + return Symbol.CreateFuture("ES", Market.CME, expiration); + } - private static Symbol CreateEsFuture() - { - var expiration = new DateTime(2026, 3, 20); - return Symbol.CreateFuture("ES", Market.CME, expiration); - } + [SetUp] + public void SetUp() + { + Log.Trace("DataBentoLiveClientTests: Using API Key: " + ApiKey); + _client = new DataBentoRawLiveClient(ApiKey); + } - [SetUp] - public void SetUp() - { - Log.Trace("DataBentoLiveClientTests: Using API Key: " + ApiKey); - _client = new DataBentoRawLiveClient(ApiKey); - } + [TearDown] + public void TearDown() + { + _client?.Dispose(); + } - [TearDown] - public void TearDown() - { - _client?.Dispose(); - } + [Test] + public void Connects() + { + var connected = _client.Connect(); - [Test] - public void Connects() - { - var connected = _client.Connect(); + Assert.IsTrue(connected); + Assert.IsTrue(_client.IsConnected); - Assert.IsTrue(connected); - Assert.IsTrue(_client.IsConnected); + Log.Trace("Connected successfully"); + } - Log.Trace("Connected successfully"); - } + [Test] + public void SubscribesToLeanFutureSymbol() + { + Assert.IsTrue(_client.Connect()); + + var symbol = CreateEsFuture(); + + Assert.IsTrue(_client.Subscribe(symbol, TickType.Trade)); + Assert.IsTrue(_client.StartSession()); + + Thread.Sleep(1000); + + Assert.IsTrue(_client.Unsubscribe(symbol)); + } - [Test] - public void SubscribesToLeanFutureSymbol() + [Test] + public void ReceivesTradeOrQuoteTicks() + { + var receivedEvent = new ManualResetEventSlim(false); + BaseData received = null; + + _client.DataReceived += (_, data) => { - Assert.IsTrue(_client.Connect()); + received = data; + receivedEvent.Set(); + }; - var symbol = CreateEsFuture(); + Assert.IsTrue(_client.Connect()); - Assert.IsTrue(_client.Subscribe(symbol, TickType.Trade)); - Assert.IsTrue(_client.StartSession()); + var symbol = CreateEsFuture(); - Thread.Sleep(1000); + Assert.IsTrue(_client.Subscribe(symbol, TickType.Trade)); + Assert.IsTrue(_client.StartSession()); - Assert.IsTrue(_client.Unsubscribe(symbol)); - } + var gotData = receivedEvent.Wait(TimeSpan.FromMinutes(2)); - [Test] - public void ReceivesTradeOrQuoteTicks() + if (!gotData) { - var receivedEvent = new ManualResetEventSlim(false); - BaseData received = null; - - _client.DataReceived += (_, data) => - { - received = data; - receivedEvent.Set(); - }; - - Assert.IsTrue(_client.Connect()); - - var symbol = CreateEsFuture(); - - Assert.IsTrue(_client.Subscribe(symbol, TickType.Trade)); - Assert.IsTrue(_client.StartSession()); - - var gotData = receivedEvent.Wait(TimeSpan.FromMinutes(2)); - - if (!gotData) - { - Assert.Inconclusive("No data received (likely outside market hours)"); - return; - } - - Assert.NotNull(received); - Assert.AreEqual(symbol, received.Symbol); - - if (received is Tick tick) - { - Assert.Greater(tick.Time, DateTime.MinValue); - Assert.Greater(tick.Value, 0); - } - else if (received is TradeBar bar) - { - Assert.Greater(bar.Close, 0); - } - else - { - Assert.Fail($"Unexpected data type: {received.GetType()}"); - } + Assert.Inconclusive("No data received (likely outside market hours)"); + return; } - [Test] - public void DisposeIsIdempotent() + Assert.NotNull(received); + Assert.AreEqual(symbol, received.Symbol); + + if (received is Tick tick) { - var client = new DataBentoRawLiveClient(ApiKey); - Assert.DoesNotThrow(client.Dispose); - Assert.DoesNotThrow(client.Dispose); + Assert.Greater(tick.Time, DateTime.MinValue); + Assert.Greater(tick.Value, 0); } - - [Test] - public void SymbolMappingDoesNotThrow() + else if (received is TradeBar bar) { - Assert.IsTrue(_client.Connect()); + Assert.Greater(bar.Close, 0); + } + else + { + Assert.Fail($"Unexpected data type: {received.GetType()}"); + } + } - var symbol = CreateEsFuture(); + [Test] + public void DisposeIsIdempotent() + { + var client = new DataBentoRawLiveClient(ApiKey); + Assert.DoesNotThrow(client.Dispose); + Assert.DoesNotThrow(client.Dispose); + } - Assert.DoesNotThrow(() => - { - _client.Subscribe(symbol, TickType.Trade); - _client.StartSession(); - Thread.Sleep(500); - _client.Unsubscribe(symbol); - }); - } + [Test] + public void SymbolMappingDoesNotThrow() + { + Assert.IsTrue(_client.Connect()); + + var symbol = CreateEsFuture(); + + Assert.DoesNotThrow(() => + { + _client.Subscribe(symbol, TickType.Trade); + _client.StartSession(); + Thread.Sleep(500); + _client.Unsubscribe(symbol); + }); } } diff --git a/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs b/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs index 471519a..ad14aad 100644 --- a/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs +++ b/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs @@ -14,8 +14,8 @@ * */ -using NUnit.Framework; using System; +using NUnit.Framework; using System.Collections.Generic; namespace QuantConnect.Lean.DataSource.DataBento.Tests; @@ -40,7 +40,7 @@ private static IEnumerable LeanSymbolTestCases // TSLA - Equity var es = Symbol.CreateFuture(Securities.Futures.Indices.SP500EMini, Market.CME, new DateTime(2026, 3, 20)); yield return new TestCaseData(es, "ESH6"); - + } } diff --git a/QuantConnect.DataBento.Tests/TestSetup.cs b/QuantConnect.DataBento.Tests/TestSetup.cs index 023eae0..86d2f07 100644 --- a/QuantConnect.DataBento.Tests/TestSetup.cs +++ b/QuantConnect.DataBento.Tests/TestSetup.cs @@ -4,7 +4,6 @@ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software @@ -15,52 +14,51 @@ */ using System; -using System.Collections; using System.IO; using NUnit.Framework; -using QuantConnect.Configuration; +using System.Collections; using QuantConnect.Logging; +using QuantConnect.Configuration; + +namespace QuantConnect.Lean.DataSource.DataBento.Tests; -namespace QuantConnect.Lean.DataSource.DataBento.Tests +[SetUpFixture] +public class TestSetup { - [SetUpFixture] - public class TestSetup + [OneTimeSetUp] + public void GlobalSetup() { - [OneTimeSetUp] - public void GlobalSetup() - { - Log.DebuggingEnabled = true; - Log.LogHandler = new CompositeLogHandler(); - Log.Trace("TestSetup(): starting..."); - ReloadConfiguration(); - } + Log.DebuggingEnabled = true; + Log.LogHandler = new CompositeLogHandler(); + Log.Trace("TestSetup(): starting..."); + ReloadConfiguration(); + } + + private static void ReloadConfiguration() + { + // nunit 3 sets the current folder to a temp folder we need it to be the test bin output folder + var dir = TestContext.CurrentContext.TestDirectory; + Environment.CurrentDirectory = dir; + Directory.SetCurrentDirectory(dir); + // reload config from current path + Config.Reset(); - private static void ReloadConfiguration() + var environment = Environment.GetEnvironmentVariables(); + foreach (DictionaryEntry entry in environment) { - // nunit 3 sets the current folder to a temp folder we need it to be the test bin output folder - var dir = TestContext.CurrentContext.TestDirectory; - Environment.CurrentDirectory = dir; - Directory.SetCurrentDirectory(dir); - // reload config from current path - Config.Reset(); + var envKey = entry.Key.ToString(); + var value = entry.Value.ToString(); - var environment = Environment.GetEnvironmentVariables(); - foreach (DictionaryEntry entry in environment) + if (envKey.StartsWith("QC_")) { - var envKey = entry.Key.ToString(); - var value = entry.Value.ToString(); + var key = envKey.Substring(3).Replace("_", "-").ToLower(); - if (envKey.StartsWith("QC_")) - { - var key = envKey.Substring(3).Replace("_", "-").ToLower(); - - Log.Trace($"TestSetup(): Updating config setting '{key}' from environment var '{envKey}'"); - Config.Set(key, value); - } + Log.Trace($"TestSetup(): Updating config setting '{key}' from environment var '{envKey}'"); + Config.Set(key, value); } - - // resets the version among other things - Globals.Reset(); } + + // resets the version among other things + Globals.Reset(); } } diff --git a/QuantConnect.DataBento/DataBentoDataProvider.cs b/QuantConnect.DataBento/DataBentoDataProvider.cs index c8d4eed..40fc095 100644 --- a/QuantConnect.DataBento/DataBentoDataProvider.cs +++ b/QuantConnect.DataBento/DataBentoDataProvider.cs @@ -16,342 +16,341 @@ using NodaTime; using QuantConnect.Data; -using QuantConnect.Data.Market; using QuantConnect.Util; -using QuantConnect.Interfaces; -using QuantConnect.Configuration; using QuantConnect.Logging; using QuantConnect.Packets; +using QuantConnect.Interfaces; using QuantConnect.Securities; +using QuantConnect.Data.Market; +using QuantConnect.Configuration; using System.Collections.Concurrent; using QuantConnect.Lean.DataSource.DataBento.Api; -namespace QuantConnect.Lean.DataSource.DataBento +namespace QuantConnect.Lean.DataSource.DataBento; + +/// +/// A data Provider for DataBento that provides live market data and historical data. +/// Handles Subscribing, Unsubscribing, and fetching historical data from DataBento. +/// It will handle if a symbol is subscribable and will log errors if it is not. +/// +public partial class DataBentoProvider : IDataQueueHandler { /// - /// A data Provider for DataBento that provides live market data and historical data. - /// Handles Subscribing, Unsubscribing, and fetching historical data from DataBento. - /// It will handle if a symbol is subscribable and will log errors if it is not. + /// Resolves map files to correctly handle current and historical ticker symbols. /// - public partial class DataBentoProvider : IDataQueueHandler + private readonly IMapFileProvider _mapFileProvider = Composer.Instance.GetPart(); + + private HistoricalAPIClient _historicalApiClient; + + private readonly DataBentoSymbolMapper _symbolMapper = new(); + + private readonly IDataAggregator _dataAggregator = Composer.Instance.GetExportedValueByTypeName( + Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false); + private EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager; + private DataBentoRawLiveClient _client; + private bool _potentialUnsupportedResolutionMessageLogged; + private bool _sessionStarted = false; + private readonly object _sessionLock = new(); + private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); + private readonly ConcurrentDictionary _symbolExchangeTimeZones = new(); + private bool _initialized; + + /// + /// Returns true if we're currently connected to the Data Provider + /// + public bool IsConnected => _client?.IsConnected == true; + + /// + /// Initializes a new instance of the DataBentoProvider + /// + public DataBentoProvider() + : this(Config.Get("databento-api-key")) { - /// - /// Resolves map files to correctly handle current and historical ticker symbols. - /// - private readonly IMapFileProvider _mapFileProvider = Composer.Instance.GetPart(); - - private HistoricalAPIClient _historicalApiClient; - - private readonly DataBentoSymbolMapper _symbolMapper = new DataBentoSymbolMapper(); - - private readonly IDataAggregator _dataAggregator = Composer.Instance.GetExportedValueByTypeName( - Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false); - private EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager; - private DataBentoRawLiveClient _client; - private bool _potentialUnsupportedResolutionMessageLogged; - private bool _sessionStarted = false; - private readonly object _sessionLock = new(); - private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); - private readonly ConcurrentDictionary _symbolExchangeTimeZones = new(); - private bool _initialized; - - /// - /// Returns true if we're currently connected to the Data Provider - /// - public bool IsConnected => _client?.IsConnected == true; - - /// - /// Initializes a new instance of the DataBentoProvider - /// - public DataBentoProvider() - : this(Config.Get("databento-api-key")) + } + + public DataBentoProvider(string apiKey) + { + if (string.IsNullOrWhiteSpace(apiKey)) { + // If the API key is not provided, we can't do anything. + // The handler might going to be initialized using a node packet job. + return; } - public DataBentoProvider(string apiKey) + Initialize(apiKey); + } + + /// + /// Common initialization logic + /// DataBento API key from config file retrieved on constructor + /// + private void Initialize(string apiKey) + { + Log.Debug("DataBentoProvider.Initialize(): Starting initialization"); + _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager() { - if (string.IsNullOrWhiteSpace(apiKey)) + SubscribeImpl = (symbols, tickType) => + { + return SubscriptionLogic(symbols, tickType); + }, + UnsubscribeImpl = (symbols, tickType) => { - // If the API key is not provided, we can't do anything. - // The handler might going to be initialized using a node packet job. - return; + return UnsubscribeLogic(symbols, tickType); } + }; - Initialize(apiKey); - } + // Initialize the live client + _client = new DataBentoRawLiveClient(apiKey); + _client.DataReceived += OnDataReceived; - /// - /// Common initialization logic - /// DataBento API key from config file retrieved on constructor - /// - private void Initialize(string apiKey) + // Connect to live gateway + Log.Debug("DataBentoProvider.Initialize(): Attempting connection to DataBento live gateway"); + var cancellationTokenSource = new CancellationTokenSource(); + Task.Factory.StartNew(() => { - Log.Debug("DataBentoProvider.Initialize(): Starting initialization"); - _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager() + try { - SubscribeImpl = (symbols, tickType) => - { - return SubscriptionLogic(symbols, tickType); - }, - UnsubscribeImpl = (symbols, tickType) => - { - return UnsubscribeLogic(symbols, tickType); - } - }; + var connected = _client.Connect(); + Log.Debug($"DataBentoProvider.Initialize(): Connect() returned {connected}"); - // Initialize the live client - _client = new DataBentoRawLiveClient(apiKey); - _client.DataReceived += OnDataReceived; - - // Connect to live gateway - Log.Debug("DataBentoProvider.Initialize(): Attempting connection to DataBento live gateway"); - var cancellationTokenSource = new CancellationTokenSource(); - Task.Factory.StartNew(() => - { - try + if (connected) { - var connected = _client.Connect(); - Log.Debug($"DataBentoProvider.Initialize(): Connect() returned {connected}"); - - if (connected) - { - Log.Debug("DataBentoProvider.Initialize(): Successfully connected to DataBento live gateway"); - } - else - { - Log.Error("DataBentoProvider.Initialize(): Failed to connect to DataBento live gateway"); - } + Log.Debug("DataBentoProvider.Initialize(): Successfully connected to DataBento live gateway"); } - catch (Exception ex) + else { - Log.Error($"DataBentoProvider.Initialize(): Exception during Connect(): {ex.Message}\n{ex.StackTrace}"); + Log.Error("DataBentoProvider.Initialize(): Failed to connect to DataBento live gateway"); } - }, - cancellationTokenSource.Token, - TaskCreationOptions.LongRunning, - TaskScheduler.Default); + } + catch (Exception ex) + { + Log.Error($"DataBentoProvider.Initialize(): Exception during Connect(): {ex.Message}\n{ex.StackTrace}"); + } + }, + cancellationTokenSource.Token, + TaskCreationOptions.LongRunning, + TaskScheduler.Default); - _historicalApiClient = new(apiKey); - _initialized = true; + _historicalApiClient = new(apiKey); + _initialized = true; - Log.Debug("DataBentoProvider.Initialize(): Initialization complete"); - } + Log.Debug("DataBentoProvider.Initialize(): Initialization complete"); + } - /// - /// Logic to unsubscribe from the specified symbols - /// - public bool UnsubscribeLogic(IEnumerable symbols, TickType tickType) + /// + /// Logic to unsubscribe from the specified symbols + /// + public bool UnsubscribeLogic(IEnumerable symbols, TickType tickType) + { + foreach (var symbol in symbols) { - foreach (var symbol in symbols) + Log.Debug($"DataBentoProvider.UnsubscribeImpl(): Processing symbol {symbol}"); + if (_client?.IsConnected != true) { - Log.Debug($"DataBentoProvider.UnsubscribeImpl(): Processing symbol {symbol}"); - if (_client?.IsConnected != true) - { - throw new InvalidOperationException($"DataBentoProvider.UnsubscribeImpl(): Client is not connected. Cannot unsubscribe from {symbol}"); - } - - _client.Unsubscribe(symbol); + throw new InvalidOperationException($"DataBentoProvider.UnsubscribeImpl(): Client is not connected. Cannot unsubscribe from {symbol}"); } - return true; + _client.Unsubscribe(symbol); } - /// - /// Logic to subscribe to the specified symbols - /// - public bool SubscriptionLogic(IEnumerable symbols, TickType tickType) + return true; + } + + /// + /// Logic to subscribe to the specified symbols + /// + public bool SubscriptionLogic(IEnumerable symbols, TickType tickType) + { + if (_client?.IsConnected != true) { - if (_client?.IsConnected != true) + Log.Error("DataBentoProvider.SubscriptionLogic(): Client is not connected. Cannot subscribe to symbols"); + return false; + } + + foreach (var symbol in symbols) + { + if (!CanSubscribe(symbol)) { - Log.Error("DataBentoProvider.SubscriptionLogic(): Client is not connected. Cannot subscribe to symbols"); + Log.Error($"DataBentoProvider.SubscriptionLogic(): Unsupported subscription: {symbol}"); return false; } - foreach (var symbol in symbols) - { - if (!CanSubscribe(symbol)) - { - Log.Error($"DataBentoProvider.SubscriptionLogic(): Unsupported subscription: {symbol}"); - return false; - } + _client.Subscribe(symbol, tickType); + } - _client.Subscribe(symbol, tickType); - } + return true; + } - return true; - } + /// + /// Checks if this Data provider supports the specified symbol + /// + /// The symbol + /// returns true if Data Provider supports the specified symbol; otherwise false + private static bool CanSubscribe(Symbol symbol) + { + return !symbol.Value.Contains("universe", StringComparison.InvariantCultureIgnoreCase) && + !symbol.IsCanonical() && + symbol.SecurityType == SecurityType.Future; + } - /// - /// Checks if this Data provider supports the specified symbol - /// - /// The symbol - /// returns true if Data Provider supports the specified symbol; otherwise false - private static bool CanSubscribe(Symbol symbol) + /// + /// Subscribe to the specified configuration + /// + /// defines the parameters to subscribe to a data feed + /// handler to be fired on new data available + /// The new enumerator for this subscription request + public IEnumerator? Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler) + { + if (!IsSupported(dataConfig.SecurityType, dataConfig.Type, dataConfig.TickType, dataConfig.Resolution)) { - return !symbol.Value.Contains("universe", StringComparison.InvariantCultureIgnoreCase) && - !symbol.IsCanonical() && - symbol.SecurityType == SecurityType.Future; + return null; } - /// - /// Subscribe to the specified configuration - /// - /// defines the parameters to subscribe to a data feed - /// handler to be fired on new data available - /// The new enumerator for this subscription request - public IEnumerator? Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler) + lock (_sessionLock) { - if (!IsSupported(dataConfig.SecurityType, dataConfig.Type, dataConfig.TickType, dataConfig.Resolution)) + if (!_sessionStarted) { - return null; + Log.Debug("DataBentoProvider.SubscriptionLogic(): Starting session"); + _sessionStarted = _client.StartSession(); } + } - lock (_sessionLock) - { - if (!_sessionStarted) - { - Log.Debug("DataBentoProvider.SubscriptionLogic(): Starting session"); - _sessionStarted = _client.StartSession(); - } - } + var enumerator = _dataAggregator.Add(dataConfig, newDataAvailableHandler); + _subscriptionManager.Subscribe(dataConfig); - var enumerator = _dataAggregator.Add(dataConfig, newDataAvailableHandler); - _subscriptionManager.Subscribe(dataConfig); + return enumerator; + } - return enumerator; - } + /// + /// Removes the specified configuration + /// + /// Subscription config to be removed + public void Unsubscribe(SubscriptionDataConfig dataConfig) + { + Log.Debug($"DataBentoProvider.Unsubscribe(): Received unsubscription request for {dataConfig.Symbol}, Resolution={dataConfig.Resolution}, TickType={dataConfig.TickType}"); + _subscriptionManager.Unsubscribe(dataConfig); + _dataAggregator.Remove(dataConfig); + } - /// - /// Removes the specified configuration - /// - /// Subscription config to be removed - public void Unsubscribe(SubscriptionDataConfig dataConfig) + /// + /// Sets the job we're subscribing for + /// + /// Job we're subscribing for + public void SetJob(LiveNodePacket job) + { + if (_initialized) { - Log.Debug($"DataBentoProvider.Unsubscribe(): Received unsubscription request for {dataConfig.Symbol}, Resolution={dataConfig.Resolution}, TickType={dataConfig.TickType}"); - _subscriptionManager.Unsubscribe(dataConfig); - _dataAggregator.Remove(dataConfig); + return; } + } + + /// + /// Dispose of unmanaged resources. + /// + public void Dispose() + { + _dataAggregator?.DisposeSafely(); + _subscriptionManager?.DisposeSafely(); + _client?.DisposeSafely(); + } - /// - /// Sets the job we're subscribing for - /// - /// Job we're subscribing for - public void SetJob(LiveNodePacket job) + /// + /// Determines if the specified subscription is supported + /// + private bool IsSupported(SecurityType securityType, Type dataType, TickType tickType, Resolution resolution) + { + // Check supported data types + if (dataType != typeof(TradeBar) && + dataType != typeof(QuoteBar) && + dataType != typeof(Tick) && + dataType != typeof(OpenInterest)) { - if (_initialized) - { - return; - } + throw new NotSupportedException($"Unsupported data type: {dataType}"); } - /// - /// Dispose of unmanaged resources. - /// - public void Dispose() + // Warn about potential limitations for tick data + // I'm mimicing polygon implementation with this + if (!_potentialUnsupportedResolutionMessageLogged) { - _dataAggregator?.DisposeSafely(); - _subscriptionManager?.DisposeSafely(); - _client?.DisposeSafely(); + _potentialUnsupportedResolutionMessageLogged = true; + Log.Trace("DataBentoDataProvider.IsSupported(): " + + $"Subscription for {securityType}-{dataType}-{tickType}-{resolution} will be attempted. " + + $"An Advanced DataBento subscription plan is required to stream tick data."); } - /// - /// Determines if the specified subscription is supported - /// - private bool IsSupported(SecurityType securityType, Type dataType, TickType tickType, Resolution resolution) + return true; + } + + /// + /// Converts the given UTC time into the symbol security exchange time zone + /// + private DateTime GetTickTime(Symbol symbol, DateTime utcTime) + { + DateTimeZone exchangeTimeZone; + lock (_symbolExchangeTimeZones) { - // Check supported data types - if (dataType != typeof(TradeBar) && - dataType != typeof(QuoteBar) && - dataType != typeof(Tick) && - dataType != typeof(OpenInterest)) + if (!_symbolExchangeTimeZones.TryGetValue(symbol, out exchangeTimeZone)) { - throw new NotSupportedException($"Unsupported data type: {dataType}"); - } + // read the exchange time zone from market-hours-database + if (_marketHoursDatabase.TryGetEntry(symbol.ID.Market, symbol, symbol.SecurityType, out var entry)) + { + exchangeTimeZone = entry.ExchangeHours.TimeZone; + } + // If there is no entry for the given Symbol, default to New York + else + { + exchangeTimeZone = TimeZones.NewYork; + } - // Warn about potential limitations for tick data - // I'm mimicing polygon implementation with this - if (!_potentialUnsupportedResolutionMessageLogged) - { - _potentialUnsupportedResolutionMessageLogged = true; - Log.Trace("DataBentoDataProvider.IsSupported(): " + - $"Subscription for {securityType}-{dataType}-{tickType}-{resolution} will be attempted. " + - $"An Advanced DataBento subscription plan is required to stream tick data."); + _symbolExchangeTimeZones[symbol] = exchangeTimeZone; } - - return true; } - /// - /// Converts the given UTC time into the symbol security exchange time zone - /// - private DateTime GetTickTime(Symbol symbol, DateTime utcTime) + return utcTime.ConvertFromUtc(exchangeTimeZone); + } + + /// + /// Handles data received from the live client + /// + private void OnDataReceived(object _, BaseData data) + { + try { - DateTimeZone exchangeTimeZone; - lock (_symbolExchangeTimeZones) + switch (data) { - if (!_symbolExchangeTimeZones.TryGetValue(symbol, out exchangeTimeZone)) - { - // read the exchange time zone from market-hours-database - if (_marketHoursDatabase.TryGetEntry(symbol.ID.Market, symbol, symbol.SecurityType, out var entry)) + case Tick tick: + tick.Time = GetTickTime(tick.Symbol, tick.Time); + lock (_dataAggregator) { - exchangeTimeZone = entry.ExchangeHours.TimeZone; + _dataAggregator.Update(tick); } - // If there is no entry for the given Symbol, default to New York - else + // Log.Trace($"DataBentoProvider.OnDataReceived(): Updated tick - Symbol: {tick.Symbol}, " + + // $"TickType: {tick.TickType}, Price: {tick.Value}, Quantity: {tick.Quantity}"); + break; + + case TradeBar tradeBar: + tradeBar.Time = GetTickTime(tradeBar.Symbol, tradeBar.Time); + tradeBar.EndTime = GetTickTime(tradeBar.Symbol, tradeBar.EndTime); + lock (_dataAggregator) { - exchangeTimeZone = TimeZones.NewYork; + _dataAggregator.Update(tradeBar); } + // Log.Trace($"DataBentoProvider.OnDataReceived(): Updated TradeBar - Symbol: {tradeBar.Symbol}, " + + // $"O:{tradeBar.Open} H:{tradeBar.High} L:{tradeBar.Low} C:{tradeBar.Close} V:{tradeBar.Volume}"); + break; - _symbolExchangeTimeZones[symbol] = exchangeTimeZone; - } + default: + data.Time = GetTickTime(data.Symbol, data.Time); + lock (_dataAggregator) + { + _dataAggregator.Update(data); + } + break; } - - return utcTime.ConvertFromUtc(exchangeTimeZone); } - - /// - /// Handles data received from the live client - /// - private void OnDataReceived(object _, BaseData data) + catch (Exception ex) { - try - { - switch (data) - { - case Tick tick: - tick.Time = GetTickTime(tick.Symbol, tick.Time); - lock (_dataAggregator) - { - _dataAggregator.Update(tick); - } - // Log.Trace($"DataBentoProvider.OnDataReceived(): Updated tick - Symbol: {tick.Symbol}, " + - // $"TickType: {tick.TickType}, Price: {tick.Value}, Quantity: {tick.Quantity}"); - break; - - case TradeBar tradeBar: - tradeBar.Time = GetTickTime(tradeBar.Symbol, tradeBar.Time); - tradeBar.EndTime = GetTickTime(tradeBar.Symbol, tradeBar.EndTime); - lock (_dataAggregator) - { - _dataAggregator.Update(tradeBar); - } - // Log.Trace($"DataBentoProvider.OnDataReceived(): Updated TradeBar - Symbol: {tradeBar.Symbol}, " + - // $"O:{tradeBar.Open} H:{tradeBar.High} L:{tradeBar.Low} C:{tradeBar.Close} V:{tradeBar.Volume}"); - break; - - default: - data.Time = GetTickTime(data.Symbol, data.Time); - lock (_dataAggregator) - { - _dataAggregator.Update(data); - } - break; - } - } - catch (Exception ex) - { - Log.Error($"DataBentoProvider.OnDataReceived(): Error updating data aggregator: {ex.Message}\n{ex.StackTrace}"); - } + Log.Error($"DataBentoProvider.OnDataReceived(): Error updating data aggregator: {ex.Message}\n{ex.StackTrace}"); } } } diff --git a/QuantConnect.DataBento/DataBentoHistoryProivder.cs b/QuantConnect.DataBento/DataBentoHistoryProivder.cs index b92744e..2485ff5 100644 --- a/QuantConnect.DataBento/DataBentoHistoryProivder.cs +++ b/QuantConnect.DataBento/DataBentoHistoryProivder.cs @@ -24,226 +24,225 @@ using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.Lean.Engine.HistoricalData; -namespace QuantConnect.Lean.DataSource.DataBento +namespace QuantConnect.Lean.DataSource.DataBento; + +/// +/// Implements a history provider for DataBento historical data. +/// Uses consolidators to produce the requested resolution when necessary. +/// +public partial class DataBentoProvider : SynchronizingHistoryProvider { + private static int _dataPointCount; + /// - /// Implements a history provider for DataBento historical data. - /// Uses consolidators to produce the requested resolution when necessary. + /// Indicates whether a error for an invalid start time has been fired, where the start time is greater than or equal to the end time in UTC. /// - public partial class DataBentoProvider : SynchronizingHistoryProvider - { - private static int _dataPointCount; - - /// - /// Indicates whether a error for an invalid start time has been fired, where the start time is greater than or equal to the end time in UTC. - /// - private volatile bool _invalidStartTimeErrorFired; - - /// - /// Indicates whether the warning for invalid has been fired. - /// - private volatile bool _invalidSecurityTypeWarningFired; - - /// - /// Gets the total number of data points emitted by this history provider - /// - public override int DataPointCount => _dataPointCount; - - /// - /// Initializes this history provider to work for the specified job - /// - /// The initialization parameters - public override void Initialize(HistoryProviderInitializeParameters parameters) - { - } + private volatile bool _invalidStartTimeErrorFired; - /// - /// Gets the history for the requested securities - /// - /// The historical data requests - /// The time zone used when time stamping the slice instances - /// An enumerable of the slices of data covering the span specified in each request - public override IEnumerable? GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) - { - var subscriptions = new List(); - foreach (var request in requests) - { - var history = request.SplitHistoryRequestWithUpdatedMappedSymbol(_mapFileProvider).SelectMany(x => GetHistory(x) ?? []); + /// + /// Indicates whether the warning for invalid has been fired. + /// + private volatile bool _invalidSecurityTypeWarningFired; - var subscription = CreateSubscription(request, history); - if (!subscription.MoveNext()) - { - continue; - } + /// + /// Gets the total number of data points emitted by this history provider + /// + public override int DataPointCount => _dataPointCount; - subscriptions.Add(subscription); - } + /// + /// Initializes this history provider to work for the specified job + /// + /// The initialization parameters + public override void Initialize(HistoryProviderInitializeParameters parameters) + { + } + + /// + /// Gets the history for the requested securities + /// + /// The historical data requests + /// The time zone used when time stamping the slice instances + /// An enumerable of the slices of data covering the span specified in each request + public override IEnumerable? GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) + { + var subscriptions = new List(); + foreach (var request in requests) + { + var history = request.SplitHistoryRequestWithUpdatedMappedSymbol(_mapFileProvider).SelectMany(x => GetHistory(x) ?? []); - if (subscriptions.Count == 0) + var subscription = CreateSubscription(request, history); + if (!subscription.MoveNext()) { - return null; + continue; } - return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone); + + subscriptions.Add(subscription); } - /// - /// Gets the history for the requested security - /// - /// The historical data request - /// An enumerable of BaseData points - public IEnumerable? GetHistory(HistoryRequest historyRequest) + if (subscriptions.Count == 0) { - if (!CanSubscribe(historyRequest.Symbol)) - { - if (!_invalidSecurityTypeWarningFired) - { - _invalidSecurityTypeWarningFired = true; - LogTrace(nameof(GetHistory), $"Unsupported SecurityType '{historyRequest.Symbol.SecurityType}' for symbol '{historyRequest.Symbol}'."); - } - return null; - } + return null; + } + return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone); + } - if (historyRequest.EndTimeUtc < historyRequest.StartTimeUtc) + /// + /// Gets the history for the requested security + /// + /// The historical data request + /// An enumerable of BaseData points + public IEnumerable? GetHistory(HistoryRequest historyRequest) + { + if (!CanSubscribe(historyRequest.Symbol)) + { + if (!_invalidSecurityTypeWarningFired) { - if (!_invalidStartTimeErrorFired) - { - _invalidStartTimeErrorFired = true; - Log.Error($"{nameof(DataBentoProvider)}.{nameof(GetHistory)}: Invalid date range: the start date must be earlier than the end date."); - } - return null; + _invalidSecurityTypeWarningFired = true; + LogTrace(nameof(GetHistory), $"Unsupported SecurityType '{historyRequest.Symbol.SecurityType}' for symbol '{historyRequest.Symbol}'."); } + return null; + } - var history = default(IEnumerable); - var brokerageSymbol = _symbolMapper.GetBrokerageSymbol(historyRequest.Symbol); - switch (historyRequest.TickType) + if (historyRequest.EndTimeUtc < historyRequest.StartTimeUtc) + { + if (!_invalidStartTimeErrorFired) { - case TickType.Trade when historyRequest.Resolution == Resolution.Tick: - history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol); - break; - case TickType.Trade: - history = GetAggregatedTradeBars(historyRequest, brokerageSymbol); - break; - case TickType.Quote: - history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol); - break; - case TickType.OpenInterest: - history = GetOpenInterestBars(historyRequest, brokerageSymbol); - break; - default: - throw new ArgumentException(""); + _invalidStartTimeErrorFired = true; + Log.Error($"{nameof(DataBentoProvider)}.{nameof(GetHistory)}: Invalid date range: the start date must be earlier than the end date."); } + return null; + } - if (history == null) - { - return null; - } + var history = default(IEnumerable); + var brokerageSymbol = _symbolMapper.GetBrokerageSymbol(historyRequest.Symbol); + switch (historyRequest.TickType) + { + case TickType.Trade when historyRequest.Resolution == Resolution.Tick: + history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol); + break; + case TickType.Trade: + history = GetAggregatedTradeBars(historyRequest, brokerageSymbol); + break; + case TickType.Quote: + history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol); + break; + case TickType.OpenInterest: + history = GetOpenInterestBars(historyRequest, brokerageSymbol); + break; + default: + throw new ArgumentException(""); + } - return FilterHistory(history, historyRequest, historyRequest.StartTimeLocal, historyRequest.EndTimeLocal); + if (history == null) + { + return null; } - private static IEnumerable FilterHistory(IEnumerable history, HistoryRequest request, DateTime startTimeLocal, DateTime endTimeLocal) + return FilterHistory(history, historyRequest, historyRequest.StartTimeLocal, historyRequest.EndTimeLocal); + } + + private static IEnumerable FilterHistory(IEnumerable history, HistoryRequest request, DateTime startTimeLocal, DateTime endTimeLocal) + { + // cleaning the data before returning it back to user + foreach (var bar in history) { - // cleaning the data before returning it back to user - foreach (var bar in history) + if (bar.Time >= startTimeLocal && bar.EndTime <= endTimeLocal) { - if (bar.Time >= startTimeLocal && bar.EndTime <= endTimeLocal) + if (request.ExchangeHours.IsOpen(bar.Time, bar.EndTime, request.IncludeExtendedMarketHours)) { - if (request.ExchangeHours.IsOpen(bar.Time, bar.EndTime, request.IncludeExtendedMarketHours)) - { - Interlocked.Increment(ref _dataPointCount); - yield return bar; - } + Interlocked.Increment(ref _dataPointCount); + yield return bar; } } } + } - private IEnumerable GetOpenInterestBars(HistoryRequest request, string brokerageSymbol) - { - foreach (var oi in _historicalApiClient.GetOpenInterest(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) - { - yield return new OpenInterest(oi.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, oi.Quantity); - } + private IEnumerable GetOpenInterestBars(HistoryRequest request, string brokerageSymbol) + { + foreach (var oi in _historicalApiClient.GetOpenInterest(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) + { + yield return new OpenInterest(oi.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, oi.Quantity); } + } - private IEnumerable GetAggregatedTradeBars(HistoryRequest request, string brokerageSymbol) + private IEnumerable GetAggregatedTradeBars(HistoryRequest request, string brokerageSymbol) + { + var period = request.Resolution.ToTimeSpan(); + foreach (var b in _historicalApiClient.GetHistoricalOhlcvBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc, request.Resolution, request.TickType)) { - var period = request.Resolution.ToTimeSpan(); - foreach (var b in _historicalApiClient.GetHistoricalOhlcvBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc, request.Resolution, request.TickType)) - { - yield return new TradeBar(b.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, b.Open, b.High, b.Low, b.Close, b.Volume, period); - } + yield return new TradeBar(b.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, b.Open, b.High, b.Low, b.Close, b.Volume, period); } + } - private IEnumerable? GetHistoryThroughDataConsolidator(HistoryRequest request, string brokerageSymbol) - { - IDataConsolidator consolidator; - IEnumerable history; + private IEnumerable? GetHistoryThroughDataConsolidator(HistoryRequest request, string brokerageSymbol) + { + IDataConsolidator consolidator; + IEnumerable history; - if (request.TickType == TickType.Trade) - { - consolidator = request.Resolution != Resolution.Tick - ? new TickConsolidator(request.Resolution.ToTimeSpan()) - : FilteredIdentityDataConsolidator.ForTickType(request.TickType); - history = GetTrades(request, brokerageSymbol); - } - else - { - consolidator = request.Resolution != Resolution.Tick - ? new TickQuoteBarConsolidator(request.Resolution.ToTimeSpan()) - : FilteredIdentityDataConsolidator.ForTickType(request.TickType); - history = GetQuotes(request, brokerageSymbol); - } + if (request.TickType == TickType.Trade) + { + consolidator = request.Resolution != Resolution.Tick + ? new TickConsolidator(request.Resolution.ToTimeSpan()) + : FilteredIdentityDataConsolidator.ForTickType(request.TickType); + history = GetTrades(request, brokerageSymbol); + } + else + { + consolidator = request.Resolution != Resolution.Tick + ? new TickQuoteBarConsolidator(request.Resolution.ToTimeSpan()) + : FilteredIdentityDataConsolidator.ForTickType(request.TickType); + history = GetQuotes(request, brokerageSymbol); + } - BaseData? consolidatedData = null; - DataConsolidatedHandler onDataConsolidated = (s, e) => - { - consolidatedData = (BaseData)e; - }; - consolidator.DataConsolidated += onDataConsolidated; + BaseData? consolidatedData = null; + DataConsolidatedHandler onDataConsolidated = (s, e) => + { + consolidatedData = (BaseData)e; + }; + consolidator.DataConsolidated += onDataConsolidated; - foreach (var data in history) + foreach (var data in history) + { + consolidator.Update(data); + if (consolidatedData != null) { - consolidator.Update(data); - if (consolidatedData != null) - { - yield return consolidatedData; - consolidatedData = null; - } + yield return consolidatedData; + consolidatedData = null; } - - consolidator.DataConsolidated -= onDataConsolidated; - consolidator.DisposeSafely(); } - /// - /// Gets the trade ticks that will potentially be aggregated for the specified history request - /// - private IEnumerable GetTrades(HistoryRequest request, string brokerageSymbol) + consolidator.DataConsolidated -= onDataConsolidated; + consolidator.DisposeSafely(); + } + + /// + /// Gets the trade ticks that will potentially be aggregated for the specified history request + /// + private IEnumerable GetTrades(HistoryRequest request, string brokerageSymbol) + { + foreach (var t in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) { - foreach (var t in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) - { - yield return new Tick(t.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, "", "", t.Size, t.Price); - } + yield return new Tick(t.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, "", "", t.Size, t.Price); } + } - /// - /// Gets the quote ticks that will potentially be aggregated for the specified history request - /// - private IEnumerable GetQuotes(HistoryRequest request, string brokerageSymbol) + /// + /// Gets the quote ticks that will potentially be aggregated for the specified history request + /// + private IEnumerable GetQuotes(HistoryRequest request, string brokerageSymbol) + { + foreach (var quoteBar in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) { - foreach (var quoteBar in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) + var time = quoteBar.Header.UtcTime.ConvertFromUtc(request.DataTimeZone); + foreach (var level in quoteBar.Levels) { - var time = quoteBar.Header.UtcTime.ConvertFromUtc(request.DataTimeZone); - foreach (var level in quoteBar.Levels) - { - yield return new Tick(time, request.Symbol, level.BidSz, level.BidPx, level.AskSz, level.AskPx); - } + yield return new Tick(time, request.Symbol, level.BidSz, level.BidPx, level.AskSz, level.AskPx); } } + } - private static void LogTrace(string methodName, string message) - { - Log.Trace($"{nameof(DataBentoProvider)}.{methodName}: {message}"); - } + private static void LogTrace(string methodName, string message) + { + Log.Trace($"{nameof(DataBentoProvider)}.{methodName}: {message}"); } } diff --git a/QuantConnect.DataBento/DataBentoRawLiveClient.cs b/QuantConnect.DataBento/DataBentoRawLiveClient.cs index 8d8277c..9c65544 100644 --- a/QuantConnect.DataBento/DataBentoRawLiveClient.cs +++ b/QuantConnect.DataBento/DataBentoRawLiveClient.cs @@ -19,678 +19,676 @@ using System.Security.Cryptography; using System.Collections.Concurrent; using System.Text.Json; -using System.Threading.Tasks; using QuantConnect.Data; using QuantConnect.Data.Market; using QuantConnect.Logging; -namespace QuantConnect.Lean.DataSource.DataBento +namespace QuantConnect.Lean.DataSource.DataBento; + +/// +/// DataBento Raw TCP client for live streaming data +/// +public class DataBentoRawLiveClient : IDisposable { /// - /// DataBento Raw TCP client for live streaming data + /// The DataBento API key for authentication + /// + private readonly string _apiKey; + /// + /// The DataBento live gateway address to receive data from + /// + private const string _gateway = "glbx-mdp3.lsg.databento.com:13000"; + /// + /// The dataset to subscribe to + /// + private readonly string _dataset; + private readonly TcpClient? _tcpClient; + private readonly string _host; + private readonly int _port; + private NetworkStream? _stream; + private StreamReader _reader; + private StreamWriter _writer; + private readonly CancellationTokenSource _cancellationTokenSource; + private readonly ConcurrentDictionary _subscriptions; + private readonly object _connectionLock = new object(); + private bool _isConnected; + private bool _disposed; + private const decimal PriceScaleFactor = 1e-9m; + private readonly ConcurrentDictionary _instrumentIdToSymbol = new ConcurrentDictionary(); + private readonly DataBentoSymbolMapper _symbolMapper; + + /// + /// Event fired when new data is received + /// + public event EventHandler DataReceived; + + /// + /// Event fired when connection status changes /// - public class DataBentoRawLiveClient : IDisposable + public event EventHandler ConnectionStatusChanged; + + /// + /// Gets whether the client is currently connected + /// + public bool IsConnected => _isConnected && _tcpClient?.Connected == true; + + /// + /// Initializes a new instance of the DataBentoRawLiveClient + /// The DataBento API key. + /// + public DataBentoRawLiveClient(string apiKey, string dataset = "GLBX.MDP3") { - /// - /// The DataBento API key for authentication - /// - private readonly string _apiKey; - /// - /// The DataBento live gateway address to receive data from - /// - private const string _gateway = "glbx-mdp3.lsg.databento.com:13000"; - /// - /// The dataset to subscribe to - /// - private readonly string _dataset; - private readonly TcpClient? _tcpClient; - private readonly string _host; - private readonly int _port; - private NetworkStream? _stream; - private StreamReader _reader; - private StreamWriter _writer; - private readonly CancellationTokenSource _cancellationTokenSource; - private readonly ConcurrentDictionary _subscriptions; - private readonly object _connectionLock = new object(); - private bool _isConnected; - private bool _disposed; - private const decimal PriceScaleFactor = 1e-9m; - private readonly ConcurrentDictionary _instrumentIdToSymbol = new ConcurrentDictionary(); - private readonly DataBentoSymbolMapper _symbolMapper; - - /// - /// Event fired when new data is received - /// - public event EventHandler DataReceived; - - /// - /// Event fired when connection status changes - /// - public event EventHandler ConnectionStatusChanged; - - /// - /// Gets whether the client is currently connected - /// - public bool IsConnected => _isConnected && _tcpClient?.Connected == true; - - /// - /// Initializes a new instance of the DataBentoRawLiveClient - /// The DataBento API key. - /// - public DataBentoRawLiveClient(string apiKey, string dataset = "GLBX.MDP3") + _apiKey = apiKey ?? throw new ArgumentNullException(nameof(apiKey)); + _dataset = dataset; + _tcpClient = new TcpClient(); + _subscriptions = new ConcurrentDictionary(); + _cancellationTokenSource = new CancellationTokenSource(); + _symbolMapper = new DataBentoSymbolMapper(); + + var parts = _gateway.Split(':'); + _host = parts[0]; + _port = parts.Length > 1 ? int.Parse(parts[1]) : 13000; + } + + /// + /// Connects to the DataBento live gateway + /// + public bool Connect() + { + Log.Trace("DataBentoRawLiveClient.Connect(): Connecting to DataBento live gateway"); + if (_isConnected) { - _apiKey = apiKey ?? throw new ArgumentNullException(nameof(apiKey)); - _dataset = dataset; - _tcpClient = new TcpClient(); - _subscriptions = new ConcurrentDictionary(); - _cancellationTokenSource = new CancellationTokenSource(); - _symbolMapper = new DataBentoSymbolMapper(); - - var parts = _gateway.Split(':'); - _host = parts[0]; - _port = parts.Length > 1 ? int.Parse(parts[1]) : 13000; + return _isConnected; } - /// - /// Connects to the DataBento live gateway - /// - public bool Connect() + try { - Log.Trace("DataBentoRawLiveClient.Connect(): Connecting to DataBento live gateway"); - if (_isConnected) - { - return _isConnected; - } + _tcpClient.Connect(_host, _port); + _stream = _tcpClient.GetStream(); + _reader = new StreamReader(_stream, Encoding.ASCII); + _writer = new StreamWriter(_stream, Encoding.ASCII) { AutoFlush = true }; - try + // Perform authentication handshake + if (Authenticate()) { - _tcpClient.Connect(_host, _port); - _stream = _tcpClient.GetStream(); - _reader = new StreamReader(_stream, Encoding.ASCII); - _writer = new StreamWriter(_stream, Encoding.ASCII) { AutoFlush = true }; + _isConnected = true; + ConnectionStatusChanged?.Invoke(this, true); - // Perform authentication handshake - if (Authenticate()) - { - _isConnected = true; - ConnectionStatusChanged?.Invoke(this, true); - - // Start message processing - Task.Run(ProcessMessages, _cancellationTokenSource.Token); + // Start message processing + Task.Run(ProcessMessages, _cancellationTokenSource.Token); - Log.Trace("DataBentoRawLiveClient.Connect(): Connected and authenticated to DataBento live gateway"); - return true; - } - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.Connect(): Failed to connect: {ex.Message}"); - Disconnect(); + Log.Trace("DataBentoRawLiveClient.Connect(): Connected and authenticated to DataBento live gateway"); + return true; } - - return false; } - - /// - /// Authenticates with the DataBento gateway using CRAM-SHA256 - /// - private bool Authenticate() + catch (Exception ex) { - try - { - // Read greeting and challenge - var versionLine = _reader.ReadLine(); - var cramLine = _reader.ReadLine(); + Log.Error($"DataBentoRawLiveClient.Connect(): Failed to connect: {ex.Message}"); + Disconnect(); + } - if (string.IsNullOrEmpty(versionLine) || string.IsNullOrEmpty(cramLine)) - { - Log.Error("DataBentoRawLiveClient.Authenticate(): Failed to receive greeting or challenge"); - return false; - } + return false; + } - // Parse challenge - var cramParts = cramLine.Split('='); - if (cramParts.Length != 2 || cramParts[0] != "cram") - { - Log.Error("DataBentoRawLiveClient.Authenticate(): Invalid challenge format"); - return false; - } - var cram = cramParts[1].Trim(); + /// + /// Authenticates with the DataBento gateway using CRAM-SHA256 + /// + private bool Authenticate() + { + try + { + // Read greeting and challenge + var versionLine = _reader.ReadLine(); + var cramLine = _reader.ReadLine(); - // Auth - _writer.WriteLine($"auth={GetAuthStringFromCram(cram)}|dataset={_dataset}|encoding=json|ts_out=0"); - var authResp = _reader.ReadLine(); - if (!authResp.Contains("success=1")) - { - Log.Error($"DataBentoRawLiveClient.Authenticate(): Authentication failed: {authResp}"); - return false; - } + if (string.IsNullOrEmpty(versionLine) || string.IsNullOrEmpty(cramLine)) + { + Log.Error("DataBentoRawLiveClient.Authenticate(): Failed to receive greeting or challenge"); + return false; + } - Log.Trace("DataBentoRawLiveClient.Authenticate(): Authentication successful"); - return true; + // Parse challenge + var cramParts = cramLine.Split('='); + if (cramParts.Length != 2 || cramParts[0] != "cram") + { + Log.Error("DataBentoRawLiveClient.Authenticate(): Invalid challenge format"); + return false; } - catch (Exception ex) + var cram = cramParts[1].Trim(); + + // Auth + _writer.WriteLine($"auth={GetAuthStringFromCram(cram)}|dataset={_dataset}|encoding=json|ts_out=0"); + var authResp = _reader.ReadLine(); + if (!authResp.Contains("success=1")) { - Log.Error($"DataBentoRawLiveClient.Authenticate(): Authentication failed: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient.Authenticate(): Authentication failed: {authResp}"); return false; } - } - /// - /// Handles the DataBento authentication string from a CRAM challenge - /// - /// The CRAM challenge string - /// The auth string to send to the server - private string GetAuthStringFromCram(string cram) + Log.Trace("DataBentoRawLiveClient.Authenticate(): Authentication successful"); + return true; + } + catch (Exception ex) { - if (string.IsNullOrWhiteSpace(cram)) - throw new ArgumentException("CRAM challenge cannot be null or empty", nameof(cram)); + Log.Error($"DataBentoRawLiveClient.Authenticate(): Authentication failed: {ex.Message}"); + return false; + } + } + + /// + /// Handles the DataBento authentication string from a CRAM challenge + /// + /// The CRAM challenge string + /// The auth string to send to the server + private string GetAuthStringFromCram(string cram) + { + if (string.IsNullOrWhiteSpace(cram)) + throw new ArgumentException("CRAM challenge cannot be null or empty", nameof(cram)); - string concat = $"{cram}|{_apiKey}"; - string hashHex = ComputeSHA256(concat); - string bucketId = _apiKey.Substring(_apiKey.Length - 5); + string concat = $"{cram}|{_apiKey}"; + string hashHex = ComputeSHA256(concat); + string bucketId = _apiKey.Substring(_apiKey.Length - 5); - return $"{hashHex}-{bucketId}"; - } + return $"{hashHex}-{bucketId}"; + } - /// - /// Subscribes to live data for a symbol - /// - public bool Subscribe(Symbol symbol, TickType tickType) + /// + /// Subscribes to live data for a symbol + /// + public bool Subscribe(Symbol symbol, TickType tickType) + { + if (!IsConnected) { - if (!IsConnected) - { - Log.Error("DataBentoRawLiveClient.Subscribe(): Not connected to gateway"); - return false; - } + Log.Error("DataBentoRawLiveClient.Subscribe(): Not connected to gateway"); + return false; + } - try - { - // Get the databento symbol form LEAN symbol - var databentoSymbol = _symbolMapper.GetBrokerageSymbol(symbol); - var schema = "mbp-1"; - var resolution = Resolution.Tick; + try + { + // Get the databento symbol form LEAN symbol + var databentoSymbol = _symbolMapper.GetBrokerageSymbol(symbol); + var schema = "mbp-1"; + var resolution = Resolution.Tick; - // subscribe - var subscribeMessage = $"schema={schema}|stype_in=parent|symbols={databentoSymbol}"; - Log.Debug($"DataBentoRawLiveClient.Subscribe(): Subscribing with message: {subscribeMessage}"); + // subscribe + var subscribeMessage = $"schema={schema}|stype_in=parent|symbols={databentoSymbol}"; + Log.Debug($"DataBentoRawLiveClient.Subscribe(): Subscribing with message: {subscribeMessage}"); - // Send subscribe message - _writer.WriteLine(subscribeMessage); + // Send subscribe message + _writer.WriteLine(subscribeMessage); - // Store subscription - _subscriptions.TryAdd(symbol, (resolution, tickType)); - Log.Debug($"DataBentoRawLiveClient.Subscribe(): Subscribed to {symbol} ({databentoSymbol}) at {resolution} resolution for {tickType}"); + // Store subscription + _subscriptions.TryAdd(symbol, (resolution, tickType)); + Log.Debug($"DataBentoRawLiveClient.Subscribe(): Subscribed to {symbol} ({databentoSymbol}) at {resolution} resolution for {tickType}"); - return true; - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.Subscribe(): Failed to subscribe to {symbol}: {ex.Message}"); - return false; - } + return true; + } + catch (Exception ex) + { + Log.Error($"DataBentoRawLiveClient.Subscribe(): Failed to subscribe to {symbol}: {ex.Message}"); + return false; } + } - /// - /// Starts the session to begin receiving data - /// - public bool StartSession() + /// + /// Starts the session to begin receiving data + /// + public bool StartSession() + { + if (!IsConnected) { - if (!IsConnected) - { - Log.Error("DataBentoRawLiveClient.StartSession(): Not connected"); - return false; - } + Log.Error("DataBentoRawLiveClient.StartSession(): Not connected"); + return false; + } - try - { - Log.Trace("DataBentoRawLiveClient.StartSession(): Starting session"); - _writer.WriteLine("start_session=1"); - return true; - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.StartSession(): Failed to start session: {ex.Message}"); - return false; - } + try + { + Log.Trace("DataBentoRawLiveClient.StartSession(): Starting session"); + _writer.WriteLine("start_session=1"); + return true; } + catch (Exception ex) + { + Log.Error($"DataBentoRawLiveClient.StartSession(): Failed to start session: {ex.Message}"); + return false; + } + } - /// - /// Unsubscribes from live data for a symbol - /// - public bool Unsubscribe(Symbol symbol) + /// + /// Unsubscribes from live data for a symbol + /// + public bool Unsubscribe(Symbol symbol) + { + try { - try - { - if (_subscriptions.TryRemove(symbol, out _)) - { - Log.Debug($"DataBentoRawLiveClient.Unsubscribe(): Unsubscribed from {symbol}"); - } - return true; - } - catch (Exception ex) + if (_subscriptions.TryRemove(symbol, out _)) { - Log.Error($"DataBentoRawLiveClient.Unsubscribe(): Failed to unsubscribe from {symbol}: {ex.Message}"); - return false; + Log.Debug($"DataBentoRawLiveClient.Unsubscribe(): Unsubscribed from {symbol}"); } + return true; } - - /// - /// Processes incoming messages from the DataBento gateway - /// - private void ProcessMessages() + catch (Exception ex) { - Log.Debug("DataBentoRawLiveClient.ProcessMessages(): Starting message processing"); + Log.Error($"DataBentoRawLiveClient.Unsubscribe(): Failed to unsubscribe from {symbol}: {ex.Message}"); + return false; + } + } - try + /// + /// Processes incoming messages from the DataBento gateway + /// + private void ProcessMessages() + { + Log.Debug("DataBentoRawLiveClient.ProcessMessages(): Starting message processing"); + + try + { + while (!_cancellationTokenSource.IsCancellationRequested && IsConnected) { - while (!_cancellationTokenSource.IsCancellationRequested && IsConnected) + var line = _reader.ReadLine(); + if (string.IsNullOrWhiteSpace(line)) { - var line = _reader.ReadLine(); - if (string.IsNullOrWhiteSpace(line)) - { - Log.Trace("DataBentoRawLiveClient.ProcessMessages(): Line is null or empty. Issue receiving data."); - break; - } - - ProcessSingleMessage(line); + Log.Trace("DataBentoRawLiveClient.ProcessMessages(): Line is null or empty. Issue receiving data."); + break; } + + ProcessSingleMessage(line); } - catch (OperationCanceledException) - { - Log.Trace("DataBentoRawLiveClient.ProcessMessages(): Message processing cancelled"); - } - catch (IOException ex) when (ex.InnerException is SocketException) - { - Log.Trace($"DataBentoRawLiveClient.ProcessMessages(): Socket exception: {ex.Message}"); - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.ProcessMessages(): Error processing messages: {ex.Message}\n{ex.StackTrace}"); - } - finally - { - Disconnect(); - } } + catch (OperationCanceledException) + { + Log.Trace("DataBentoRawLiveClient.ProcessMessages(): Message processing cancelled"); + } + catch (IOException ex) when (ex.InnerException is SocketException) + { + Log.Trace($"DataBentoRawLiveClient.ProcessMessages(): Socket exception: {ex.Message}"); + } + catch (Exception ex) + { + Log.Error($"DataBentoRawLiveClient.ProcessMessages(): Error processing messages: {ex.Message}\n{ex.StackTrace}"); + } + finally + { + Disconnect(); + } + } - /// - /// Processes a single message from DataBento - /// - private void ProcessSingleMessage(string message) + /// + /// Processes a single message from DataBento + /// + private void ProcessSingleMessage(string message) + { + try { - try - { - using var document = JsonDocument.Parse(message); - var root = document.RootElement; + using var document = JsonDocument.Parse(message); + var root = document.RootElement; - // Check for error messages - if (root.TryGetProperty("hd", out var headerElement)) + // Check for error messages + if (root.TryGetProperty("hd", out var headerElement)) + { + if (headerElement.TryGetProperty("rtype", out var rtypeElement)) { - if (headerElement.TryGetProperty("rtype", out var rtypeElement)) - { - var rtype = rtypeElement.GetInt32(); + var rtype = rtypeElement.GetInt32(); - switch (rtype) - { - case 23: - // System message - if (root.TryGetProperty("msg", out var msgElement)) - { - Log.Debug($"DataBentoRawLiveClient: System message: {msgElement.GetString()}"); - } - return; - - case 22: - // Symbol mapping message - if (root.TryGetProperty("stype_in_symbol", out var inSymbol) && - root.TryGetProperty("stype_out_symbol", out var outSymbol) && - headerElement.TryGetProperty("instrument_id", out var instId)) + switch (rtype) + { + case 23: + // System message + if (root.TryGetProperty("msg", out var msgElement)) + { + Log.Debug($"DataBentoRawLiveClient: System message: {msgElement.GetString()}"); + } + return; + + case 22: + // Symbol mapping message + if (root.TryGetProperty("stype_in_symbol", out var inSymbol) && + root.TryGetProperty("stype_out_symbol", out var outSymbol) && + headerElement.TryGetProperty("instrument_id", out var instId)) + { + var instrumentId = instId.GetInt64(); + var outSymbolStr = outSymbol.GetString(); + + Log.Debug($"DataBentoRawLiveClient: Symbol mapping: {inSymbol.GetString()} -> {outSymbolStr} (instrument_id: {instrumentId})"); + + if (outSymbolStr != null) { - var instrumentId = instId.GetInt64(); - var outSymbolStr = outSymbol.GetString(); - - Log.Debug($"DataBentoRawLiveClient: Symbol mapping: {inSymbol.GetString()} -> {outSymbolStr} (instrument_id: {instrumentId})"); - - if (outSymbolStr != null) + // Let's find the subscribed symbol to get the market and security type + var inSymbolStr = inSymbol.GetString(); + var subscription = _subscriptions.Keys.FirstOrDefault(s => _symbolMapper.GetBrokerageSymbol(s) == inSymbolStr); + if (subscription != null) { - // Let's find the subscribed symbol to get the market and security type - var inSymbolStr = inSymbol.GetString(); - var subscription = _subscriptions.Keys.FirstOrDefault(s => _symbolMapper.GetBrokerageSymbol(s) == inSymbolStr); - if (subscription != null) + if (subscription.SecurityType == SecurityType.Future) { - if (subscription.SecurityType == SecurityType.Future) + var leanSymbol = _symbolMapper.GetLeanSymbolForFuture(outSymbolStr); + if (leanSymbol == null) { - var leanSymbol = _symbolMapper.GetLeanSymbolForFuture(outSymbolStr); - if (leanSymbol == null) - { - Log.Trace($"DataBentoRawLiveClient: Future spreads are not supported: {outSymbolStr}. Skipping mapping."); - return; - } - _instrumentIdToSymbol[instrumentId] = leanSymbol; - Log.Debug($"DataBentoRawLiveClient: Mapped instrument_id {instrumentId} to {leanSymbol}"); + Log.Trace($"DataBentoRawLiveClient: Future spreads are not supported: {outSymbolStr}. Skipping mapping."); + return; } + _instrumentIdToSymbol[instrumentId] = leanSymbol; + Log.Debug($"DataBentoRawLiveClient: Mapped instrument_id {instrumentId} to {leanSymbol}"); } } } - return; - - case 1: - // MBP-1 (Market By Price) - HandleMBPMessage(root, headerElement); - return; - - case 0: - // Trade messages - HandleTradeTickMessage(root, headerElement); - return; - - case 32: - case 33: - case 34: - case 35: - // OHLCV bar messages - HandleOHLCVMessage(root, headerElement); - return; - - default: - Log.Error($"DataBentoRawLiveClient: Unknown rtype {rtype} in message"); - return; - } + } + return; + + case 1: + // MBP-1 (Market By Price) + HandleMBPMessage(root, headerElement); + return; + + case 0: + // Trade messages + HandleTradeTickMessage(root, headerElement); + return; + + case 32: + case 33: + case 34: + case 35: + // OHLCV bar messages + HandleOHLCVMessage(root, headerElement); + return; + + default: + Log.Error($"DataBentoRawLiveClient: Unknown rtype {rtype} in message"); + return; } } - - // Handle other message types if needed - if (root.TryGetProperty("error", out var errorElement)) - { - Log.Error($"DataBentoRawLiveClient: Server error: {errorElement.GetString()}"); - } } - catch (JsonException ex) - { - Log.Error($"DataBentoRawLiveClient.ProcessSingleMessage(): JSON parse error: {ex.Message}"); - } - catch (Exception ex) + + // Handle other message types if needed + if (root.TryGetProperty("error", out var errorElement)) { - Log.Error($"DataBentoRawLiveClient.ProcessSingleMessage(): Error: {ex.Message}"); + Log.Error($"DataBentoRawLiveClient: Server error: {errorElement.GetString()}"); } } - - /// - /// Handles OHLCV messages and converts to LEAN TradeBar data - /// - private void HandleOHLCVMessage(JsonElement root, JsonElement header) + catch (JsonException ex) { - try + Log.Error($"DataBentoRawLiveClient.ProcessSingleMessage(): JSON parse error: {ex.Message}"); + } + catch (Exception ex) + { + Log.Error($"DataBentoRawLiveClient.ProcessSingleMessage(): Error: {ex.Message}"); + } + } + + /// + /// Handles OHLCV messages and converts to LEAN TradeBar data + /// + private void HandleOHLCVMessage(JsonElement root, JsonElement header) + { + try + { + if (!header.TryGetProperty("ts_event", out var tsElement) || + !header.TryGetProperty("instrument_id", out var instIdElement)) { - if (!header.TryGetProperty("ts_event", out var tsElement) || - !header.TryGetProperty("instrument_id", out var instIdElement)) - { - return; - } + return; + } - // Convert timestamp from nanoseconds to DateTime - var timestampNs = long.Parse(tsElement.GetString()!); - var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); - var timestamp = unixEpoch.AddTicks(timestampNs / 100); + // Convert timestamp from nanoseconds to DateTime + var timestampNs = long.Parse(tsElement.GetString()!); + var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var timestamp = unixEpoch.AddTicks(timestampNs / 100); - var instrumentId = instIdElement.GetInt64(); + var instrumentId = instIdElement.GetInt64(); - if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) - { - Log.Debug($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in OHLCV message."); - return; - } + if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) + { + Log.Debug($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in OHLCV message."); + return; + } - // Get the resolution for this symbol - if (!_subscriptions.TryGetValue(matchedSymbol, out var subscription)) - { - return; - } + // Get the resolution for this symbol + if (!_subscriptions.TryGetValue(matchedSymbol, out var subscription)) + { + return; + } - var resolution = subscription.Item1; + var resolution = subscription.Item1; - // Extract OHLCV data - if (root.TryGetProperty("open", out var openElement) && - root.TryGetProperty("high", out var highElement) && - root.TryGetProperty("low", out var lowElement) && - root.TryGetProperty("close", out var closeElement) && - root.TryGetProperty("volume", out var volumeElement)) - { - // Parse prices - var openRaw = long.Parse(openElement.GetString()!); - var highRaw = long.Parse(highElement.GetString()!); - var lowRaw = long.Parse(lowElement.GetString()!); - var closeRaw = long.Parse(closeElement.GetString()!); - var volume = volumeElement.GetInt64(); - - var open = openRaw * PriceScaleFactor; - var high = highRaw * PriceScaleFactor; - var low = lowRaw * PriceScaleFactor; - var close = closeRaw * PriceScaleFactor; - - // Determine the period based on resolution - TimeSpan period = resolution switch - { - Resolution.Second => TimeSpan.FromSeconds(1), - Resolution.Minute => TimeSpan.FromMinutes(1), - Resolution.Hour => TimeSpan.FromHours(1), - Resolution.Daily => TimeSpan.FromDays(1), - _ => TimeSpan.FromMinutes(1) - }; - - // Create TradeBar - var tradeBar = new TradeBar( - timestamp, - matchedSymbol, - open, - high, - low, - close, - volume, - period - ); - - // Log.Trace($"DataBentoRawLiveClient: OHLCV bar: {matchedSymbol} O={open} H={high} L={low} C={close} V={volume} at {timestamp}"); - DataReceived?.Invoke(this, tradeBar); - } - } - catch (Exception ex) + // Extract OHLCV data + if (root.TryGetProperty("open", out var openElement) && + root.TryGetProperty("high", out var highElement) && + root.TryGetProperty("low", out var lowElement) && + root.TryGetProperty("close", out var closeElement) && + root.TryGetProperty("volume", out var volumeElement)) { - Log.Error($"DataBentoRawLiveClient.HandleOHLCVMessage(): Error: {ex.Message}"); + // Parse prices + var openRaw = long.Parse(openElement.GetString()!); + var highRaw = long.Parse(highElement.GetString()!); + var lowRaw = long.Parse(lowElement.GetString()!); + var closeRaw = long.Parse(closeElement.GetString()!); + var volume = volumeElement.GetInt64(); + + var open = openRaw * PriceScaleFactor; + var high = highRaw * PriceScaleFactor; + var low = lowRaw * PriceScaleFactor; + var close = closeRaw * PriceScaleFactor; + + // Determine the period based on resolution + TimeSpan period = resolution switch + { + Resolution.Second => TimeSpan.FromSeconds(1), + Resolution.Minute => TimeSpan.FromMinutes(1), + Resolution.Hour => TimeSpan.FromHours(1), + Resolution.Daily => TimeSpan.FromDays(1), + _ => TimeSpan.FromMinutes(1) + }; + + // Create TradeBar + var tradeBar = new TradeBar( + timestamp, + matchedSymbol, + open, + high, + low, + close, + volume, + period + ); + + // Log.Trace($"DataBentoRawLiveClient: OHLCV bar: {matchedSymbol} O={open} H={high} L={low} C={close} V={volume} at {timestamp}"); + DataReceived?.Invoke(this, tradeBar); } } + catch (Exception ex) + { + Log.Error($"DataBentoRawLiveClient.HandleOHLCVMessage(): Error: {ex.Message}"); + } + } - /// - /// Handles MBP messages for quote ticks - /// - private void HandleMBPMessage(JsonElement root, JsonElement header) + /// + /// Handles MBP messages for quote ticks + /// + private void HandleMBPMessage(JsonElement root, JsonElement header) + { + try { - try + if (!header.TryGetProperty("ts_event", out var tsElement) || + !header.TryGetProperty("instrument_id", out var instIdElement)) { - if (!header.TryGetProperty("ts_event", out var tsElement) || - !header.TryGetProperty("instrument_id", out var instIdElement)) - { - return; - } - - // Convert timestamp from nanoseconds to DateTime - var timestampNs = long.Parse(tsElement.GetString()!); - var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); - var timestamp = unixEpoch.AddTicks(timestampNs / 100); + return; + } - var instrumentId = instIdElement.GetInt64(); + // Convert timestamp from nanoseconds to DateTime + var timestampNs = long.Parse(tsElement.GetString()!); + var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var timestamp = unixEpoch.AddTicks(timestampNs / 100); - if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) - { - Log.Trace($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in MBP message."); - return; - } + var instrumentId = instIdElement.GetInt64(); - // For MBP-1, bid/ask data is in the levels array at index 0 - if (root.TryGetProperty("levels", out var levelsElement) && - levelsElement.GetArrayLength() > 0) - { - var level0 = levelsElement[0]; - - var quoteTick = new Tick - { - Symbol = matchedSymbol, - Time = timestamp, - TickType = TickType.Quote - }; - - if (level0.TryGetProperty("ask_px", out var askPxElement) && - level0.TryGetProperty("ask_sz", out var askSzElement)) - { - var askPriceRaw = long.Parse(askPxElement.GetString()!); - quoteTick.AskPrice = askPriceRaw * PriceScaleFactor; - quoteTick.AskSize = askSzElement.GetInt32(); - } - - if (level0.TryGetProperty("bid_px", out var bidPxElement) && - level0.TryGetProperty("bid_sz", out var bidSzElement)) - { - var bidPriceRaw = long.Parse(bidPxElement.GetString()!); - quoteTick.BidPrice = bidPriceRaw * PriceScaleFactor; - quoteTick.BidSize = bidSzElement.GetInt32(); - } - - // Set the tick value to the mid price - quoteTick.Value = (quoteTick.BidPrice + quoteTick.AskPrice) / 2; - - // QuantConnect convention: Quote ticks should have zero Price and Quantity - quoteTick.Quantity = 0; - - // Log.Trace($"DataBentoRawLiveClient: Quote tick: {matchedSymbol} Bid={quoteTick.BidPrice}x{quoteTick.BidSize} Ask={quoteTick.AskPrice}x{quoteTick.AskSize}"); - DataReceived?.Invoke(this, quoteTick); - } - } - catch (Exception ex) + if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) { - Log.Error($"DataBentoRawLiveClient.HandleMBPMessage(): Error: {ex.Message}"); + Log.Trace($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in MBP message."); + return; } - } - /// - /// Handles trade tick messages. Aggressor fills - /// - private void HandleTradeTickMessage(JsonElement root, JsonElement header) - { - try + // For MBP-1, bid/ask data is in the levels array at index 0 + if (root.TryGetProperty("levels", out var levelsElement) && + levelsElement.GetArrayLength() > 0) { - if (!header.TryGetProperty("ts_event", out var tsElement) || - !header.TryGetProperty("instrument_id", out var instIdElement)) - { - return; - } + var level0 = levelsElement[0]; - // Convert timestamp from nanoseconds to DateTime - var timestampNs = long.Parse(tsElement.GetString()!); - var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); - var timestamp = unixEpoch.AddTicks(timestampNs / 100); - - var instrumentId = instIdElement.GetInt64(); + var quoteTick = new Tick + { + Symbol = matchedSymbol, + Time = timestamp, + TickType = TickType.Quote + }; - if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) + if (level0.TryGetProperty("ask_px", out var askPxElement) && + level0.TryGetProperty("ask_sz", out var askSzElement)) { - Log.Trace($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in trade message."); - return; + var askPriceRaw = long.Parse(askPxElement.GetString()!); + quoteTick.AskPrice = askPriceRaw * PriceScaleFactor; + quoteTick.AskSize = askSzElement.GetInt32(); } - if (root.TryGetProperty("price", out var priceElement) && - root.TryGetProperty("size", out var sizeElement)) + if (level0.TryGetProperty("bid_px", out var bidPxElement) && + level0.TryGetProperty("bid_sz", out var bidSzElement)) { - var priceRaw = long.Parse(priceElement.GetString()!); - var size = sizeElement.GetInt32(); - var price = priceRaw * PriceScaleFactor; - - var tradeTick = new Tick - { - Symbol = matchedSymbol, - Time = timestamp, - Value = price, - Quantity = size, - TickType = TickType.Trade, - // Trade ticks should have zero bid/ask values - BidPrice = 0, - BidSize = 0, - AskPrice = 0, - AskSize = 0 - }; - - // Log.Trace($"DataBentoRawLiveClient: Trade tick: {matchedSymbol} Price={price} Quantity={size}"); - DataReceived?.Invoke(this, tradeTick); + var bidPriceRaw = long.Parse(bidPxElement.GetString()!); + quoteTick.BidPrice = bidPriceRaw * PriceScaleFactor; + quoteTick.BidSize = bidSzElement.GetInt32(); } + + // Set the tick value to the mid price + quoteTick.Value = (quoteTick.BidPrice + quoteTick.AskPrice) / 2; + + // QuantConnect convention: Quote ticks should have zero Price and Quantity + quoteTick.Quantity = 0; + + // Log.Trace($"DataBentoRawLiveClient: Quote tick: {matchedSymbol} Bid={quoteTick.BidPrice}x{quoteTick.BidSize} Ask={quoteTick.AskPrice}x{quoteTick.AskSize}"); + DataReceived?.Invoke(this, quoteTick); } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.HandleTradeTickMessage(): Error: {ex.Message}"); - } } + catch (Exception ex) + { + Log.Error($"DataBentoRawLiveClient.HandleMBPMessage(): Error: {ex.Message}"); + } + } - /// - /// Disconnects from the DataBento gateway - /// - public void Disconnect() + /// + /// Handles trade tick messages. Aggressor fills + /// + private void HandleTradeTickMessage(JsonElement root, JsonElement header) + { + try { - lock (_connectionLock) + if (!header.TryGetProperty("ts_event", out var tsElement) || + !header.TryGetProperty("instrument_id", out var instIdElement)) { - if (!_isConnected) - return; + return; + } - _isConnected = false; - _cancellationTokenSource?.Cancel(); + // Convert timestamp from nanoseconds to DateTime + var timestampNs = long.Parse(tsElement.GetString()!); + var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var timestamp = unixEpoch.AddTicks(timestampNs / 100); - try - { - _reader?.Dispose(); - _writer?.Dispose(); - _stream?.Close(); - _tcpClient?.Close(); - } - catch (Exception ex) - { - Log.Trace($"DataBentoRawLiveClient.Disconnect(): Error during disconnect: {ex.Message}"); - } + var instrumentId = instIdElement.GetInt64(); - ConnectionStatusChanged?.Invoke(this, false); - Log.Trace("DataBentoRawLiveClient.Disconnect(): Disconnected from DataBento gateway"); + if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) + { + Log.Trace($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in trade message."); + return; } + + if (root.TryGetProperty("price", out var priceElement) && + root.TryGetProperty("size", out var sizeElement)) + { + var priceRaw = long.Parse(priceElement.GetString()!); + var size = sizeElement.GetInt32(); + var price = priceRaw * PriceScaleFactor; + + var tradeTick = new Tick + { + Symbol = matchedSymbol, + Time = timestamp, + Value = price, + Quantity = size, + TickType = TickType.Trade, + // Trade ticks should have zero bid/ask values + BidPrice = 0, + BidSize = 0, + AskPrice = 0, + AskSize = 0 + }; + + // Log.Trace($"DataBentoRawLiveClient: Trade tick: {matchedSymbol} Price={price} Quantity={size}"); + DataReceived?.Invoke(this, tradeTick); + } + } + catch (Exception ex) + { + Log.Error($"DataBentoRawLiveClient.HandleTradeTickMessage(): Error: {ex.Message}"); } + } - /// - /// Disposes of resources - /// - public void Dispose() + /// + /// Disconnects from the DataBento gateway + /// + public void Disconnect() + { + lock (_connectionLock) { - if (_disposed) + if (!_isConnected) return; - _disposed = true; - Disconnect(); + _isConnected = false; + _cancellationTokenSource?.Cancel(); - _cancellationTokenSource?.Dispose(); - _reader?.Dispose(); - _writer?.Dispose(); - _stream?.Dispose(); - _tcpClient?.Dispose(); - } - - /// - /// Computes the SHA-256 hash of the input string - /// - private static string ComputeSHA256(string input) - { - using var sha = SHA256.Create(); - var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(input)); - var sb = new StringBuilder(); - foreach (byte b in hash) + try { - sb.Append(b.ToString("x2")); + _reader?.Dispose(); + _writer?.Dispose(); + _stream?.Close(); + _tcpClient?.Close(); } - return sb.ToString(); + catch (Exception ex) + { + Log.Trace($"DataBentoRawLiveClient.Disconnect(): Error during disconnect: {ex.Message}"); + } + + ConnectionStatusChanged?.Invoke(this, false); + Log.Trace("DataBentoRawLiveClient.Disconnect(): Disconnected from DataBento gateway"); } + } + + /// + /// Disposes of resources + /// + public void Dispose() + { + if (_disposed) + return; + _disposed = true; + Disconnect(); + + _cancellationTokenSource?.Dispose(); + _reader?.Dispose(); + _writer?.Dispose(); + _stream?.Dispose(); + _tcpClient?.Dispose(); } + + /// + /// Computes the SHA-256 hash of the input string + /// + private static string ComputeSHA256(string input) + { + using var sha = SHA256.Create(); + var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(input)); + var sb = new StringBuilder(); + foreach (byte b in hash) + { + sb.Append(b.ToString("x2")); + } + return sb.ToString(); + } + } diff --git a/QuantConnect.DataBento/DataBentoSymbolMapper.cs b/QuantConnect.DataBento/DataBentoSymbolMapper.cs index 5c90304..7294922 100644 --- a/QuantConnect.DataBento/DataBentoSymbolMapper.cs +++ b/QuantConnect.DataBento/DataBentoSymbolMapper.cs @@ -15,64 +15,63 @@ using QuantConnect.Brokerages; -namespace QuantConnect.Lean.DataSource.DataBento +namespace QuantConnect.Lean.DataSource.DataBento; + +/// +/// Provides the mapping between Lean symbols and DataBento symbols. +/// +public class DataBentoSymbolMapper : ISymbolMapper { + /// - /// Provides the mapping between Lean symbols and DataBento symbols. + /// Converts a Lean symbol instance to a brokerage symbol /// - public class DataBentoSymbolMapper : ISymbolMapper + /// A Lean symbol instance + /// The brokerage symbol + public string GetBrokerageSymbol(Symbol symbol) { - - /// - /// Converts a Lean symbol instance to a brokerage symbol - /// - /// A Lean symbol instance - /// The brokerage symbol - public string GetBrokerageSymbol(Symbol symbol) + switch (symbol.SecurityType) { - switch (symbol.SecurityType) - { - case SecurityType.Future: - return SymbolRepresentation.GenerateFutureTicker(symbol.ID.Symbol, symbol.ID.Date, doubleDigitsYear: false, includeExpirationDate: false); - default: - throw new Exception($"The unsupported security type: {symbol.SecurityType}"); - } + case SecurityType.Future: + return SymbolRepresentation.GenerateFutureTicker(symbol.ID.Symbol, symbol.ID.Date, doubleDigitsYear: false, includeExpirationDate: false); + default: + throw new Exception($"The unsupported security type: {symbol.SecurityType}"); } + } - /// - /// Converts a brokerage symbol to a Lean symbol instance - /// - /// The brokerage symbol - /// The security type - /// The market - /// Expiration date of the security(if applicable) - /// A new Lean Symbol instance - public Symbol GetLeanSymbol(string brokerageSymbol, SecurityType securityType, string market, - DateTime expirationDate = new DateTime(), decimal strike = 0, OptionRight optionRight = 0) + /// + /// Converts a brokerage symbol to a Lean symbol instance + /// + /// The brokerage symbol + /// The security type + /// The market + /// Expiration date of the security(if applicable) + /// A new Lean Symbol instance + public Symbol GetLeanSymbol(string brokerageSymbol, SecurityType securityType, string market, + DateTime expirationDate = new DateTime(), decimal strike = 0, OptionRight optionRight = 0) + { + switch (securityType) { - switch (securityType) - { - case SecurityType.Future: - return Symbol.CreateFuture(brokerageSymbol, market, expirationDate); - default: - throw new Exception($"The unsupported security type: {securityType}"); - } + case SecurityType.Future: + return Symbol.CreateFuture(brokerageSymbol, market, expirationDate); + default: + throw new Exception($"The unsupported security type: {securityType}"); } + } - /// - /// Converts a brokerage future symbol to a Lean symbol instance - /// - /// The brokerage symbol - /// A new Lean Symbol instance - public Symbol GetLeanSymbolForFuture(string brokerageSymbol) + /// + /// Converts a brokerage future symbol to a Lean symbol instance + /// + /// The brokerage symbol + /// A new Lean Symbol instance + public Symbol GetLeanSymbolForFuture(string brokerageSymbol) + { + // ignore futures spreads + if (brokerageSymbol.Contains("-")) { - // ignore futures spreads - if (brokerageSymbol.Contains("-")) - { - return null; - } - - return SymbolRepresentation.ParseFutureSymbol(brokerageSymbol); + return null; } + + return SymbolRepresentation.ParseFutureSymbol(brokerageSymbol); } } diff --git a/QuantConnect.DataBento/Extensions.cs b/QuantConnect.DataBento/Extensions.cs index 14e8834..a4514db 100644 --- a/QuantConnect.DataBento/Extensions.cs +++ b/QuantConnect.DataBento/Extensions.cs @@ -14,7 +14,6 @@ */ using Newtonsoft.Json; -using Newtonsoft.Json.Serialization; using QuantConnect.Lean.DataSource.DataBento.Serialization; namespace QuantConnect.Lean.DataSource.DataBento; From 3513bcfbf78760a6fa71aa07dc1a23d02655228a Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 26 Jan 2026 15:22:05 +0200 Subject: [PATCH 08/18] feat: debug mode config --- QuantConnect.DataBento.Tests/TestSetup.cs | 4 ++-- QuantConnect.DataBento.Tests/config.json | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/QuantConnect.DataBento.Tests/TestSetup.cs b/QuantConnect.DataBento.Tests/TestSetup.cs index 86d2f07..e56a03d 100644 --- a/QuantConnect.DataBento.Tests/TestSetup.cs +++ b/QuantConnect.DataBento.Tests/TestSetup.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. * @@ -28,7 +28,7 @@ public class TestSetup [OneTimeSetUp] public void GlobalSetup() { - Log.DebuggingEnabled = true; + Log.DebuggingEnabled = Config.GetBool("debug-mode"); Log.LogHandler = new CompositeLogHandler(); Log.Trace("TestSetup(): starting..."); ReloadConfiguration(); diff --git a/QuantConnect.DataBento.Tests/config.json b/QuantConnect.DataBento.Tests/config.json index 2464c3e..0d61d83 100644 --- a/QuantConnect.DataBento.Tests/config.json +++ b/QuantConnect.DataBento.Tests/config.json @@ -3,5 +3,6 @@ "job-user-id": "", "api-access-token": "", "job-organization-id": "", - "databento-api-key": "" + "databento-api-key": "", + "debug-mode": false } \ No newline at end of file From ee90b571aba7d97f2ded8aa437b5131a23253527 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 26 Jan 2026 15:24:05 +0200 Subject: [PATCH 09/18] remove: IsSupported() doesn't validate anything --- .../DataBentoDataProvider.cs | 32 ------------------- 1 file changed, 32 deletions(-) diff --git a/QuantConnect.DataBento/DataBentoDataProvider.cs b/QuantConnect.DataBento/DataBentoDataProvider.cs index 40fc095..9b74d9e 100644 --- a/QuantConnect.DataBento/DataBentoDataProvider.cs +++ b/QuantConnect.DataBento/DataBentoDataProvider.cs @@ -201,11 +201,6 @@ private static bool CanSubscribe(Symbol symbol) /// The new enumerator for this subscription request public IEnumerator? Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler) { - if (!IsSupported(dataConfig.SecurityType, dataConfig.Type, dataConfig.TickType, dataConfig.Resolution)) - { - return null; - } - lock (_sessionLock) { if (!_sessionStarted) @@ -254,33 +249,6 @@ public void Dispose() _client?.DisposeSafely(); } - /// - /// Determines if the specified subscription is supported - /// - private bool IsSupported(SecurityType securityType, Type dataType, TickType tickType, Resolution resolution) - { - // Check supported data types - if (dataType != typeof(TradeBar) && - dataType != typeof(QuoteBar) && - dataType != typeof(Tick) && - dataType != typeof(OpenInterest)) - { - throw new NotSupportedException($"Unsupported data type: {dataType}"); - } - - // Warn about potential limitations for tick data - // I'm mimicing polygon implementation with this - if (!_potentialUnsupportedResolutionMessageLogged) - { - _potentialUnsupportedResolutionMessageLogged = true; - Log.Trace("DataBentoDataProvider.IsSupported(): " + - $"Subscription for {securityType}-{dataType}-{tickType}-{resolution} will be attempted. " + - $"An Advanced DataBento subscription plan is required to stream tick data."); - } - - return true; - } - /// /// Converts the given UTC time into the symbol security exchange time zone /// From 531056a75c254d323974d26727f8a355b7114897 Mon Sep 17 00:00:00 2001 From: Romazes Date: Mon, 26 Jan 2026 15:35:15 +0200 Subject: [PATCH 10/18] feat: map Lean.Market <-> DataBento.DataSet --- .../DataBentoHistoricalApiClientTests.cs | 13 ++++- .../DataBentoSymbolMapperTests.cs.cs | 10 ++++ .../Api/HistoricalAPIClient.cs | 29 ++++-------- .../DataBentoDataProvider.cs | 15 ++++++ .../DataBentoHistoryProivder.cs | 47 +++++++++++++------ .../DataBentoSymbolMapper.cs | 17 +++++++ 6 files changed, 94 insertions(+), 37 deletions(-) diff --git a/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs index 7edac6d..4623776 100644 --- a/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs @@ -26,6 +26,15 @@ public class DataBentoHistoricalApiClientTests { private HistoricalAPIClient _client; + /// + /// Dataset for CME Globex futures + /// https://databento.com/docs/venues-and-datasets has more information on datasets through DataBento + /// + /// + /// TODO: Hard coded for now. Later on can add equities and options with different mapping + /// + private const string Dataset = "GLBX.MDP3"; + [OneTimeSetUp] public void OneTimeSetUp() { @@ -48,7 +57,7 @@ public void CanInitializeHistoricalApiClient(string ticker, DateTime startDate, { var dataCounter = 0; var previousEndTime = DateTime.MinValue; - foreach (var data in _client.GetHistoricalOhlcvBars(ticker, startDate, endDate, resolution, TickType.Trade)) + foreach (var data in _client.GetHistoricalOhlcvBars(ticker, startDate, endDate, resolution, Dataset)) { Assert.IsNotNull(data); @@ -75,7 +84,7 @@ public void ShouldFetchOpenInterest(string ticker, DateTime startDate, DateTime { var dataCounter = 0; var previousEndTime = DateTime.MinValue; - foreach (var data in _client.GetOpenInterest(ticker, startDate, endDate)) + foreach (var data in _client.GetOpenInterest(ticker, startDate, endDate, Dataset)) { Assert.IsNotNull(data); diff --git a/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs b/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs index ad14aad..feb393a 100644 --- a/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs +++ b/QuantConnect.DataBento.Tests/DataBentoSymbolMapperTests.cs.cs @@ -53,4 +53,14 @@ public void ReturnsCorrectBrokerageSymbol(Symbol symbol, string expectedBrokerag Assert.IsNotEmpty(brokerageSymbol); Assert.AreEqual(expectedBrokerageSymbol, brokerageSymbol); } + + [TestCase(Market.CME, "GLBX.MDP3", true)] + [TestCase(Market.EUREX, "XEUR.EOBI", true)] + [TestCase(Market.USA, null, false)] + public void ReturnsCorrectDataBentoDataSet(string market, string expectedDataSet, bool expectedExist) + { + var actualExist = _symbolMapper.DataBentoDataSetByLeanMarket.TryGetValue(market, out var actualDataSet); + Assert.AreEqual(expectedExist, actualExist); + Assert.AreEqual(expectedDataSet, actualDataSet); + } } diff --git a/QuantConnect.DataBento/Api/HistoricalAPIClient.cs b/QuantConnect.DataBento/Api/HistoricalAPIClient.cs index 92dc868..23c79ee 100644 --- a/QuantConnect.DataBento/Api/HistoricalAPIClient.cs +++ b/QuantConnect.DataBento/Api/HistoricalAPIClient.cs @@ -24,17 +24,6 @@ namespace QuantConnect.Lean.DataSource.DataBento.Api; public class HistoricalAPIClient : IDisposable { - //private const string - - /// - /// Dataset for CME Globex futures - /// https://databento.com/docs/venues-and-datasets has more information on datasets through DataBento - /// - /// - /// TODO: Hard coded for now. Later on can add equities and options with different mapping - /// - private const string Dataset = "GLBX.MDP3"; - private readonly HttpClient _httpClient = new() { BaseAddress = new Uri("https://hist.databento.com") @@ -45,11 +34,11 @@ public HistoricalAPIClient(string apiKey) _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( AuthenticationSchemes.Basic.ToString(), // Basic Auth expects "username:password". Using ":" means API key with an empty password. - Convert.ToBase64String(Encoding.UTF8.GetBytes($"{apiKey}:")) + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{apiKey}:")) ); } - public IEnumerable GetHistoricalOhlcvBars(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, Resolution resolution, TickType tickType) + public IEnumerable GetHistoricalOhlcvBars(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, Resolution resolution, string dataSet) { string schema; switch (resolution) @@ -70,17 +59,17 @@ public IEnumerable GetHistoricalOhlcvBars(string symbol, DateTime star throw new ArgumentException($"Unsupported resolution {resolution} for OHLCV data."); } - return GetRange(symbol, startDateTimeUtc, endDateTimeUtc, schema); + return GetRange(symbol, startDateTimeUtc, endDateTimeUtc, schema, dataSet); } - public IEnumerable GetTickBars(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc) + public IEnumerable GetTickBars(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, string dataSet) { - return GetRange(symbol, startDateTimeUtc, endDateTimeUtc, "mbp-1", useLimit: true); + return GetRange(symbol, startDateTimeUtc, endDateTimeUtc, "mbp-1", dataSet, useLimit: true); } - public IEnumerable GetOpenInterest(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc) + public IEnumerable GetOpenInterest(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, string dataSet) { - foreach (var statistics in GetRange(symbol, startDateTimeUtc, endDateTimeUtc, "statistics")) + foreach (var statistics in GetRange(symbol, startDateTimeUtc, endDateTimeUtc, "statistics", dataSet)) { if (statistics.StatType == Models.Enums.StatisticType.OpenInterest) { @@ -89,11 +78,11 @@ public IEnumerable GetOpenInterest(string symbol, DateTime start } } - private IEnumerable GetRange(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, string schema, bool useLimit = false) where T : MarketDataRecord + private IEnumerable GetRange(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, string schema, string dataSet, bool useLimit = false) where T : MarketDataRecord { var formData = new Dictionary { - { "dataset", Dataset }, + { "dataset", dataSet }, { "end", Time.DateTimeToUnixTimeStampNanoseconds(endDateTimeUtc).ToStringInvariant() }, { "symbols", symbol }, { "schema", schema }, diff --git a/QuantConnect.DataBento/DataBentoDataProvider.cs b/QuantConnect.DataBento/DataBentoDataProvider.cs index 9b74d9e..ba228d1 100644 --- a/QuantConnect.DataBento/DataBentoDataProvider.cs +++ b/QuantConnect.DataBento/DataBentoDataProvider.cs @@ -156,6 +156,21 @@ public bool UnsubscribeLogic(IEnumerable symbols, TickType tickType) return true; } + /// + /// Attempts to resolve the DataBento dataset for the specified symbol based on its Lean market. + /// + /// The symbol whose market is used to determine the DataBento dataset. + /// + /// When this method returns true, contains the resolved DataBento dataset; otherwise, null. + /// + /// + /// true if a DataBento dataset mapping exists for the symbol's market; otherwise, false. + /// + private bool TryGetDataBentoDataSet(Symbol symbol, out string? dataSet) + { + return _symbolMapper.DataBentoDataSetByLeanMarket.TryGetValue(symbol.ID.Market, out dataSet); + } + /// /// Logic to subscribe to the specified symbols /// diff --git a/QuantConnect.DataBento/DataBentoHistoryProivder.cs b/QuantConnect.DataBento/DataBentoHistoryProivder.cs index 2485ff5..fabf42e 100644 --- a/QuantConnect.DataBento/DataBentoHistoryProivder.cs +++ b/QuantConnect.DataBento/DataBentoHistoryProivder.cs @@ -44,6 +44,11 @@ public partial class DataBentoProvider : SynchronizingHistoryProvider /// private volatile bool _invalidSecurityTypeWarningFired; + /// + /// Indicates whether a DataBento dataset error has already been logged. + /// + private bool _dataBentoDatasetErrorFired; + /// /// Gets the total number of data points emitted by this history provider /// @@ -113,21 +118,33 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) return null; } + if (!TryGetDataBentoDataSet(historyRequest.Symbol, out var dataSet) || dataSet == null) + { + if (!_dataBentoDatasetErrorFired) + { + _dataBentoDatasetErrorFired = true; + Log.Error($"{nameof(DataBentoProvider)}.{nameof(GetHistory)}: " + + $"DataBento dataset not found for symbol '{historyRequest.Symbol.Value}, Market = {historyRequest.Symbol.ID.Market}." + ); + } + return null; + } + var history = default(IEnumerable); var brokerageSymbol = _symbolMapper.GetBrokerageSymbol(historyRequest.Symbol); switch (historyRequest.TickType) { case TickType.Trade when historyRequest.Resolution == Resolution.Tick: - history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol); + history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol, dataSet); break; case TickType.Trade: - history = GetAggregatedTradeBars(historyRequest, brokerageSymbol); + history = GetAggregatedTradeBars(historyRequest, brokerageSymbol, dataSet); break; case TickType.Quote: - history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol); + history = GetHistoryThroughDataConsolidator(historyRequest, brokerageSymbol, dataSet); break; case TickType.OpenInterest: - history = GetOpenInterestBars(historyRequest, brokerageSymbol); + history = GetOpenInterestBars(historyRequest, brokerageSymbol, dataSet); break; default: throw new ArgumentException(""); @@ -157,24 +174,24 @@ private static IEnumerable FilterHistory(IEnumerable history } } - private IEnumerable GetOpenInterestBars(HistoryRequest request, string brokerageSymbol) + private IEnumerable GetOpenInterestBars(HistoryRequest request, string brokerageSymbol, string dataBentoDataSet) { - foreach (var oi in _historicalApiClient.GetOpenInterest(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) + foreach (var oi in _historicalApiClient.GetOpenInterest(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc, dataBentoDataSet)) { yield return new OpenInterest(oi.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, oi.Quantity); } } - private IEnumerable GetAggregatedTradeBars(HistoryRequest request, string brokerageSymbol) + private IEnumerable GetAggregatedTradeBars(HistoryRequest request, string brokerageSymbol, string dataBentoDataSet) { var period = request.Resolution.ToTimeSpan(); - foreach (var b in _historicalApiClient.GetHistoricalOhlcvBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc, request.Resolution, request.TickType)) + foreach (var b in _historicalApiClient.GetHistoricalOhlcvBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc, request.Resolution, dataBentoDataSet)) { yield return new TradeBar(b.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, b.Open, b.High, b.Low, b.Close, b.Volume, period); } } - private IEnumerable? GetHistoryThroughDataConsolidator(HistoryRequest request, string brokerageSymbol) + private IEnumerable? GetHistoryThroughDataConsolidator(HistoryRequest request, string brokerageSymbol, string dataBentoDataSet) { IDataConsolidator consolidator; IEnumerable history; @@ -184,14 +201,14 @@ private IEnumerable GetAggregatedTradeBars(HistoryRequest request, str consolidator = request.Resolution != Resolution.Tick ? new TickConsolidator(request.Resolution.ToTimeSpan()) : FilteredIdentityDataConsolidator.ForTickType(request.TickType); - history = GetTrades(request, brokerageSymbol); + history = GetTrades(request, brokerageSymbol, dataBentoDataSet); } else { consolidator = request.Resolution != Resolution.Tick ? new TickQuoteBarConsolidator(request.Resolution.ToTimeSpan()) : FilteredIdentityDataConsolidator.ForTickType(request.TickType); - history = GetQuotes(request, brokerageSymbol); + history = GetQuotes(request, brokerageSymbol, dataBentoDataSet); } BaseData? consolidatedData = null; @@ -218,9 +235,9 @@ private IEnumerable GetAggregatedTradeBars(HistoryRequest request, str /// /// Gets the trade ticks that will potentially be aggregated for the specified history request /// - private IEnumerable GetTrades(HistoryRequest request, string brokerageSymbol) + private IEnumerable GetTrades(HistoryRequest request, string brokerageSymbol, string dataBentoDataSet) { - foreach (var t in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) + foreach (var t in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc, dataBentoDataSet)) { yield return new Tick(t.Header.UtcTime.ConvertFromUtc(request.DataTimeZone), request.Symbol, "", "", t.Size, t.Price); } @@ -229,9 +246,9 @@ private IEnumerable GetTrades(HistoryRequest request, string brokerage /// /// Gets the quote ticks that will potentially be aggregated for the specified history request /// - private IEnumerable GetQuotes(HistoryRequest request, string brokerageSymbol) + private IEnumerable GetQuotes(HistoryRequest request, string brokerageSymbol, string dataBentoDataSet) { - foreach (var quoteBar in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc)) + foreach (var quoteBar in _historicalApiClient.GetTickBars(brokerageSymbol, request.StartTimeUtc, request.EndTimeUtc, dataBentoDataSet)) { var time = quoteBar.Header.UtcTime.ConvertFromUtc(request.DataTimeZone); foreach (var level in quoteBar.Levels) diff --git a/QuantConnect.DataBento/DataBentoSymbolMapper.cs b/QuantConnect.DataBento/DataBentoSymbolMapper.cs index 7294922..1a824ee 100644 --- a/QuantConnect.DataBento/DataBentoSymbolMapper.cs +++ b/QuantConnect.DataBento/DataBentoSymbolMapper.cs @@ -14,6 +14,7 @@ */ using QuantConnect.Brokerages; +using System.Collections.Frozen; namespace QuantConnect.Lean.DataSource.DataBento; @@ -22,6 +23,22 @@ namespace QuantConnect.Lean.DataSource.DataBento; /// public class DataBentoSymbolMapper : ISymbolMapper { + /// + /// Dataset for CME Globex futures + /// https://databento.com/docs/venues-and-datasets has more information on datasets through DataBento + /// + public FrozenDictionary DataBentoDataSetByLeanMarket = new Dictionary + { + { Market.EUREX, "XEUR.EOBI" }, + + { Market.CBOT, "GLBX.MDP3" }, + { Market.CME, "GLBX.MDP3" }, + { Market.COMEX, "GLBX.MDP3" }, + { Market.NYMEX, "GLBX.MDP3" }, + + { Market.ICE, "IFUS.IMPACT" }, + { Market.NYSELIFFE, "IFUS.IMPACT" } + }.ToFrozenDictionary(); /// /// Converts a Lean symbol instance to a brokerage symbol From 4c322c0e8ba4af61498b394a9c8aab1b3acf1d84 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 27 Jan 2026 13:20:31 +0200 Subject: [PATCH 11/18] feat: Data Live TcpClient Wrapper feat: Record Type of different DataBento scheme feat: handle Auth-Request/Response msgs test:feat: deserialize Heartbeat json msg, parse tcp response/request test:feat: LiveApiClient quick connection and debug process --- .../DataBentoJsonConverterTests.cs | 46 ++++ .../DataBentoLiveAPIClientTests.cs | 65 +++++ QuantConnect.DataBento/Api/LiveAPIClient.cs | 83 +++++++ .../Api/LiveDataTcpClientWrapper.cs | 229 ++++++++++++++++++ .../DataBentoRawLiveClient.cs | 2 + .../Models/Enums/RecordType.cs | 86 +++++++ QuantConnect.DataBento/Models/Header.cs | 4 +- .../Live/AuthenticationMessageRequest.cs | 69 ++++++ .../Live/AuthenticationMessageResponse.cs | 54 +++++ .../Models/Live/HeartbeatMessage.cs | 22 ++ 10 files changed, 659 insertions(+), 1 deletion(-) create mode 100644 QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs create mode 100644 QuantConnect.DataBento/Api/LiveAPIClient.cs create mode 100644 QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs create mode 100644 QuantConnect.DataBento/Models/Enums/RecordType.cs create mode 100644 QuantConnect.DataBento/Models/Live/AuthenticationMessageRequest.cs create mode 100644 QuantConnect.DataBento/Models/Live/AuthenticationMessageResponse.cs create mode 100644 QuantConnect.DataBento/Models/Live/HeartbeatMessage.cs diff --git a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs index 9d43db8..5b9de56 100644 --- a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs @@ -16,6 +16,7 @@ using NUnit.Framework; using QuantConnect.Lean.DataSource.DataBento.Models; using QuantConnect.Lean.DataSource.DataBento.Models.Enums; +using QuantConnect.Lean.DataSource.DataBento.Models.Live; namespace QuantConnect.Lean.DataSource.DataBento.Tests; @@ -146,4 +147,49 @@ public void DeserializeHistoricalStatisticsData() Assert.AreEqual(470m, res.Quantity); Assert.AreEqual(StatisticType.OpenInterest, res.StatType); } + + [Test] + public void DeserializeLiveHeartbeatMessage() + { + var json = @"{""hd"":{""ts_event"":""1769176693139629181"",""rtype"":23,""publisher_id"":0,""instrument_id"":0},""msg"":""Heartbeat""}"; + + var res = json.DeserializeKebabCase(); + + Assert.IsNotNull(res); + Assert.AreEqual(1769176693139629181, res.Header.TsEvent); + Assert.AreEqual(RecordType.System, res.Header.Rtype); + Assert.AreEqual(0, res.Header.PublisherId); + Assert.AreEqual(0, res.Header.InstrumentId); + Assert.AreEqual("Heartbeat", res.Msg); + } + + [TestCase("success=0|error=Unknown subscription param 'sssauth'", false)] + [TestCase("success=0|error=Authentication failed.", false)] + [TestCase("success=1|session_id=1769508116", true)] + public void ParsePotentialAuthenticationMessageResponses(string authenticationResponse, bool success) + { + var auth = new AuthenticationMessageResponse(authenticationResponse); + + if (success) + { + Assert.IsTrue(auth.Success); + Assert.AreNotEqual(0, auth.SessionId); + } + else + { + Assert.IsFalse(auth.Success); + Assert.That(auth.Error, Is.Not.Null.And.Not.Empty); + } + } + + [TestCase("cram=HCxTgxMcqglVMTMeaDZ2ICmcnrW8j92e", "auth=a6c5c23e06854dc0310e11ce6d3081509e415a5a37a323bb94bc90f64c9214d4-12345|dataset=GLBX.MDP3|pretty_px=1|encoding=json|heartbeat_interval_s=5")] + [TestCase("cram=HCxTgxMcqglVMTMeaDZ2ICmcnrW8j92e\n", "auth=a6c5c23e06854dc0310e11ce6d3081509e415a5a37a323bb94bc90f64c9214d4-12345|dataset=GLBX.MDP3|pretty_px=1|encoding=json|heartbeat_interval_s=5")] + public void ParsePotentialCramChallenges(string challenge, string expectedString) + { + var auth = new AuthenticationMessageRequest(challenge, "my-api-key-12345", "GLBX.MDP3"); + + var actualString = auth.ToString(); + + Assert.AreEqual(expectedString, actualString); + } } diff --git a/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs new file mode 100644 index 0000000..1124895 --- /dev/null +++ b/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs @@ -0,0 +1,65 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System; +using NUnit.Framework; +using System.Threading; +using QuantConnect.Configuration; +using QuantConnect.Lean.DataSource.DataBento.Api; + +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoLiveAPIClientTests +{ + /// + /// Dataset for CME Globex futures + /// https://databento.com/docs/venues-and-datasets has more information on datasets through DataBento + /// + /// + /// TODO: Hard coded for now. Later on can add equities and options with different mapping + /// + private const string Dataset = "GLBX.MDP3"; + + private LiveAPIClient _live; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + var apiKey = Config.Get("databento-api-key"); + if (string.IsNullOrEmpty(apiKey)) + { + Assert.Inconclusive("Please set the 'databento-api-key' in your configuration to enable these tests."); + } + + _live = new LiveAPIClient(apiKey); + } + + [OneTimeTearDown] + public void OneTimeTearDown() + { + _live.Dispose(); + } + + [Test] + public void TestExample() + { + var dataAvailableEvent = new AutoResetEvent(false); + + _live.Start(Dataset); + + dataAvailableEvent.WaitOne(TimeSpan.FromSeconds(60)); + } +} diff --git a/QuantConnect.DataBento/Api/LiveAPIClient.cs b/QuantConnect.DataBento/Api/LiveAPIClient.cs new file mode 100644 index 0000000..f8b60df --- /dev/null +++ b/QuantConnect.DataBento/Api/LiveAPIClient.cs @@ -0,0 +1,83 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using QuantConnect.Util; +using QuantConnect.Logging; + +namespace QuantConnect.Lean.DataSource.DataBento.Api; + +public sealed class LiveAPIClient : IDisposable +{ + private readonly string _apiKey; + + private readonly Dictionary _tcpClientByDataSet = []; + + public LiveAPIClient(string apiKey) + { + _apiKey = apiKey; + } + + public void Dispose() + { + foreach (var tcpClient in _tcpClientByDataSet.Values) + { + tcpClient.DisposeSafely(); + } + _tcpClientByDataSet.Clear(); + } + + public bool Start(string dataSet) + { + LogTrace(nameof(Start), "Starting connection to DataBento live API"); + + if (_tcpClientByDataSet.TryGetValue(dataSet, out var existingClient) && existingClient.IsConnected) + { + LogTrace(nameof(Start), $"Already connected to DataBento live API (Dataset: {dataSet})"); + return true; + } + + var liveDataTcpClient = new LiveDataTcpClientWrapper(dataSet, _apiKey); + _tcpClientByDataSet[dataSet] = liveDataTcpClient; + liveDataTcpClient.Connect(); + + LogTrace(nameof(Start), $"Successfully connected to DataBento live API (Dataset: {dataSet})"); + + return true; + } + + public bool Subscribe(string dataSet, string symbol) + { + if (!_tcpClientByDataSet.TryGetValue(dataSet, out var tcpClient) || !tcpClient.IsConnected) + { + LogError(nameof(Subscribe), $"Not connected to DataBento live API (Dataset: {dataSet})"); + return false; + } + + tcpClient.SubscribeOnMarketBestPriceLevelOne(symbol); + + return true; + } + + private static void LogTrace(string method, string message) + { + Log.Trace($"LiveAPIClient.{method}: {message}"); + } + + private static void LogError(string method, string message) + { + Log.Error($"LiveAPIClient.{method}: {message}"); + } +} diff --git a/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs b/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs new file mode 100644 index 0000000..205ff78 --- /dev/null +++ b/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs @@ -0,0 +1,229 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System.Text; +using QuantConnect.Util; +using System.Net.Sockets; +using QuantConnect.Logging; +using QuantConnect.Lean.DataSource.DataBento.Models.Live; + +namespace QuantConnect.Lean.DataSource.DataBento.Api; + +public sealed class LiveDataTcpClientWrapper : IDisposable +{ + private const int DefaultPort = 13000; + private const int ReceiveBufferSize = 8192; + + private readonly string _gateway; + private readonly string _dataSet; + private readonly string _apiKey; + private readonly TimeSpan _heartBeatInterval = TimeSpan.FromSeconds(10); + + private readonly TcpClient _tcpClient = new(); + private readonly byte[] _receiveBuffer = new byte[ReceiveBufferSize]; + private readonly CancellationTokenSource _cancellationTokenSource = new(); + private readonly char[] _newLine = Environment.NewLine.ToCharArray(); + + private NetworkStream? _stream; + private Task? _dataReceiverTask; + private bool _isConnected; + + /// + /// Is client connected + /// + public bool IsConnected => _isConnected; + + public LiveDataTcpClientWrapper(string dataSet, string apiKey) + { + _apiKey = apiKey; + _dataSet = dataSet; + _gateway = DetermineGateway(dataSet); + } + + public void Connect() + { + _tcpClient.Connect(_gateway, DefaultPort); + _stream = _tcpClient.GetStream(); + + if (!Authenticate(_dataSet).SynchronouslyAwaitTask()) + throw new Exception("Authentication failed"); + + _dataReceiverTask = new Task(async () => await DataReceiverAsync(_cancellationTokenSource.Token), _cancellationTokenSource.Token, TaskCreationOptions.LongRunning); + _dataReceiverTask.Start(); + + _isConnected = true; + } + + public void Dispose() + { + _isConnected = false; + + _stream?.Close(); + _stream?.DisposeSafely(); + + _cancellationTokenSource?.Cancel(); + _cancellationTokenSource?.DisposeSafely(); + + _dataReceiverTask?.DisposeSafely(); + _tcpClient?.Close(); + _tcpClient?.DisposeSafely(); + } + + public void SubscribeOnMarketBestPriceLevelOne(string symbol) + { + var request = $"schema=mbp-1|stype_in=raw_symbol|symbols={symbol}"; + WriteData(request); + } + + private async Task DataReceiverAsync(CancellationToken ct) + { + var methodName = nameof(DataReceiverAsync); + + var readTimeoutCts = CancellationTokenSource.CreateLinkedTokenSource(ct); + + var readTimeout = _heartBeatInterval.Add(TimeSpan.FromSeconds(5)); + + LogTrace(methodName, "Receiver started"); + + try + { + while (!ct.IsCancellationRequested && IsConnected) + { + // Reset timeout + readTimeoutCts.CancelAfter(readTimeout); + + var line = await ReadDataAsync(readTimeoutCts.Token); + + if (line == null) + { + Log.Error("Remote closed connection"); + break; + } + + LogTrace(methodName, $"Received: {line}"); + } + } + catch (OperationCanceledException) + { + if (!_tcpClient.Connected) + { + LogError("DataReceiverAsync", "GG"); + } + + Log.Trace("DataBentoRawLiveClient.ProcessMessages(): Message processing cancelled"); + } + catch (IOException ex) when (ex.InnerException is SocketException) + { + Log.Trace($"DataBentoRawLiveClient.ProcessMessages(): Socket exception: {ex.Message}"); + } + catch (Exception ex) + { + Log.Error($"DataBentoRawLiveClient.ProcessMessages(): Error processing messages: {ex.Message}\n{ex.StackTrace}"); + } + finally + { + LogTrace(methodName, "Receiver stopped"); + readTimeoutCts.Dispose(); + } + } + + private async Task ReadDataAsync(CancellationToken cancellationToken) + { + var numberOfBytesToRead = await _stream.ReadAsync(_receiveBuffer.AsMemory(0, _receiveBuffer.Length), cancellationToken).ConfigureAwait(false); + + if (numberOfBytesToRead == 0) + { + return null; + } + + using var memoryStream = new MemoryStream(); + await memoryStream.WriteAsync(_receiveBuffer.AsMemory(0, numberOfBytesToRead), cancellationToken).ConfigureAwait(false); + return Encoding.ASCII.GetString(memoryStream.ToArray(), 0, numberOfBytesToRead).TrimEnd(_newLine); + } + + private void WriteData(string data) + { + if (!data.EndsWith('\n')) + { + data += '\n'; + } + var bytes = Encoding.ASCII.GetBytes(data); + _stream.Write(bytes, 0, bytes.Length); + } + + private async Task Authenticate(string dataSet) + { + var cts = CancellationTokenSource.CreateLinkedTokenSource(_cancellationTokenSource.Token); + + try + { + var versionLine = await ReadDataAsync(cts.Token); + var cramLine = await ReadDataAsync(cts.Token); + + if (Log.DebuggingEnabled) + { + LogDebug(nameof(Authenticate), $"Received initial message: {versionLine}, {cramLine}"); + } + + var request = new AuthenticationMessageRequest(cramLine, _apiKey, dataSet, _heartBeatInterval); + + LogTrace("Authenticate", $"Sending CRAM reply: {request}"); + + WriteData(request.ToString()); + + var authResponse = await ReadDataAsync(cts.Token); + + var authenticationResponse = new AuthenticationMessageResponse(authResponse); + + if (!authenticationResponse.Success) + { + LogError(nameof(Authenticate), $"Authentication response: {authResponse}"); + return false; + } + + LogTrace(nameof(Authenticate), $"Successfully authenticated with session ID: {authenticationResponse.SessionId}"); + + WriteData(request.GetStartSessionMessage()); // after start_session -> we get heartbeats and data + + return true; + } + finally + { + cts.DisposeSafely(); + } + } + + private static string DetermineGateway(string dataset) + { + dataset = dataset.Replace('.', '-').ToLowerInvariant(); + return dataset + ".lsg.databento.com"; + } + + private void LogTrace(string method, string message) + { + Log.Trace($"LiveDataTcpClientWrapper[{_dataSet}].{method}: {message}"); + } + + private void LogError(string method, string message) + { + Log.Error($"LiveDataTcpClientWrapper[{_dataSet}].{method}: {message}"); + } + + private void LogDebug(string method, string message) + { + Log.Debug($"LiveDataTcpClientWrapper[{_dataSet}].{method}: {message}"); + } +} diff --git a/QuantConnect.DataBento/DataBentoRawLiveClient.cs b/QuantConnect.DataBento/DataBentoRawLiveClient.cs index 9c65544..06d8e6a 100644 --- a/QuantConnect.DataBento/DataBentoRawLiveClient.cs +++ b/QuantConnect.DataBento/DataBentoRawLiveClient.cs @@ -261,6 +261,8 @@ public bool Unsubscribe(Symbol symbol) { try { + // Please note there is no unsubscribe method. Subscriptions end when the TCP connection closes. + if (_subscriptions.TryRemove(symbol, out _)) { Log.Debug($"DataBentoRawLiveClient.Unsubscribe(): Unsubscribed from {symbol}"); diff --git a/QuantConnect.DataBento/Models/Enums/RecordType.cs b/QuantConnect.DataBento/Models/Enums/RecordType.cs new file mode 100644 index 0000000..67226e1 --- /dev/null +++ b/QuantConnect.DataBento/Models/Enums/RecordType.cs @@ -0,0 +1,86 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Enums; + +/// +/// Record type identifier (rtype) used in market data messages. +/// +public enum RecordType : byte +{ + /// Market-by-price record with book depth 0 (trades). + MarketByPriceDepth0 = 0, + + /// Market-by-price record with book depth 1 (TBBO, MBP-1). + MarketByPriceDepth1 = 1, + + /// Market-by-price record with book depth 10. + MarketByPriceDepth10 = 10, + + /// Exchange status record. + Status = 18, + + /// Instrument definition record. + Definition = 19, + + /// Order imbalance record. + Imbalance = 20, + + /// Error record from the live gateway. + Error = 21, + + /// Symbol mapping record from the live gateway. + SymbolMapping = 22, + + /// System record from the live gateway (e.g. heartbeat). + System = 23, + + /// Statistics record from the publisher. + Statistics = 24, + + /// OHLCV record at 1-second cadence. + OpenHighLowCloseVolume1Second = 32, + + /// OHLCV record at 1-minute cadence. + OpenHighLowCloseVolume1Minute = 33, + + /// OHLCV record at hourly cadence. + OpenHighLowCloseVolume1Hour = 34, + + /// OHLCV record at daily cadence. + OpenHighLowCloseVolume1Day = 35, + + /// Market-by-order record. + MarketByOrder = 160, + + /// Consolidated market-by-price record with book depth 1. + ConsolidatedMarketByPriceDepth1 = 177, + + /// Consolidated BBO at 1-second cadence. + ConsolidatedBestBidAndOffer1Second = 192, + + /// Consolidated BBO at 1-minute cadence. + ConsolidatedBestBidAndOffer1Minute = 193, + + /// Consolidated BBO with trades only. + TradeWithConsolidatedBestBidAndOffer = 194, + + /// Market-by-price BBO at 1-second cadence. + BBO1Second = 195, + + /// Market-by-price BBO at 1-minute cadence. + BBO1Minute = 196 +} diff --git a/QuantConnect.DataBento/Models/Header.cs b/QuantConnect.DataBento/Models/Header.cs index 2a2ead1..36774a2 100644 --- a/QuantConnect.DataBento/Models/Header.cs +++ b/QuantConnect.DataBento/Models/Header.cs @@ -13,6 +13,8 @@ * limitations under the License. */ +using QuantConnect.Lean.DataSource.DataBento.Models.Enums; + namespace QuantConnect.Lean.DataSource.DataBento.Models; /// @@ -29,7 +31,7 @@ public sealed class Header /// /// Record type identifier defining the data schema (e.g. trade, quote, bar). /// - public int Rtype { get; set; } + public RecordType Rtype { get; set; } /// /// DataBento publisher (exchange / data source) identifier. diff --git a/QuantConnect.DataBento/Models/Live/AuthenticationMessageRequest.cs b/QuantConnect.DataBento/Models/Live/AuthenticationMessageRequest.cs new file mode 100644 index 0000000..79d0715 --- /dev/null +++ b/QuantConnect.DataBento/Models/Live/AuthenticationMessageRequest.cs @@ -0,0 +1,69 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; + +public readonly struct AuthenticationMessageRequest +{ + private const int BucketIdLength = 5; + + private readonly string _dataset; + + private readonly string _auth; + + private readonly TimeSpan _heartBeatInterval; + + + public AuthenticationMessageRequest(string cramLine, string apiKey, string dataSet, TimeSpan? heartBeatInterval = default) + { + _dataset = dataSet; + + var newLineIndex = cramLine.IndexOf('\n'); + if (newLineIndex >= 0) + { + cramLine = cramLine[..newLineIndex]; + } + + cramLine = cramLine[(cramLine.IndexOf('=') + 1)..]; + + var challengeKey = cramLine + '|' + apiKey; + var bucketId = apiKey[^BucketIdLength..]; + + _auth = $"{QuantConnect.Extensions.ToSHA256(challengeKey)}-{bucketId}"; + + switch (heartBeatInterval) + { + case null: + _heartBeatInterval = TimeSpan.FromSeconds(5); + break; + case { TotalSeconds: < 5 }: + throw new ArgumentOutOfRangeException(nameof(heartBeatInterval), "The Heartbeat interval must be not les 5 seconds."); + default: + _heartBeatInterval = heartBeatInterval.Value; + break; + } + } + + public override string ToString() + { + return $"auth={_auth}|dataset={_dataset}|pretty_px=1|encoding=json|heartbeat_interval_s={_heartBeatInterval.TotalSeconds}"; + } + + public string GetStartSessionMessage() + { + return "start_session"; + } +} diff --git a/QuantConnect.DataBento/Models/Live/AuthenticationMessageResponse.cs b/QuantConnect.DataBento/Models/Live/AuthenticationMessageResponse.cs new file mode 100644 index 0000000..51720a3 --- /dev/null +++ b/QuantConnect.DataBento/Models/Live/AuthenticationMessageResponse.cs @@ -0,0 +1,54 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using QuantConnect.Logging; + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; + +public readonly struct AuthenticationMessageResponse +{ + public bool Success { get; } + public string? Error { get; } + public int SessionId { get; } + + public AuthenticationMessageResponse(string input) + { + if (Log.DebuggingEnabled) + { + Log.Debug($"{nameof(AuthenticationMessageResponse)}.ctor: Authentication response: {input}"); + } + + var parts = input.Split('|', StringSplitOptions.RemoveEmptyEntries); + + foreach (var part in parts) + { + var kv = part.Split('=', 2); + + switch (kv[0]) + { + case "success": + Success = kv[1] == "1"; + break; + case "error": + Error = kv[1]; + break; + case "session_id": + SessionId = int.Parse(kv[1]); + break; + } + } + } +} diff --git a/QuantConnect.DataBento/Models/Live/HeartbeatMessage.cs b/QuantConnect.DataBento/Models/Live/HeartbeatMessage.cs new file mode 100644 index 0000000..4f1b5b7 --- /dev/null +++ b/QuantConnect.DataBento/Models/Live/HeartbeatMessage.cs @@ -0,0 +1,22 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; + +public sealed class HeartbeatMessage : MarketDataRecord +{ + public required string Msg { get; set; } +} From 2a2392f76a0d7317a20ca163c50a65eade1dba80 Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 28 Jan 2026 01:50:56 +0200 Subject: [PATCH 12/18] feat: Data Queue Handler --- .../DataBentoJsonConverterTests.cs | 98 ++- .../DataBentoLiveAPIClientTests.cs | 37 +- .../DataBentoRawLiveClientTests.cs | 146 ---- QuantConnect.DataBento/Api/LiveAPIClient.cs | 57 +- .../Api/LiveDataTcpClientWrapper.cs | 27 +- .../Converters/LiveDataConverter.cs | 82 +++ .../DataBentoDataProvider.cs | 366 +++++---- .../DataBentoRawLiveClient.cs | 696 ------------------ .../DataBentoSymbolMapper.cs | 24 +- QuantConnect.DataBento/Extensions.cs | 15 + QuantConnect.DataBento/Models/Header.cs | 4 +- QuantConnect.DataBento/Models/LevelOneData.cs | 3 +- .../SymbolMappingConfirmationEventArgs.cs | 43 ++ .../Models/Live/SymbolMappingMessage.cs | 30 + .../Serialization/JsonSettings.cs | 11 + 15 files changed, 584 insertions(+), 1055 deletions(-) delete mode 100644 QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs create mode 100644 QuantConnect.DataBento/Converters/LiveDataConverter.cs delete mode 100644 QuantConnect.DataBento/DataBentoRawLiveClient.cs create mode 100644 QuantConnect.DataBento/Models/Live/SymbolMappingConfirmationEventArgs.cs create mode 100644 QuantConnect.DataBento/Models/Live/SymbolMappingMessage.cs diff --git a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs index 5b9de56..d29d488 100644 --- a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs @@ -44,7 +44,7 @@ public void DeserializeHistoricalOhlcvBar() Assert.IsNotNull(res); Assert.AreEqual(1738281600000000000m, res.Header.TsEvent); - Assert.AreEqual(35, res.Header.Rtype); + Assert.AreEqual(RecordType.OpenHighLowCloseVolume1Day, res.Header.Rtype); Assert.AreEqual(1, res.Header.PublisherId); Assert.AreEqual(42140878, res.Header.InstrumentId); @@ -92,7 +92,7 @@ public void DeserializeHistoricalLevelOneData() Assert.AreEqual(1768137063449660443, res.TsRecv); Assert.AreEqual(1768137063107829777, res.Header.TsEvent); - Assert.AreEqual(1, res.Header.Rtype); + Assert.AreEqual(RecordType.MarketByPriceDepth1, res.Header.Rtype); Assert.AreEqual(1, res.Header.PublisherId); Assert.AreEqual(42140878, res.Header.InstrumentId); @@ -141,7 +141,7 @@ public void DeserializeHistoricalStatisticsData() Assert.AreEqual(1768156232522476283, res.Header.TsEvent); - Assert.AreEqual(24, res.Header.Rtype); + Assert.AreEqual(RecordType.Statistics, res.Header.Rtype); Assert.AreEqual(1, res.Header.PublisherId); Assert.AreEqual(42566722, res.Header.InstrumentId); Assert.AreEqual(470m, res.Quantity); @@ -192,4 +192,96 @@ public void ParsePotentialCramChallenges(string challenge, string expectedString Assert.AreEqual(expectedString, actualString); } + + [Test] + public void DeserializeSymbolMappingMessage() + { + var json = @"{ + ""hd"": { + ""ts_event"": ""1769546804979770503"", + ""rtype"": 22, + ""publisher_id"": 0, + ""instrument_id"": 42140878 + }, + ""stype_in_symbol"": ""ESH6"", + ""stype_out_symbol"": ""ESH6"", + ""start_ts"": ""18446744073709551615"", + ""end_ts"": ""18446744073709551615"" +}"; + + var marketData = json.DeserializeSnakeCaseLiveData(); + + Assert.IsNotNull(marketData); + Assert.AreEqual(1769546804979770503, marketData.Header.TsEvent); + Assert.AreEqual(RecordType.SymbolMapping, marketData.Header.Rtype); + Assert.AreEqual(0, marketData.Header.PublisherId); + Assert.AreEqual(42140878, marketData.Header.InstrumentId); + + Assert.IsInstanceOf(marketData); + + var sm = marketData as SymbolMappingMessage; + + Assert.AreEqual("ESH6", sm.StypeInSymbol); + Assert.AreEqual("ESH6", sm.StypeOutSymbol); + } + + + [Test] + public void DeserializeMarketByPriceMessage() + { + var json = @" +{ + ""ts_recv"": ""1769546804990938439"", + ""hd"": { + ""ts_event"": ""1769546804990833083"", + ""rtype"": 1, + ""publisher_id"": 1, + ""instrument_id"": 42005017 + }, + ""action"": ""A"", + ""side"": ""A"", + ""depth"": 0, + ""price"": ""2676.400000000"", + ""size"": 1, + ""flags"": 128, + ""ts_in_delta"": 17695, + ""sequence"": 126257483, + ""levels"": [ + { + ""bid_px"": ""2676.300000000"", + ""ask_px"": ""2676.400000000"", + ""bid_sz"": 14, + ""ask_sz"": 2, + ""bid_ct"": 8, + ""ask_ct"": 2 + } + ] +}"; + var marketData = json.DeserializeSnakeCaseLiveData(); + + Assert.IsNotNull(marketData); + Assert.AreEqual(1769546804990833083, marketData.Header.TsEvent); + Assert.AreEqual(RecordType.MarketByPriceDepth1, marketData.Header.Rtype); + Assert.AreEqual(1, marketData.Header.PublisherId); + Assert.AreEqual(42005017, marketData.Header.InstrumentId); + + Assert.IsInstanceOf(marketData); + + var mbp = marketData as LevelOneData; + Assert.AreEqual('A', mbp.Action); + Assert.AreEqual('A', mbp.Side); + Assert.AreEqual(0, mbp.Depth); + Assert.AreEqual(2676.4m, mbp.Price); + Assert.AreEqual(1, mbp.Size); + Assert.AreEqual(128, mbp.Flags); + Assert.IsNotNull(mbp.Levels); + Assert.AreEqual(1, mbp.Levels.Count); + var level = mbp.Levels[0]; + Assert.AreEqual(2676.3m, level.BidPx); + Assert.AreEqual(2676.4m, level.AskPx); + Assert.AreEqual(14, level.BidSz); + Assert.AreEqual(2, level.AskSz); + Assert.AreEqual(8, level.BidCt); + Assert.AreEqual(2, level.AskCt); + } } diff --git a/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs index 1124895..ea66096 100644 --- a/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs @@ -17,6 +17,7 @@ using NUnit.Framework; using System.Threading; using QuantConnect.Configuration; +using System.Collections.Generic; using QuantConnect.Lean.DataSource.DataBento.Api; namespace QuantConnect.Lean.DataSource.DataBento.Tests; @@ -44,7 +45,7 @@ public void OneTimeSetUp() Assert.Inconclusive("Please set the 'databento-api-key' in your configuration to enable these tests."); } - _live = new LiveAPIClient(apiKey); + _live = new LiveAPIClient(apiKey, null); } [OneTimeTearDown] @@ -54,12 +55,40 @@ public void OneTimeTearDown() } [Test] - public void TestExample() + public void ShouldReceiveSymbolMappingConfirmation() { var dataAvailableEvent = new AutoResetEvent(false); - _live.Start(Dataset); + var subs = new Dictionary() + { + { Securities.Futures.Indices.SP500EMini + "H6", 0 }, + { Securities.Futures.Indices.Russell2000EMini + "H6", 0 } + }; + + void OnSymbolMappingConfirmation(object sender, Models.Live.SymbolMappingConfirmationEventArgs e) + { + if (subs.ContainsKey(e.Symbol)) + { + subs[e.Symbol] = e.InstrumentId; + dataAvailableEvent.Set(); + } + } + + _live.SymbolMappingConfirmation += OnSymbolMappingConfirmation; + + foreach (var s in subs.Keys) + { + _live.Subscribe(Dataset, s); + dataAvailableEvent.WaitOne(TimeSpan.FromSeconds(5)); + } + + dataAvailableEvent.WaitOne(TimeSpan.FromSeconds(1)); + + foreach (var instrumentId in subs.Values) + { + Assert.Greater(instrumentId, 0); + } - dataAvailableEvent.WaitOne(TimeSpan.FromSeconds(60)); + _live.SymbolMappingConfirmation -= OnSymbolMappingConfirmation; } } diff --git a/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs deleted file mode 100644 index ec03477..0000000 --- a/QuantConnect.DataBento.Tests/DataBentoRawLiveClientTests.cs +++ /dev/null @@ -1,146 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System; -using NUnit.Framework; -using System.Threading; -using QuantConnect.Data; -using QuantConnect.Logging; -using QuantConnect.Data.Market; -using QuantConnect.Configuration; - -namespace QuantConnect.Lean.DataSource.DataBento.Tests; - -[TestFixture] -public class DataBentoRawLiveClientSyncTests -{ - private DataBentoRawLiveClient _client; - protected readonly string ApiKey = Config.Get("databento-api-key"); - - private static Symbol CreateEsFuture() - { - var expiration = new DateTime(2026, 3, 20); - return Symbol.CreateFuture("ES", Market.CME, expiration); - } - - [SetUp] - public void SetUp() - { - Log.Trace("DataBentoLiveClientTests: Using API Key: " + ApiKey); - _client = new DataBentoRawLiveClient(ApiKey); - } - - [TearDown] - public void TearDown() - { - _client?.Dispose(); - } - - [Test] - public void Connects() - { - var connected = _client.Connect(); - - Assert.IsTrue(connected); - Assert.IsTrue(_client.IsConnected); - - Log.Trace("Connected successfully"); - } - - [Test] - public void SubscribesToLeanFutureSymbol() - { - Assert.IsTrue(_client.Connect()); - - var symbol = CreateEsFuture(); - - Assert.IsTrue(_client.Subscribe(symbol, TickType.Trade)); - Assert.IsTrue(_client.StartSession()); - - Thread.Sleep(1000); - - Assert.IsTrue(_client.Unsubscribe(symbol)); - } - - [Test] - public void ReceivesTradeOrQuoteTicks() - { - var receivedEvent = new ManualResetEventSlim(false); - BaseData received = null; - - _client.DataReceived += (_, data) => - { - received = data; - receivedEvent.Set(); - }; - - Assert.IsTrue(_client.Connect()); - - var symbol = CreateEsFuture(); - - Assert.IsTrue(_client.Subscribe(symbol, TickType.Trade)); - Assert.IsTrue(_client.StartSession()); - - var gotData = receivedEvent.Wait(TimeSpan.FromMinutes(2)); - - if (!gotData) - { - Assert.Inconclusive("No data received (likely outside market hours)"); - return; - } - - Assert.NotNull(received); - Assert.AreEqual(symbol, received.Symbol); - - if (received is Tick tick) - { - Assert.Greater(tick.Time, DateTime.MinValue); - Assert.Greater(tick.Value, 0); - } - else if (received is TradeBar bar) - { - Assert.Greater(bar.Close, 0); - } - else - { - Assert.Fail($"Unexpected data type: {received.GetType()}"); - } - } - - [Test] - public void DisposeIsIdempotent() - { - var client = new DataBentoRawLiveClient(ApiKey); - Assert.DoesNotThrow(client.Dispose); - Assert.DoesNotThrow(client.Dispose); - } - - [Test] - public void SymbolMappingDoesNotThrow() - { - Assert.IsTrue(_client.Connect()); - - var symbol = CreateEsFuture(); - - Assert.DoesNotThrow(() => - { - _client.Subscribe(symbol, TickType.Trade); - _client.StartSession(); - Thread.Sleep(500); - _client.Unsubscribe(symbol); - }); - } -} diff --git a/QuantConnect.DataBento/Api/LiveAPIClient.cs b/QuantConnect.DataBento/Api/LiveAPIClient.cs index f8b60df..810dd9b 100644 --- a/QuantConnect.DataBento/Api/LiveAPIClient.cs +++ b/QuantConnect.DataBento/Api/LiveAPIClient.cs @@ -16,6 +16,8 @@ using QuantConnect.Util; using QuantConnect.Logging; +using QuantConnect.Lean.DataSource.DataBento.Models; +using QuantConnect.Lean.DataSource.DataBento.Models.Live; namespace QuantConnect.Lean.DataSource.DataBento.Api; @@ -25,9 +27,16 @@ public sealed class LiveAPIClient : IDisposable private readonly Dictionary _tcpClientByDataSet = []; - public LiveAPIClient(string apiKey) + private readonly Action _levelOneDataHandler; + + public event EventHandler? SymbolMappingConfirmation; + + public bool IsConnected => _tcpClientByDataSet.Values.All(c => c.IsConnected); + + public LiveAPIClient(string apiKey, Action levelOneDataHandler) { _apiKey = apiKey; + _levelOneDataHandler = levelOneDataHandler; } public void Dispose() @@ -39,36 +48,52 @@ public void Dispose() _tcpClientByDataSet.Clear(); } - public bool Start(string dataSet) + private LiveDataTcpClientWrapper EnsureDatasetConnection(string dataSet) { - LogTrace(nameof(Start), "Starting connection to DataBento live API"); - - if (_tcpClientByDataSet.TryGetValue(dataSet, out var existingClient) && existingClient.IsConnected) + if (_tcpClientByDataSet.TryGetValue(dataSet, out var liveDataTcpClient)) { - LogTrace(nameof(Start), $"Already connected to DataBento live API (Dataset: {dataSet})"); - return true; + return liveDataTcpClient; } - var liveDataTcpClient = new LiveDataTcpClientWrapper(dataSet, _apiKey); + LogTrace(nameof(EnsureDatasetConnection), "Starting connection to DataBento live API"); + + liveDataTcpClient = new LiveDataTcpClientWrapper(dataSet, _apiKey, MessageReceived); _tcpClientByDataSet[dataSet] = liveDataTcpClient; liveDataTcpClient.Connect(); - LogTrace(nameof(Start), $"Successfully connected to DataBento live API (Dataset: {dataSet})"); + LogTrace(nameof(EnsureDatasetConnection), $"Successfully connected to DataBento live API (Dataset: {dataSet})"); - return true; + return liveDataTcpClient; } public bool Subscribe(string dataSet, string symbol) { - if (!_tcpClientByDataSet.TryGetValue(dataSet, out var tcpClient) || !tcpClient.IsConnected) + EnsureDatasetConnection(dataSet).SubscribeOnMarketBestPriceLevelOne(symbol); + return true; + } + + private void MessageReceived(string message) + { + var data = message.DeserializeSnakeCaseLiveData(); + + if (data == null) { - LogError(nameof(Subscribe), $"Not connected to DataBento live API (Dataset: {dataSet})"); - return false; + LogError(nameof(MessageReceived), $"Failed to deserialize live data message: {message}"); + return; } - tcpClient.SubscribeOnMarketBestPriceLevelOne(symbol); - - return true; + switch (data) + { + case SymbolMappingMessage smm: + SymbolMappingConfirmation?.Invoke(this, new(smm.StypeInSymbol, smm.Header.InstrumentId)); + break; + case LevelOneData lod: + _levelOneDataHandler?.Invoke(lod); + break; + default: + LogError(nameof(MessageReceived), $"Received unsupported record type: {data.Header.Rtype}. Message: {message}"); + break; + } } private static void LogTrace(string method, string message) diff --git a/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs b/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs index 205ff78..b55e14a 100644 --- a/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs +++ b/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs @@ -25,7 +25,6 @@ namespace QuantConnect.Lean.DataSource.DataBento.Api; public sealed class LiveDataTcpClientWrapper : IDisposable { private const int DefaultPort = 13000; - private const int ReceiveBufferSize = 8192; private readonly string _gateway; private readonly string _dataSet; @@ -33,30 +32,33 @@ public sealed class LiveDataTcpClientWrapper : IDisposable private readonly TimeSpan _heartBeatInterval = TimeSpan.FromSeconds(10); private readonly TcpClient _tcpClient = new(); - private readonly byte[] _receiveBuffer = new byte[ReceiveBufferSize]; private readonly CancellationTokenSource _cancellationTokenSource = new(); - private readonly char[] _newLine = Environment.NewLine.ToCharArray(); private NetworkStream? _stream; + private StreamReader? _reader; private Task? _dataReceiverTask; private bool _isConnected; + private readonly Action MessageReceived; + /// /// Is client connected /// public bool IsConnected => _isConnected; - public LiveDataTcpClientWrapper(string dataSet, string apiKey) + public LiveDataTcpClientWrapper(string dataSet, string apiKey, Action messageReceived) { _apiKey = apiKey; _dataSet = dataSet; _gateway = DetermineGateway(dataSet); + MessageReceived = messageReceived; } public void Connect() { _tcpClient.Connect(_gateway, DefaultPort); _stream = _tcpClient.GetStream(); + _reader = new StreamReader(_stream, Encoding.ASCII); if (!Authenticate(_dataSet).SynchronouslyAwaitTask()) throw new Exception("Authentication failed"); @@ -71,8 +73,8 @@ public void Dispose() { _isConnected = false; - _stream?.Close(); - _stream?.DisposeSafely(); + _reader?.Close(); + _reader?.DisposeSafely(); _cancellationTokenSource?.Cancel(); _cancellationTokenSource?.DisposeSafely(); @@ -113,7 +115,7 @@ private async Task DataReceiverAsync(CancellationToken ct) break; } - LogTrace(methodName, $"Received: {line}"); + MessageReceived.Invoke(line); } } catch (OperationCanceledException) @@ -142,16 +144,7 @@ private async Task DataReceiverAsync(CancellationToken ct) private async Task ReadDataAsync(CancellationToken cancellationToken) { - var numberOfBytesToRead = await _stream.ReadAsync(_receiveBuffer.AsMemory(0, _receiveBuffer.Length), cancellationToken).ConfigureAwait(false); - - if (numberOfBytesToRead == 0) - { - return null; - } - - using var memoryStream = new MemoryStream(); - await memoryStream.WriteAsync(_receiveBuffer.AsMemory(0, numberOfBytesToRead), cancellationToken).ConfigureAwait(false); - return Encoding.ASCII.GetString(memoryStream.ToArray(), 0, numberOfBytesToRead).TrimEnd(_newLine); + return await _reader.ReadLineAsync(cancellationToken); } private void WriteData(string data) diff --git a/QuantConnect.DataBento/Converters/LiveDataConverter.cs b/QuantConnect.DataBento/Converters/LiveDataConverter.cs new file mode 100644 index 0000000..395706f --- /dev/null +++ b/QuantConnect.DataBento/Converters/LiveDataConverter.cs @@ -0,0 +1,82 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using QuantConnect.Lean.DataSource.DataBento.Models; +using QuantConnect.Lean.DataSource.DataBento.Models.Enums; +using QuantConnect.Lean.DataSource.DataBento.Models.Live; + +namespace QuantConnect.Lean.DataSource.DataBento.Converters; + +public class LiveDataConverter : JsonConverter +{ + private const string headerIdentifier = "hd"; + + private const string recordTypeIdentifier = "rtype"; + + private static JsonSerializer _snakeSerializer = new JsonSerializer + { + ContractResolver = Serialization.SnakeCaseContractResolver.Instance + }; + + /// + /// Gets a value indicating whether this can write JSON. + /// + /// true if this can write JSON; otherwise, false. + public override bool CanWrite => false; + + /// + /// Gets a value indicating whether this can read JSON. + /// + /// true if this can read JSON; otherwise, false. + public override bool CanRead => true; + + /// + /// Reads the JSON representation of the object. + /// + /// The to read from. + /// Type of the object. + /// The existing property value of the JSON that is being converted. + /// The calling serializer. + /// The object value. + public override MarketDataRecord ReadJson(JsonReader reader, Type objectType, MarketDataRecord? existingValue, bool hasExistingValue, JsonSerializer serializer) + { + var jObject = JObject.Load(reader); + + var recordType = jObject[headerIdentifier]?[recordTypeIdentifier]?.ToObject(); + + switch (recordType) + { + case RecordType.SymbolMapping: + return jObject.ToObject(_snakeSerializer); + case RecordType.MarketByPriceDepth1: + return jObject.ToObject(_snakeSerializer); + default: + return null; + } + } + + /// + /// Writes the JSON representation of the object. + /// + /// The to write to. + /// The value. + /// The calling serializer. + public override void WriteJson(JsonWriter writer, MarketDataRecord? value, JsonSerializer serializer) + { + throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/QuantConnect.DataBento/DataBentoDataProvider.cs b/QuantConnect.DataBento/DataBentoDataProvider.cs index ba228d1..5127a45 100644 --- a/QuantConnect.DataBento/DataBentoDataProvider.cs +++ b/QuantConnect.DataBento/DataBentoDataProvider.cs @@ -14,17 +14,24 @@ * */ -using NodaTime; -using QuantConnect.Data; +using System.Net; +using System.Text; +using Newtonsoft.Json; +using QuantConnect.Api; using QuantConnect.Util; +using QuantConnect.Data; +using Newtonsoft.Json.Linq; using QuantConnect.Logging; using QuantConnect.Packets; using QuantConnect.Interfaces; -using QuantConnect.Securities; -using QuantConnect.Data.Market; using QuantConnect.Configuration; +using System.Security.Cryptography; +using System.Net.NetworkInformation; using System.Collections.Concurrent; +using QuantConnect.Brokerages.LevelOneOrderBook; using QuantConnect.Lean.DataSource.DataBento.Api; +using QuantConnect.Lean.DataSource.DataBento.Models; +using QuantConnect.Lean.DataSource.DataBento.Models.Live; namespace QuantConnect.Lean.DataSource.DataBento; @@ -44,21 +51,27 @@ public partial class DataBentoProvider : IDataQueueHandler private readonly DataBentoSymbolMapper _symbolMapper = new(); - private readonly IDataAggregator _dataAggregator = Composer.Instance.GetExportedValueByTypeName( - Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"), forceTypeNameOnExisting: false); - private EventBasedDataQueueHandlerSubscriptionManager _subscriptionManager; - private DataBentoRawLiveClient _client; - private bool _potentialUnsupportedResolutionMessageLogged; - private bool _sessionStarted = false; - private readonly object _sessionLock = new(); - private readonly MarketHoursDatabase _marketHoursDatabase = MarketHoursDatabase.FromDataFolder(); - private readonly ConcurrentDictionary _symbolExchangeTimeZones = new(); + private readonly ConcurrentDictionary _pendingSubscriptions = []; + + private readonly Dictionary _subscribedSymbolsByDataBentoInstrumentId = []; + + /// + /// Manages Level 1 market data subscriptions and routing of updates to the shared . + /// Responsible for tracking and updating individual instances per symbol. + /// + private LevelOneServiceManager _levelOneServiceManager; + + private IDataAggregator _aggregator; + + private LiveAPIClient _liveApiClient; + private bool _initialized; /// /// Returns true if we're currently connected to the Data Provider /// - public bool IsConnected => _client?.IsConnected == true; + public bool IsConnected => _liveApiClient.IsConnected; + /// /// Initializes a new instance of the DataBentoProvider @@ -86,74 +99,50 @@ public DataBentoProvider(string apiKey) /// private void Initialize(string apiKey) { - Log.Debug("DataBentoProvider.Initialize(): Starting initialization"); - _subscriptionManager = new EventBasedDataQueueHandlerSubscriptionManager() - { - SubscribeImpl = (symbols, tickType) => - { - return SubscriptionLogic(symbols, tickType); - }, - UnsubscribeImpl = (symbols, tickType) => - { - return UnsubscribeLogic(symbols, tickType); - } - }; - - // Initialize the live client - _client = new DataBentoRawLiveClient(apiKey); - _client.DataReceived += OnDataReceived; + ValidateSubscription(); - // Connect to live gateway - Log.Debug("DataBentoProvider.Initialize(): Attempting connection to DataBento live gateway"); - var cancellationTokenSource = new CancellationTokenSource(); - Task.Factory.StartNew(() => + _aggregator = Composer.Instance.GetPart(); + if (_aggregator == null) { - try - { - var connected = _client.Connect(); - Log.Debug($"DataBentoProvider.Initialize(): Connect() returned {connected}"); + var aggregatorName = Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"); + Log.Trace($"{nameof(DataBentoProvider)}.{nameof(Initialize)}: found no data aggregator instance, creating {aggregatorName}"); + _aggregator = Composer.Instance.GetExportedValueByTypeName(aggregatorName); + } - if (connected) - { - Log.Debug("DataBentoProvider.Initialize(): Successfully connected to DataBento live gateway"); - } - else - { - Log.Error("DataBentoProvider.Initialize(): Failed to connect to DataBento live gateway"); - } - } - catch (Exception ex) - { - Log.Error($"DataBentoProvider.Initialize(): Exception during Connect(): {ex.Message}\n{ex.StackTrace}"); - } - }, - cancellationTokenSource.Token, - TaskCreationOptions.LongRunning, - TaskScheduler.Default); + _liveApiClient = new LiveAPIClient(apiKey, HandleLevelOneData); + _liveApiClient.SymbolMappingConfirmation += OnSymbolMappingConfirmation; _historicalApiClient = new(apiKey); + + _levelOneServiceManager = new LevelOneServiceManager( + _aggregator, + (symbols, _) => Subscribe(symbols), + (symbols, _) => Unsubscribe(symbols)); + _initialized = true; + } - Log.Debug("DataBentoProvider.Initialize(): Initialization complete"); + private void OnSymbolMappingConfirmation(object? _, SymbolMappingConfirmationEventArgs smce) + { + if (_pendingSubscriptions.TryRemove(smce.Symbol, out var symbol)) + { + _subscribedSymbolsByDataBentoInstrumentId[smce.InstrumentId] = symbol; + } } - /// - /// Logic to unsubscribe from the specified symbols - /// - public bool UnsubscribeLogic(IEnumerable symbols, TickType tickType) + private void HandleLevelOneData(LevelOneData levelOneData) { - foreach (var symbol in symbols) + if (_subscribedSymbolsByDataBentoInstrumentId.TryGetValue(levelOneData.Header.InstrumentId, out var symbol)) { - Log.Debug($"DataBentoProvider.UnsubscribeImpl(): Processing symbol {symbol}"); - if (_client?.IsConnected != true) + var time = levelOneData.Header.UtcTime; + + _levelOneServiceManager.HandleLastTrade(symbol, time, levelOneData.Size, levelOneData.Price); + + foreach (var l in levelOneData.Levels) { - throw new InvalidOperationException($"DataBentoProvider.UnsubscribeImpl(): Client is not connected. Cannot unsubscribe from {symbol}"); + _levelOneServiceManager.HandleQuote(symbol, time, l.BidPx, l.BidSz, l.AskPx, l.AskSz); } - - _client.Unsubscribe(symbol); } - - return true; } /// @@ -174,23 +163,37 @@ private bool TryGetDataBentoDataSet(Symbol symbol, out string? dataSet) /// /// Logic to subscribe to the specified symbols /// - public bool SubscriptionLogic(IEnumerable symbols, TickType tickType) + public bool Subscribe(IEnumerable symbols) { - if (_client?.IsConnected != true) + foreach (var symbol in symbols) { - Log.Error("DataBentoProvider.SubscriptionLogic(): Client is not connected. Cannot subscribe to symbols"); - return false; + if (!TryGetDataBentoDataSet(symbol, out var dataSet)) + { + throw new ArgumentException($"No DataBento dataset mapping found for symbol {symbol} in market {symbol.ID.Market}. Cannot subscribe."); + } + + var brokerageSymbol = _symbolMapper.GetBrokerageSymbol(symbol); + + _pendingSubscriptions[brokerageSymbol] = symbol; + + _liveApiClient.Subscribe(dataSet, brokerageSymbol); } - foreach (var symbol in symbols) + return true; + } + + public bool Unsubscribe(IEnumerable symbols) + { + // Please note there is no unsubscribe method. Subscriptions end when the TCP connection closes. + + var symbolsToRemove = symbols.ToHashSet(); + + foreach (var (instrumentId, symbol) in _subscribedSymbolsByDataBentoInstrumentId) { - if (!CanSubscribe(symbol)) + if (symbolsToRemove.Contains(symbol)) { - Log.Error($"DataBentoProvider.SubscriptionLogic(): Unsupported subscription: {symbol}"); - return false; + _subscribedSymbolsByDataBentoInstrumentId.Remove(instrumentId); } - - _client.Subscribe(symbol, tickType); } return true; @@ -216,17 +219,13 @@ private static bool CanSubscribe(Symbol symbol) /// The new enumerator for this subscription request public IEnumerator? Subscribe(SubscriptionDataConfig dataConfig, EventHandler newDataAvailableHandler) { - lock (_sessionLock) + if (!CanSubscribe(dataConfig.Symbol)) { - if (!_sessionStarted) - { - Log.Debug("DataBentoProvider.SubscriptionLogic(): Starting session"); - _sessionStarted = _client.StartSession(); - } + return null; } - var enumerator = _dataAggregator.Add(dataConfig, newDataAvailableHandler); - _subscriptionManager.Subscribe(dataConfig); + var enumerator = _aggregator.Add(dataConfig, newDataAvailableHandler); + _levelOneServiceManager.Subscribe(dataConfig); return enumerator; } @@ -237,9 +236,8 @@ private static bool CanSubscribe(Symbol symbol) /// Subscription config to be removed public void Unsubscribe(SubscriptionDataConfig dataConfig) { - Log.Debug($"DataBentoProvider.Unsubscribe(): Received unsubscription request for {dataConfig.Symbol}, Resolution={dataConfig.Resolution}, TickType={dataConfig.TickType}"); - _subscriptionManager.Unsubscribe(dataConfig); - _dataAggregator.Remove(dataConfig); + _levelOneServiceManager.Unsubscribe(dataConfig); + _aggregator.Remove(dataConfig); } /// @@ -252,6 +250,13 @@ public void SetJob(LiveNodePacket job) { return; } + + if (!job.BrokerageData.TryGetValue("databento-api-key", out var apiKey) || string.IsNullOrWhiteSpace(apiKey)) + { + throw new ArgumentException("The DataBento API key is missing from the brokerage data."); + } + + Initialize(apiKey); } /// @@ -259,81 +264,150 @@ public void SetJob(LiveNodePacket job) /// public void Dispose() { - _dataAggregator?.DisposeSafely(); - _subscriptionManager?.DisposeSafely(); - _client?.DisposeSafely(); + _levelOneServiceManager?.DisposeSafely(); + _aggregator?.DisposeSafely(); + _liveApiClient?.DisposeSafely(); + _historicalApiClient?.DisposeSafely(); } - /// - /// Converts the given UTC time into the symbol security exchange time zone - /// - private DateTime GetTickTime(Symbol symbol, DateTime utcTime) + private class ModulesReadLicenseRead : RestResponse { - DateTimeZone exchangeTimeZone; - lock (_symbolExchangeTimeZones) - { - if (!_symbolExchangeTimeZones.TryGetValue(symbol, out exchangeTimeZone)) - { - // read the exchange time zone from market-hours-database - if (_marketHoursDatabase.TryGetEntry(symbol.ID.Market, symbol, symbol.SecurityType, out var entry)) - { - exchangeTimeZone = entry.ExchangeHours.TimeZone; - } - // If there is no entry for the given Symbol, default to New York - else - { - exchangeTimeZone = TimeZones.NewYork; - } + [JsonProperty(PropertyName = "license")] + public string License; - _symbolExchangeTimeZones[symbol] = exchangeTimeZone; - } - } - - return utcTime.ConvertFromUtc(exchangeTimeZone); + [JsonProperty(PropertyName = "organizationId")] + public string OrganizationId; } /// - /// Handles data received from the live client + /// Validate the user of this project has permission to be using it via our web API. /// - private void OnDataReceived(object _, BaseData data) + private static void ValidateSubscription() { try { - switch (data) + const int productId = 306; + var userId = Globals.UserId; + var token = Globals.UserToken; + var organizationId = Globals.OrganizationID; + // Verify we can authenticate with this user and token + var api = new ApiConnection(userId, token); + if (!api.Connected) { - case Tick tick: - tick.Time = GetTickTime(tick.Symbol, tick.Time); - lock (_dataAggregator) - { - _dataAggregator.Update(tick); - } - // Log.Trace($"DataBentoProvider.OnDataReceived(): Updated tick - Symbol: {tick.Symbol}, " + - // $"TickType: {tick.TickType}, Price: {tick.Value}, Quantity: {tick.Quantity}"); - break; - - case TradeBar tradeBar: - tradeBar.Time = GetTickTime(tradeBar.Symbol, tradeBar.Time); - tradeBar.EndTime = GetTickTime(tradeBar.Symbol, tradeBar.EndTime); - lock (_dataAggregator) + throw new ArgumentException("Invalid api user id or token, cannot authenticate subscription."); + } + // Compile the information we want to send when validating + var information = new Dictionary() + { + {"productId", productId}, + {"machineName", Environment.MachineName}, + {"userName", Environment.UserName}, + {"domainName", Environment.UserDomainName}, + {"os", Environment.OSVersion} + }; + // IP and Mac Address Information + try + { + var interfaceDictionary = new List>(); + foreach (var nic in NetworkInterface.GetAllNetworkInterfaces().Where(nic => nic.OperationalStatus == OperationalStatus.Up)) + { + var interfaceInformation = new Dictionary(); + // Get UnicastAddresses + var addresses = nic.GetIPProperties().UnicastAddresses + .Select(uniAddress => uniAddress.Address) + .Where(address => !IPAddress.IsLoopback(address)).Select(x => x.ToString()); + // If this interface has non-loopback addresses, we will include it + if (!addresses.IsNullOrEmpty()) { - _dataAggregator.Update(tradeBar); + interfaceInformation.Add("unicastAddresses", addresses); + // Get MAC address + interfaceInformation.Add("MAC", nic.GetPhysicalAddress().ToString()); + // Add Interface name + interfaceInformation.Add("name", nic.Name); + // Add these to our dictionary + interfaceDictionary.Add(interfaceInformation); } - // Log.Trace($"DataBentoProvider.OnDataReceived(): Updated TradeBar - Symbol: {tradeBar.Symbol}, " + - // $"O:{tradeBar.Open} H:{tradeBar.High} L:{tradeBar.Low} C:{tradeBar.Close} V:{tradeBar.Volume}"); - break; + } + information.Add("networkInterfaces", interfaceDictionary); + } + catch (Exception) + { + // NOP, not necessary to crash if fails to extract and add this information + } + // Include our OrganizationId if specified + if (!string.IsNullOrEmpty(organizationId)) + { + information.Add("organizationId", organizationId); + } - default: - data.Time = GetTickTime(data.Symbol, data.Time); - lock (_dataAggregator) - { - _dataAggregator.Update(data); - } - break; + // Create HTTP request + using var request = ApiUtils.CreateJsonPostRequest("modules/license/read", information); + + api.TryRequest(request, out ModulesReadLicenseRead result); + if (!result.Success) + { + throw new InvalidOperationException($"Request for subscriptions from web failed, Response Errors : {string.Join(',', result.Errors)}"); + } + + var encryptedData = result.License; + // Decrypt the data we received + DateTime? expirationDate = null; + long? stamp = null; + bool? isValid = null; + if (encryptedData != null) + { + // Fetch the org id from the response if it was not set, we need it to generate our validation key + if (string.IsNullOrEmpty(organizationId)) + { + organizationId = result.OrganizationId; + } + // Create our combination key + var password = $"{token}-{organizationId}"; + var key = SHA256.HashData(Encoding.UTF8.GetBytes(password)); + // Split the data + var info = encryptedData.Split("::"); + var buffer = Convert.FromBase64String(info[0]); + var iv = Convert.FromBase64String(info[1]); + // Decrypt our information + using var aes = new AesManaged(); + var decryptor = aes.CreateDecryptor(key, iv); + using var memoryStream = new MemoryStream(buffer); + using var cryptoStream = new CryptoStream(memoryStream, decryptor, CryptoStreamMode.Read); + using var streamReader = new StreamReader(cryptoStream); + var decryptedData = streamReader.ReadToEnd(); + if (!decryptedData.IsNullOrEmpty()) + { + var jsonInfo = JsonConvert.DeserializeObject(decryptedData); + expirationDate = jsonInfo["expiration"]?.Value(); + isValid = jsonInfo["isValid"]?.Value(); + stamp = jsonInfo["stamped"]?.Value(); + } + } + // Validate our conditions + if (!expirationDate.HasValue || !isValid.HasValue || !stamp.HasValue) + { + throw new InvalidOperationException("Failed to validate subscription."); + } + + var nowUtc = DateTime.UtcNow; + var timeSpan = nowUtc - Time.UnixTimeStampToDateTime(stamp.Value); + if (timeSpan > TimeSpan.FromHours(12)) + { + throw new InvalidOperationException("Invalid API response."); + } + if (!isValid.Value) + { + throw new ArgumentException($"Your subscription is not valid, please check your product subscriptions on our website."); + } + if (expirationDate < nowUtc) + { + throw new ArgumentException($"Your subscription expired {expirationDate}, please renew in order to use this product."); } } - catch (Exception ex) + catch (Exception e) { - Log.Error($"DataBentoProvider.OnDataReceived(): Error updating data aggregator: {ex.Message}\n{ex.StackTrace}"); + Log.Error($"PolygonDataProvider.ValidateSubscription(): Failed during validation, shutting down. Error : {e.Message}"); + throw; } } } diff --git a/QuantConnect.DataBento/DataBentoRawLiveClient.cs b/QuantConnect.DataBento/DataBentoRawLiveClient.cs deleted file mode 100644 index 06d8e6a..0000000 --- a/QuantConnect.DataBento/DataBentoRawLiveClient.cs +++ /dev/null @@ -1,696 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ - -using System.Text; -using System.Net.Sockets; -using System.Security.Cryptography; -using System.Collections.Concurrent; -using System.Text.Json; -using QuantConnect.Data; -using QuantConnect.Data.Market; -using QuantConnect.Logging; - -namespace QuantConnect.Lean.DataSource.DataBento; - -/// -/// DataBento Raw TCP client for live streaming data -/// -public class DataBentoRawLiveClient : IDisposable -{ - /// - /// The DataBento API key for authentication - /// - private readonly string _apiKey; - /// - /// The DataBento live gateway address to receive data from - /// - private const string _gateway = "glbx-mdp3.lsg.databento.com:13000"; - /// - /// The dataset to subscribe to - /// - private readonly string _dataset; - private readonly TcpClient? _tcpClient; - private readonly string _host; - private readonly int _port; - private NetworkStream? _stream; - private StreamReader _reader; - private StreamWriter _writer; - private readonly CancellationTokenSource _cancellationTokenSource; - private readonly ConcurrentDictionary _subscriptions; - private readonly object _connectionLock = new object(); - private bool _isConnected; - private bool _disposed; - private const decimal PriceScaleFactor = 1e-9m; - private readonly ConcurrentDictionary _instrumentIdToSymbol = new ConcurrentDictionary(); - private readonly DataBentoSymbolMapper _symbolMapper; - - /// - /// Event fired when new data is received - /// - public event EventHandler DataReceived; - - /// - /// Event fired when connection status changes - /// - public event EventHandler ConnectionStatusChanged; - - /// - /// Gets whether the client is currently connected - /// - public bool IsConnected => _isConnected && _tcpClient?.Connected == true; - - /// - /// Initializes a new instance of the DataBentoRawLiveClient - /// The DataBento API key. - /// - public DataBentoRawLiveClient(string apiKey, string dataset = "GLBX.MDP3") - { - _apiKey = apiKey ?? throw new ArgumentNullException(nameof(apiKey)); - _dataset = dataset; - _tcpClient = new TcpClient(); - _subscriptions = new ConcurrentDictionary(); - _cancellationTokenSource = new CancellationTokenSource(); - _symbolMapper = new DataBentoSymbolMapper(); - - var parts = _gateway.Split(':'); - _host = parts[0]; - _port = parts.Length > 1 ? int.Parse(parts[1]) : 13000; - } - - /// - /// Connects to the DataBento live gateway - /// - public bool Connect() - { - Log.Trace("DataBentoRawLiveClient.Connect(): Connecting to DataBento live gateway"); - if (_isConnected) - { - return _isConnected; - } - - try - { - _tcpClient.Connect(_host, _port); - _stream = _tcpClient.GetStream(); - _reader = new StreamReader(_stream, Encoding.ASCII); - _writer = new StreamWriter(_stream, Encoding.ASCII) { AutoFlush = true }; - - // Perform authentication handshake - if (Authenticate()) - { - _isConnected = true; - ConnectionStatusChanged?.Invoke(this, true); - - // Start message processing - Task.Run(ProcessMessages, _cancellationTokenSource.Token); - - Log.Trace("DataBentoRawLiveClient.Connect(): Connected and authenticated to DataBento live gateway"); - return true; - } - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.Connect(): Failed to connect: {ex.Message}"); - Disconnect(); - } - - return false; - } - - /// - /// Authenticates with the DataBento gateway using CRAM-SHA256 - /// - private bool Authenticate() - { - try - { - // Read greeting and challenge - var versionLine = _reader.ReadLine(); - var cramLine = _reader.ReadLine(); - - if (string.IsNullOrEmpty(versionLine) || string.IsNullOrEmpty(cramLine)) - { - Log.Error("DataBentoRawLiveClient.Authenticate(): Failed to receive greeting or challenge"); - return false; - } - - // Parse challenge - var cramParts = cramLine.Split('='); - if (cramParts.Length != 2 || cramParts[0] != "cram") - { - Log.Error("DataBentoRawLiveClient.Authenticate(): Invalid challenge format"); - return false; - } - var cram = cramParts[1].Trim(); - - // Auth - _writer.WriteLine($"auth={GetAuthStringFromCram(cram)}|dataset={_dataset}|encoding=json|ts_out=0"); - var authResp = _reader.ReadLine(); - if (!authResp.Contains("success=1")) - { - Log.Error($"DataBentoRawLiveClient.Authenticate(): Authentication failed: {authResp}"); - return false; - } - - Log.Trace("DataBentoRawLiveClient.Authenticate(): Authentication successful"); - return true; - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.Authenticate(): Authentication failed: {ex.Message}"); - return false; - } - } - - /// - /// Handles the DataBento authentication string from a CRAM challenge - /// - /// The CRAM challenge string - /// The auth string to send to the server - private string GetAuthStringFromCram(string cram) - { - if (string.IsNullOrWhiteSpace(cram)) - throw new ArgumentException("CRAM challenge cannot be null or empty", nameof(cram)); - - string concat = $"{cram}|{_apiKey}"; - string hashHex = ComputeSHA256(concat); - string bucketId = _apiKey.Substring(_apiKey.Length - 5); - - return $"{hashHex}-{bucketId}"; - } - - /// - /// Subscribes to live data for a symbol - /// - public bool Subscribe(Symbol symbol, TickType tickType) - { - if (!IsConnected) - { - Log.Error("DataBentoRawLiveClient.Subscribe(): Not connected to gateway"); - return false; - } - - try - { - // Get the databento symbol form LEAN symbol - var databentoSymbol = _symbolMapper.GetBrokerageSymbol(symbol); - var schema = "mbp-1"; - var resolution = Resolution.Tick; - - // subscribe - var subscribeMessage = $"schema={schema}|stype_in=parent|symbols={databentoSymbol}"; - Log.Debug($"DataBentoRawLiveClient.Subscribe(): Subscribing with message: {subscribeMessage}"); - - // Send subscribe message - _writer.WriteLine(subscribeMessage); - - // Store subscription - _subscriptions.TryAdd(symbol, (resolution, tickType)); - Log.Debug($"DataBentoRawLiveClient.Subscribe(): Subscribed to {symbol} ({databentoSymbol}) at {resolution} resolution for {tickType}"); - - return true; - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.Subscribe(): Failed to subscribe to {symbol}: {ex.Message}"); - return false; - } - } - - /// - /// Starts the session to begin receiving data - /// - public bool StartSession() - { - if (!IsConnected) - { - Log.Error("DataBentoRawLiveClient.StartSession(): Not connected"); - return false; - } - - try - { - Log.Trace("DataBentoRawLiveClient.StartSession(): Starting session"); - _writer.WriteLine("start_session=1"); - return true; - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.StartSession(): Failed to start session: {ex.Message}"); - return false; - } - } - - /// - /// Unsubscribes from live data for a symbol - /// - public bool Unsubscribe(Symbol symbol) - { - try - { - // Please note there is no unsubscribe method. Subscriptions end when the TCP connection closes. - - if (_subscriptions.TryRemove(symbol, out _)) - { - Log.Debug($"DataBentoRawLiveClient.Unsubscribe(): Unsubscribed from {symbol}"); - } - return true; - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.Unsubscribe(): Failed to unsubscribe from {symbol}: {ex.Message}"); - return false; - } - } - - /// - /// Processes incoming messages from the DataBento gateway - /// - private void ProcessMessages() - { - Log.Debug("DataBentoRawLiveClient.ProcessMessages(): Starting message processing"); - - try - { - while (!_cancellationTokenSource.IsCancellationRequested && IsConnected) - { - var line = _reader.ReadLine(); - if (string.IsNullOrWhiteSpace(line)) - { - Log.Trace("DataBentoRawLiveClient.ProcessMessages(): Line is null or empty. Issue receiving data."); - break; - } - - ProcessSingleMessage(line); - } - } - catch (OperationCanceledException) - { - Log.Trace("DataBentoRawLiveClient.ProcessMessages(): Message processing cancelled"); - } - catch (IOException ex) when (ex.InnerException is SocketException) - { - Log.Trace($"DataBentoRawLiveClient.ProcessMessages(): Socket exception: {ex.Message}"); - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.ProcessMessages(): Error processing messages: {ex.Message}\n{ex.StackTrace}"); - } - finally - { - Disconnect(); - } - } - - /// - /// Processes a single message from DataBento - /// - private void ProcessSingleMessage(string message) - { - try - { - using var document = JsonDocument.Parse(message); - var root = document.RootElement; - - // Check for error messages - if (root.TryGetProperty("hd", out var headerElement)) - { - if (headerElement.TryGetProperty("rtype", out var rtypeElement)) - { - var rtype = rtypeElement.GetInt32(); - - switch (rtype) - { - case 23: - // System message - if (root.TryGetProperty("msg", out var msgElement)) - { - Log.Debug($"DataBentoRawLiveClient: System message: {msgElement.GetString()}"); - } - return; - - case 22: - // Symbol mapping message - if (root.TryGetProperty("stype_in_symbol", out var inSymbol) && - root.TryGetProperty("stype_out_symbol", out var outSymbol) && - headerElement.TryGetProperty("instrument_id", out var instId)) - { - var instrumentId = instId.GetInt64(); - var outSymbolStr = outSymbol.GetString(); - - Log.Debug($"DataBentoRawLiveClient: Symbol mapping: {inSymbol.GetString()} -> {outSymbolStr} (instrument_id: {instrumentId})"); - - if (outSymbolStr != null) - { - // Let's find the subscribed symbol to get the market and security type - var inSymbolStr = inSymbol.GetString(); - var subscription = _subscriptions.Keys.FirstOrDefault(s => _symbolMapper.GetBrokerageSymbol(s) == inSymbolStr); - if (subscription != null) - { - if (subscription.SecurityType == SecurityType.Future) - { - var leanSymbol = _symbolMapper.GetLeanSymbolForFuture(outSymbolStr); - if (leanSymbol == null) - { - Log.Trace($"DataBentoRawLiveClient: Future spreads are not supported: {outSymbolStr}. Skipping mapping."); - return; - } - _instrumentIdToSymbol[instrumentId] = leanSymbol; - Log.Debug($"DataBentoRawLiveClient: Mapped instrument_id {instrumentId} to {leanSymbol}"); - } - } - } - } - return; - - case 1: - // MBP-1 (Market By Price) - HandleMBPMessage(root, headerElement); - return; - - case 0: - // Trade messages - HandleTradeTickMessage(root, headerElement); - return; - - case 32: - case 33: - case 34: - case 35: - // OHLCV bar messages - HandleOHLCVMessage(root, headerElement); - return; - - default: - Log.Error($"DataBentoRawLiveClient: Unknown rtype {rtype} in message"); - return; - } - } - } - - // Handle other message types if needed - if (root.TryGetProperty("error", out var errorElement)) - { - Log.Error($"DataBentoRawLiveClient: Server error: {errorElement.GetString()}"); - } - } - catch (JsonException ex) - { - Log.Error($"DataBentoRawLiveClient.ProcessSingleMessage(): JSON parse error: {ex.Message}"); - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.ProcessSingleMessage(): Error: {ex.Message}"); - } - } - - /// - /// Handles OHLCV messages and converts to LEAN TradeBar data - /// - private void HandleOHLCVMessage(JsonElement root, JsonElement header) - { - try - { - if (!header.TryGetProperty("ts_event", out var tsElement) || - !header.TryGetProperty("instrument_id", out var instIdElement)) - { - return; - } - - // Convert timestamp from nanoseconds to DateTime - var timestampNs = long.Parse(tsElement.GetString()!); - var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); - var timestamp = unixEpoch.AddTicks(timestampNs / 100); - - var instrumentId = instIdElement.GetInt64(); - - if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) - { - Log.Debug($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in OHLCV message."); - return; - } - - // Get the resolution for this symbol - if (!_subscriptions.TryGetValue(matchedSymbol, out var subscription)) - { - return; - } - - var resolution = subscription.Item1; - - // Extract OHLCV data - if (root.TryGetProperty("open", out var openElement) && - root.TryGetProperty("high", out var highElement) && - root.TryGetProperty("low", out var lowElement) && - root.TryGetProperty("close", out var closeElement) && - root.TryGetProperty("volume", out var volumeElement)) - { - // Parse prices - var openRaw = long.Parse(openElement.GetString()!); - var highRaw = long.Parse(highElement.GetString()!); - var lowRaw = long.Parse(lowElement.GetString()!); - var closeRaw = long.Parse(closeElement.GetString()!); - var volume = volumeElement.GetInt64(); - - var open = openRaw * PriceScaleFactor; - var high = highRaw * PriceScaleFactor; - var low = lowRaw * PriceScaleFactor; - var close = closeRaw * PriceScaleFactor; - - // Determine the period based on resolution - TimeSpan period = resolution switch - { - Resolution.Second => TimeSpan.FromSeconds(1), - Resolution.Minute => TimeSpan.FromMinutes(1), - Resolution.Hour => TimeSpan.FromHours(1), - Resolution.Daily => TimeSpan.FromDays(1), - _ => TimeSpan.FromMinutes(1) - }; - - // Create TradeBar - var tradeBar = new TradeBar( - timestamp, - matchedSymbol, - open, - high, - low, - close, - volume, - period - ); - - // Log.Trace($"DataBentoRawLiveClient: OHLCV bar: {matchedSymbol} O={open} H={high} L={low} C={close} V={volume} at {timestamp}"); - DataReceived?.Invoke(this, tradeBar); - } - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.HandleOHLCVMessage(): Error: {ex.Message}"); - } - } - - /// - /// Handles MBP messages for quote ticks - /// - private void HandleMBPMessage(JsonElement root, JsonElement header) - { - try - { - if (!header.TryGetProperty("ts_event", out var tsElement) || - !header.TryGetProperty("instrument_id", out var instIdElement)) - { - return; - } - - // Convert timestamp from nanoseconds to DateTime - var timestampNs = long.Parse(tsElement.GetString()!); - var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); - var timestamp = unixEpoch.AddTicks(timestampNs / 100); - - var instrumentId = instIdElement.GetInt64(); - - if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) - { - Log.Trace($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in MBP message."); - return; - } - - // For MBP-1, bid/ask data is in the levels array at index 0 - if (root.TryGetProperty("levels", out var levelsElement) && - levelsElement.GetArrayLength() > 0) - { - var level0 = levelsElement[0]; - - var quoteTick = new Tick - { - Symbol = matchedSymbol, - Time = timestamp, - TickType = TickType.Quote - }; - - if (level0.TryGetProperty("ask_px", out var askPxElement) && - level0.TryGetProperty("ask_sz", out var askSzElement)) - { - var askPriceRaw = long.Parse(askPxElement.GetString()!); - quoteTick.AskPrice = askPriceRaw * PriceScaleFactor; - quoteTick.AskSize = askSzElement.GetInt32(); - } - - if (level0.TryGetProperty("bid_px", out var bidPxElement) && - level0.TryGetProperty("bid_sz", out var bidSzElement)) - { - var bidPriceRaw = long.Parse(bidPxElement.GetString()!); - quoteTick.BidPrice = bidPriceRaw * PriceScaleFactor; - quoteTick.BidSize = bidSzElement.GetInt32(); - } - - // Set the tick value to the mid price - quoteTick.Value = (quoteTick.BidPrice + quoteTick.AskPrice) / 2; - - // QuantConnect convention: Quote ticks should have zero Price and Quantity - quoteTick.Quantity = 0; - - // Log.Trace($"DataBentoRawLiveClient: Quote tick: {matchedSymbol} Bid={quoteTick.BidPrice}x{quoteTick.BidSize} Ask={quoteTick.AskPrice}x{quoteTick.AskSize}"); - DataReceived?.Invoke(this, quoteTick); - } - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.HandleMBPMessage(): Error: {ex.Message}"); - } - } - - /// - /// Handles trade tick messages. Aggressor fills - /// - private void HandleTradeTickMessage(JsonElement root, JsonElement header) - { - try - { - if (!header.TryGetProperty("ts_event", out var tsElement) || - !header.TryGetProperty("instrument_id", out var instIdElement)) - { - return; - } - - // Convert timestamp from nanoseconds to DateTime - var timestampNs = long.Parse(tsElement.GetString()!); - var unixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); - var timestamp = unixEpoch.AddTicks(timestampNs / 100); - - var instrumentId = instIdElement.GetInt64(); - - if (!_instrumentIdToSymbol.TryGetValue(instrumentId, out var matchedSymbol)) - { - Log.Trace($"DataBentoRawLiveClient: No mapping for instrument_id {instrumentId} in trade message."); - return; - } - - if (root.TryGetProperty("price", out var priceElement) && - root.TryGetProperty("size", out var sizeElement)) - { - var priceRaw = long.Parse(priceElement.GetString()!); - var size = sizeElement.GetInt32(); - var price = priceRaw * PriceScaleFactor; - - var tradeTick = new Tick - { - Symbol = matchedSymbol, - Time = timestamp, - Value = price, - Quantity = size, - TickType = TickType.Trade, - // Trade ticks should have zero bid/ask values - BidPrice = 0, - BidSize = 0, - AskPrice = 0, - AskSize = 0 - }; - - // Log.Trace($"DataBentoRawLiveClient: Trade tick: {matchedSymbol} Price={price} Quantity={size}"); - DataReceived?.Invoke(this, tradeTick); - } - } - catch (Exception ex) - { - Log.Error($"DataBentoRawLiveClient.HandleTradeTickMessage(): Error: {ex.Message}"); - } - } - - /// - /// Disconnects from the DataBento gateway - /// - public void Disconnect() - { - lock (_connectionLock) - { - if (!_isConnected) - return; - - _isConnected = false; - _cancellationTokenSource?.Cancel(); - - try - { - _reader?.Dispose(); - _writer?.Dispose(); - _stream?.Close(); - _tcpClient?.Close(); - } - catch (Exception ex) - { - Log.Trace($"DataBentoRawLiveClient.Disconnect(): Error during disconnect: {ex.Message}"); - } - - ConnectionStatusChanged?.Invoke(this, false); - Log.Trace("DataBentoRawLiveClient.Disconnect(): Disconnected from DataBento gateway"); - } - } - - /// - /// Disposes of resources - /// - public void Dispose() - { - if (_disposed) - return; - - _disposed = true; - Disconnect(); - - _cancellationTokenSource?.Dispose(); - _reader?.Dispose(); - _writer?.Dispose(); - _stream?.Dispose(); - _tcpClient?.Dispose(); - } - - /// - /// Computes the SHA-256 hash of the input string - /// - private static string ComputeSHA256(string input) - { - using var sha = SHA256.Create(); - var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(input)); - var sb = new StringBuilder(); - foreach (byte b in hash) - { - sb.Append(b.ToString("x2")); - } - return sb.ToString(); - } - -} diff --git a/QuantConnect.DataBento/DataBentoSymbolMapper.cs b/QuantConnect.DataBento/DataBentoSymbolMapper.cs index 1a824ee..a0b8913 100644 --- a/QuantConnect.DataBento/DataBentoSymbolMapper.cs +++ b/QuantConnect.DataBento/DataBentoSymbolMapper.cs @@ -67,28 +67,6 @@ public string GetBrokerageSymbol(Symbol symbol) public Symbol GetLeanSymbol(string brokerageSymbol, SecurityType securityType, string market, DateTime expirationDate = new DateTime(), decimal strike = 0, OptionRight optionRight = 0) { - switch (securityType) - { - case SecurityType.Future: - return Symbol.CreateFuture(brokerageSymbol, market, expirationDate); - default: - throw new Exception($"The unsupported security type: {securityType}"); - } - } - - /// - /// Converts a brokerage future symbol to a Lean symbol instance - /// - /// The brokerage symbol - /// A new Lean Symbol instance - public Symbol GetLeanSymbolForFuture(string brokerageSymbol) - { - // ignore futures spreads - if (brokerageSymbol.Contains("-")) - { - return null; - } - - return SymbolRepresentation.ParseFutureSymbol(brokerageSymbol); + throw new NotImplementedException("This method is not used in the current implementation."); } } diff --git a/QuantConnect.DataBento/Extensions.cs b/QuantConnect.DataBento/Extensions.cs index a4514db..7240d6a 100644 --- a/QuantConnect.DataBento/Extensions.cs +++ b/QuantConnect.DataBento/Extensions.cs @@ -14,6 +14,7 @@ */ using Newtonsoft.Json; +using QuantConnect.Lean.DataSource.DataBento.Models; using QuantConnect.Lean.DataSource.DataBento.Serialization; namespace QuantConnect.Lean.DataSource.DataBento; @@ -31,4 +32,18 @@ public static class Extensions { return JsonConvert.DeserializeObject(json, JsonSettings.SnakeCase); } + + /// + /// Deserializes the specified JSON string to a + /// using snake-case property name resolution. + /// + /// The JSON string to deserialize. + /// + /// The deserialized object, + /// or null if deserialization fails or the input is null or empty. + /// + public static MarketDataRecord? DeserializeSnakeCaseLiveData(this string json) + { + return JsonConvert.DeserializeObject(json, JsonSettings.LiveDataSnakeCase); + } } diff --git a/QuantConnect.DataBento/Models/Header.cs b/QuantConnect.DataBento/Models/Header.cs index 36774a2..08b585d 100644 --- a/QuantConnect.DataBento/Models/Header.cs +++ b/QuantConnect.DataBento/Models/Header.cs @@ -24,7 +24,7 @@ namespace QuantConnect.Lean.DataSource.DataBento.Models; public sealed class Header { /// - /// Event timestamp in nanoseconds since Unix epoch (UTC). + /// The matching-engine-received timestamp expressed as the number of nanoseconds since the UNIX epoch. /// public long TsEvent { get; set; } @@ -41,7 +41,7 @@ public sealed class Header /// /// Internal instrument identifier for the symbol. /// - public long InstrumentId { get; set; } + public uint InstrumentId { get; set; } /// /// Event time converted to UTC . diff --git a/QuantConnect.DataBento/Models/LevelOneData.cs b/QuantConnect.DataBento/Models/LevelOneData.cs index a4d3454..cc68dd4 100644 --- a/QuantConnect.DataBento/Models/LevelOneData.cs +++ b/QuantConnect.DataBento/Models/LevelOneData.cs @@ -21,8 +21,7 @@ namespace QuantConnect.Lean.DataSource.DataBento.Models; public sealed class LevelOneData : MarketDataRecord { /// - /// Timestamp when the message was received by the gateway, - /// expressed as nanoseconds since the UNIX epoch. + /// The capture-server-received timestamp expressed as the number of nanoseconds since the UNIX epoch. /// public long TsRecv { get; set; } diff --git a/QuantConnect.DataBento/Models/Live/SymbolMappingConfirmationEventArgs.cs b/QuantConnect.DataBento/Models/Live/SymbolMappingConfirmationEventArgs.cs new file mode 100644 index 0000000..461d0a2 --- /dev/null +++ b/QuantConnect.DataBento/Models/Live/SymbolMappingConfirmationEventArgs.cs @@ -0,0 +1,43 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; + +/// +/// Provides data for the symbol-mapping confirmation event. +/// +public sealed class SymbolMappingConfirmationEventArgs : EventArgs +{ + /// + /// Gets the original symbol that was mapped. + /// + public string Symbol { get; } + + /// + /// Gets the internal instrument identifier associated with the symbol. + /// + public uint InstrumentId { get; } + + /// + /// Initializes a new instance of the class. + /// + /// The symbol that was mapped. + /// The internal instrument identifier. + public SymbolMappingConfirmationEventArgs(string symbol, uint instrumentId) + { + Symbol = symbol; + InstrumentId = instrumentId; + } +} diff --git a/QuantConnect.DataBento/Models/Live/SymbolMappingMessage.cs b/QuantConnect.DataBento/Models/Live/SymbolMappingMessage.cs new file mode 100644 index 0000000..f0ffdfc --- /dev/null +++ b/QuantConnect.DataBento/Models/Live/SymbolMappingMessage.cs @@ -0,0 +1,30 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; + +public class SymbolMappingMessage : MarketDataRecord +{ + /// + /// The input symbol from the subscription + /// + public string? StypeInSymbol { get; set; } + + /// + /// The output symbol from the subscription + /// + public string? StypeOutSymbol { get; set; } +} diff --git a/QuantConnect.DataBento/Serialization/JsonSettings.cs b/QuantConnect.DataBento/Serialization/JsonSettings.cs index 5247d97..4badd80 100644 --- a/QuantConnect.DataBento/Serialization/JsonSettings.cs +++ b/QuantConnect.DataBento/Serialization/JsonSettings.cs @@ -31,4 +31,15 @@ public static class JsonSettings { ContractResolver = SnakeCaseContractResolver.Instance }; + + /// + /// Gets a reusable instance of that uses + /// for snake-case property name formatting + /// and custom converter. + /// + public static readonly JsonSerializerSettings LiveDataSnakeCase = new() + { + ContractResolver = SnakeCaseContractResolver.Instance, + Converters = { new Converters.LiveDataConverter() } + }; } From a80bdccd702c09afede9599c48de5b7c0b4ad58b Mon Sep 17 00:00:00 2001 From: Romazes Date: Wed, 28 Jan 2026 16:12:13 +0200 Subject: [PATCH 13/18] feat: reconnection process and resubscription --- .../DataBentoDataDownloaderTests.cs | 2 +- .../DataBentoDataProviderHistoryTests.cs | 2 +- .../DataBentoDataQueueHandlerTests.cs | 234 ++++++++++++++++++ .../DataBentoLiveAPIClientTests.cs | 3 +- QuantConnect.DataBento/Api/LiveAPIClient.cs | 26 +- .../Api/LiveDataTcpClientWrapper.cs | 74 ++++-- .../DataBentoDataProvider.cs | 14 +- .../Models/Live/ConnectionLostEventArgs.cs | 34 +++ 8 files changed, 359 insertions(+), 30 deletions(-) create mode 100644 QuantConnect.DataBento.Tests/DataBentoDataQueueHandlerTests.cs create mode 100644 QuantConnect.DataBento/Models/Live/ConnectionLostEventArgs.cs diff --git a/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs index b84893c..940813c 100644 --- a/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoDataDownloaderTests.cs @@ -40,7 +40,7 @@ public void SetUp() [TearDown] public void TearDown() { - _downloader?.Dispose(); + _downloader?.DisposeSafely(); } [TestCase(Resolution.Daily)] diff --git a/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs index 27a52d3..be19cbd 100644 --- a/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoDataProviderHistoryTests.cs @@ -38,7 +38,7 @@ public void SetUp() [TearDown] public void TearDown() { - _historyDataProvider?.Dispose(); + _historyDataProvider?.DisposeSafely(); } internal static IEnumerable TestParameters diff --git a/QuantConnect.DataBento.Tests/DataBentoDataQueueHandlerTests.cs b/QuantConnect.DataBento.Tests/DataBentoDataQueueHandlerTests.cs new file mode 100644 index 0000000..dd57c25 --- /dev/null +++ b/QuantConnect.DataBento.Tests/DataBentoDataQueueHandlerTests.cs @@ -0,0 +1,234 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + + +using System; +using System.Text; +using NUnit.Framework; +using System.Threading; +using QuantConnect.Util; +using QuantConnect.Data; +using QuantConnect.Logging; +using System.Threading.Tasks; +using QuantConnect.Securities; +using QuantConnect.Data.Market; +using System.Collections.Generic; +using QuantConnect.Lean.Engine.DataFeeds.Enumerators; + +namespace QuantConnect.Lean.DataSource.DataBento.Tests; + +[TestFixture] +public class DataBentoDataQueueHandlerTests +{ + private DataBentoProvider _dataProvider; + private CancellationTokenSource _cancellationTokenSource; + + [SetUp] + public void SetUp() + { + _cancellationTokenSource = new(); + _dataProvider = new(); + } + + [TearDown] + public void TearDown() + { + _cancellationTokenSource?.Cancel(); + _cancellationTokenSource.DisposeSafely(); + _dataProvider?.DisposeSafely(); + } + + private static IEnumerable TestParameters + { + get + { + var sp500EMiniMarch = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, new(2026, 03, 20)); + + yield return new TestCaseData(new Symbol[] { sp500EMiniMarch }, Resolution.Second); + } + } + + [Test, TestCaseSource(nameof(TestParameters))] + public void CanSubscribeAndUnsubscribeOnDifferentResolution(Symbol[] symbols, Resolution resolution) + { + + var configs = new List(); + + var dataFromEnumerator = new Dictionary>(); + + foreach (var symbol in symbols) + { + dataFromEnumerator[symbol] = new Dictionary(); + foreach (var config in GetSubscriptionDataConfigs(symbol, resolution)) + { + configs.Add(config); + + var tickType = config.TickType switch + { + TickType.Quote => typeof(QuoteBar), + TickType.Trade => typeof(TradeBar), + _ => throw new NotImplementedException() + }; + + dataFromEnumerator[symbol][tickType] = 0; + } + } + + Assert.That(configs, Is.Not.Empty); + + Action callback = (dataPoint) => + { + if (dataPoint == null) + { + return; + } + + switch (dataPoint) + { + case TradeBar tb: + dataFromEnumerator[tb.Symbol][typeof(TradeBar)] += 1; + break; + case QuoteBar qb: + Assert.GreaterOrEqual(qb.Ask.Open, qb.Bid.Open, $"QuoteBar validation failed for {qb.Symbol}: Ask.Open ({qb.Ask.Open}) <= Bid.Open ({qb.Bid.Open}). Full data: {DisplayBaseData(qb)}"); + Assert.GreaterOrEqual(qb.Ask.High, qb.Bid.High, $"QuoteBar validation failed for {qb.Symbol}: Ask.High ({qb.Ask.High}) <= Bid.High ({qb.Bid.High}). Full data: {DisplayBaseData(qb)}"); + Assert.GreaterOrEqual(qb.Ask.Low, qb.Bid.Low, $"QuoteBar validation failed for {qb.Symbol}: Ask.Low ({qb.Ask.Low}) <= Bid.Low ({qb.Bid.Low}). Full data: {DisplayBaseData(qb)}"); + Assert.GreaterOrEqual(qb.Ask.Close, qb.Bid.Close, $"QuoteBar validation failed for {qb.Symbol}: Ask.Close ({qb.Ask.Close}) <= Bid.Close ({qb.Bid.Close}). Full data: {DisplayBaseData(qb)}"); + dataFromEnumerator[qb.Symbol][typeof(QuoteBar)] += 1; + break; + } + ; + }; + + foreach (var config in configs) + { + ProcessFeed(_dataProvider.Subscribe(config, (sender, args) => + { + var dataPoint = ((NewDataAvailableEventArgs)args).DataPoint; + Log.Trace($"{dataPoint}. Time span: {dataPoint.Time} - {dataPoint.EndTime}"); + }), _cancellationTokenSource.Token, callback: callback); + } + + Thread.Sleep(TimeSpan.FromSeconds(120)); + + Log.Trace("Unsubscribing symbols"); + foreach (var config in configs) + { + _dataProvider.Unsubscribe(config); + } + + Thread.Sleep(TimeSpan.FromSeconds(5)); + + _cancellationTokenSource.Cancel(); + + var str = new StringBuilder(); + + str.AppendLine($"{nameof(DataBentoDataQueueHandlerTests)}.{nameof(CanSubscribeAndUnsubscribeOnDifferentResolution)}: ***** Summary *****"); + + foreach (var symbol in symbols) + { + str.AppendLine($"Input parameters: ticker:{symbol} | securityType:{symbol.SecurityType} | resolution:{resolution}"); + + foreach (var tickType in dataFromEnumerator[symbol]) + { + str.AppendLine($"[{tickType.Key}] = {tickType.Value}"); + + if (symbol.SecurityType != SecurityType.Index) + { + Assert.Greater(tickType.Value, 0); + } + // The ThetaData returns TradeBar seldom. Perhaps should find more relevant ticker. + Assert.GreaterOrEqual(tickType.Value, 0); + } + str.AppendLine(new string('-', 30)); + } + + Log.Trace(str.ToString()); + } + + private static string DisplayBaseData(BaseData item) + { + switch (item) + { + case TradeBar tradeBar: + return $"Data Type: {item.DataType} | " + tradeBar.ToString() + $" Time: {tradeBar.Time}, EndTime: {tradeBar.EndTime}"; + default: + return $"DEFAULT: Data Type: {item.DataType} | Time: {item.Time} | End Time: {item.EndTime} | Symbol: {item.Symbol} | Price: {item.Price} | IsFillForward: {item.IsFillForward}"; + } + } + + private static IEnumerable GetSubscriptionDataConfigs(Symbol symbol, Resolution resolution) + { + yield return GetSubscriptionDataConfig(symbol, resolution); + yield return GetSubscriptionDataConfig(symbol, resolution); + } + + public static IEnumerable GetSubscriptionTickDataConfigs(Symbol symbol) + { + yield return new SubscriptionDataConfig(GetSubscriptionDataConfig(symbol, Resolution.Tick), tickType: TickType.Trade); + yield return new SubscriptionDataConfig(GetSubscriptionDataConfig(symbol, Resolution.Tick), tickType: TickType.Quote); + } + + private static SubscriptionDataConfig GetSubscriptionDataConfig(Symbol symbol, Resolution resolution) + { + return new SubscriptionDataConfig( + typeof(T), + symbol, + resolution, + TimeZones.Utc, + TimeZones.Utc, + true, + extendedHours: false, + false); + } + + private Task ProcessFeed( + IEnumerator enumerator, + CancellationToken cancellationToken, + int cancellationTokenDelayMilliseconds = 100, + Action callback = null, + Action throwExceptionCallback = null) + { + return Task.Factory.StartNew(() => + { + try + { + while (enumerator.MoveNext() && !cancellationToken.IsCancellationRequested) + { + BaseData tick = enumerator.Current; + + if (tick != null) + { + callback?.Invoke(tick); + } + + cancellationToken.WaitHandle.WaitOne(TimeSpan.FromMilliseconds(cancellationTokenDelayMilliseconds)); + } + } + catch (Exception ex) + { + Log.Debug($"{nameof(DataBentoDataQueueHandlerTests)}.{nameof(ProcessFeed)}.Exception: {ex.Message}"); + throw; + } + }, cancellationToken).ContinueWith(task => + { + if (throwExceptionCallback != null) + { + throwExceptionCallback(); + } + Log.Debug("The throwExceptionCallback is null."); + }, TaskContinuationOptions.OnlyOnFaulted); + } +} diff --git a/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs index ea66096..69d5f1d 100644 --- a/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs @@ -16,6 +16,7 @@ using System; using NUnit.Framework; using System.Threading; +using QuantConnect.Util; using QuantConnect.Configuration; using System.Collections.Generic; using QuantConnect.Lean.DataSource.DataBento.Api; @@ -51,7 +52,7 @@ public void OneTimeSetUp() [OneTimeTearDown] public void OneTimeTearDown() { - _live.Dispose(); + _live?.DisposeSafely(); } [Test] diff --git a/QuantConnect.DataBento/Api/LiveAPIClient.cs b/QuantConnect.DataBento/Api/LiveAPIClient.cs index 810dd9b..3629b95 100644 --- a/QuantConnect.DataBento/Api/LiveAPIClient.cs +++ b/QuantConnect.DataBento/Api/LiveAPIClient.cs @@ -31,6 +31,8 @@ public sealed class LiveAPIClient : IDisposable public event EventHandler? SymbolMappingConfirmation; + public event EventHandler? ConnectionLost; + public bool IsConnected => _tcpClientByDataSet.Values.All(c => c.IsConnected); public LiveAPIClient(string apiKey, Action levelOneDataHandler) @@ -50,17 +52,35 @@ public void Dispose() private LiveDataTcpClientWrapper EnsureDatasetConnection(string dataSet) { - if (_tcpClientByDataSet.TryGetValue(dataSet, out var liveDataTcpClient)) + if (_tcpClientByDataSet.TryGetValue(dataSet, out var liveDataTcpClient) && liveDataTcpClient.IsConnected) { return liveDataTcpClient; } LogTrace(nameof(EnsureDatasetConnection), "Starting connection to DataBento live API"); - liveDataTcpClient = new LiveDataTcpClientWrapper(dataSet, _apiKey, MessageReceived); - _tcpClientByDataSet[dataSet] = liveDataTcpClient; + if (liveDataTcpClient == null) + { + liveDataTcpClient = new LiveDataTcpClientWrapper(dataSet, _apiKey, MessageReceived); + + liveDataTcpClient.ConnectionLost += (sender, message) => + { + LogError(nameof(EnsureDatasetConnection), $"Connection lost to DataBento live API (Dataset: {dataSet}). Reason: {message}"); + ConnectionLost?.Invoke(this, new ConnectionLostEventArgs(dataSet, message)); + }; + + _tcpClientByDataSet[dataSet] = liveDataTcpClient; + } + liveDataTcpClient.Connect(); + if (!liveDataTcpClient.IsConnected) + { + var msg = $"Unable to establish a connection to the DataBento Live API (Dataset: {dataSet})."; + LogError(nameof(EnsureDatasetConnection), msg); + throw new Exception(msg); + } + LogTrace(nameof(EnsureDatasetConnection), $"Successfully connected to DataBento live API (Dataset: {dataSet})"); return liveDataTcpClient; diff --git a/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs b/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs index b55e14a..87f1581 100644 --- a/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs +++ b/QuantConnect.DataBento/Api/LiveDataTcpClientWrapper.cs @@ -31,7 +31,7 @@ public sealed class LiveDataTcpClientWrapper : IDisposable private readonly string _apiKey; private readonly TimeSpan _heartBeatInterval = TimeSpan.FromSeconds(10); - private readonly TcpClient _tcpClient = new(); + private TcpClient _tcpClient; private readonly CancellationTokenSource _cancellationTokenSource = new(); private NetworkStream? _stream; @@ -41,6 +41,8 @@ public sealed class LiveDataTcpClientWrapper : IDisposable private readonly Action MessageReceived; + public event EventHandler? ConnectionLost; + /// /// Is client connected /// @@ -56,17 +58,46 @@ public LiveDataTcpClientWrapper(string dataSet, string apiKey, Action me public void Connect() { - _tcpClient.Connect(_gateway, DefaultPort); - _stream = _tcpClient.GetStream(); - _reader = new StreamReader(_stream, Encoding.ASCII); + var attemptToConnect = 1; + var error = default(string); + do + { + try + { + _tcpClient = new(_gateway, DefaultPort); + _stream = _tcpClient.GetStream(); + _reader = new StreamReader(_stream, Encoding.ASCII); + + if (!Authenticate(_dataSet).SynchronouslyAwaitTask()) + throw new Exception("Authentication failed"); - if (!Authenticate(_dataSet).SynchronouslyAwaitTask()) - throw new Exception("Authentication failed"); + _dataReceiverTask = new Task(async () => await DataReceiverAsync(_cancellationTokenSource.Token), _cancellationTokenSource.Token, TaskCreationOptions.LongRunning); + _dataReceiverTask.Start(); - _dataReceiverTask = new Task(async () => await DataReceiverAsync(_cancellationTokenSource.Token), _cancellationTokenSource.Token, TaskCreationOptions.LongRunning); - _dataReceiverTask.Start(); + _isConnected = true; + } + catch (Exception ex) + { + error = ex.Message; + } - _isConnected = true; + var retryDelayMs = attemptToConnect * 2 * 1000; + LogError(nameof(Connect), $"Connection attempt #{attemptToConnect} failed. Retrying in {retryDelayMs} ms. Error: {error}"); + _cancellationTokenSource.Token.WaitHandle.WaitOne(attemptToConnect * 2 * 1000); + + } while (attemptToConnect++ < 5 && !_isConnected); + } + + private void Close() + { + _isConnected = false; + + _reader?.Close(); + _reader?.DisposeSafely(); + + _dataReceiverTask?.DisposeSafely(); + _tcpClient?.Close(); + _tcpClient?.DisposeSafely(); } public void Dispose() @@ -98,7 +129,9 @@ private async Task DataReceiverAsync(CancellationToken ct) var readTimeout = _heartBeatInterval.Add(TimeSpan.FromSeconds(5)); - LogTrace(methodName, "Receiver started"); + LogTrace(methodName, "Task Receiver started"); + + var errorMessage = string.Empty; try { @@ -118,27 +151,22 @@ private async Task DataReceiverAsync(CancellationToken ct) MessageReceived.Invoke(line); } } - catch (OperationCanceledException) - { - if (!_tcpClient.Connected) - { - LogError("DataReceiverAsync", "GG"); - } - - Log.Trace("DataBentoRawLiveClient.ProcessMessages(): Message processing cancelled"); - } - catch (IOException ex) when (ex.InnerException is SocketException) + catch (OperationCanceledException oce) { - Log.Trace($"DataBentoRawLiveClient.ProcessMessages(): Socket exception: {ex.Message}"); + errorMessage = $"Read timeout exceeded: Outer CancellationToken: {ct.IsCancellationRequested}, Read Timeout: {readTimeoutCts.IsCancellationRequested}"; + LogTrace(methodName, errorMessage); } catch (Exception ex) { - Log.Error($"DataBentoRawLiveClient.ProcessMessages(): Error processing messages: {ex.Message}\n{ex.StackTrace}"); + errorMessage += ex.Message; + LogError(methodName, $"Error processing messages: {ex.Message}\n{ex.StackTrace}"); } finally { - LogTrace(methodName, "Receiver stopped"); + LogTrace(methodName, "Task Receiver stopped"); + Close(); readTimeoutCts.Dispose(); + ConnectionLost?.Invoke(this, new($"{errorMessage}. TcpConnected: {_tcpClient.Connected}")); } } diff --git a/QuantConnect.DataBento/DataBentoDataProvider.cs b/QuantConnect.DataBento/DataBentoDataProvider.cs index 5127a45..23fd465 100644 --- a/QuantConnect.DataBento/DataBentoDataProvider.cs +++ b/QuantConnect.DataBento/DataBentoDataProvider.cs @@ -106,11 +106,12 @@ private void Initialize(string apiKey) { var aggregatorName = Config.Get("data-aggregator", "QuantConnect.Lean.Engine.DataFeeds.AggregationManager"); Log.Trace($"{nameof(DataBentoProvider)}.{nameof(Initialize)}: found no data aggregator instance, creating {aggregatorName}"); - _aggregator = Composer.Instance.GetExportedValueByTypeName(aggregatorName); + _aggregator = Composer.Instance.GetExportedValueByTypeName(aggregatorName, forceTypeNameOnExisting: false); } _liveApiClient = new LiveAPIClient(apiKey, HandleLevelOneData); _liveApiClient.SymbolMappingConfirmation += OnSymbolMappingConfirmation; + _liveApiClient.ConnectionLost += OnConnectionLost; _historicalApiClient = new(apiKey); @@ -122,6 +123,17 @@ private void Initialize(string apiKey) _initialized = true; } + private void OnConnectionLost(object? _, ConnectionLostEventArgs cle) + { + LogTrace(nameof(OnConnectionLost), "The connection was lost. Starting ReSubscription process"); + + var symbols = _levelOneServiceManager.GetSubscribedSymbols(); + + Subscribe(symbols); + + LogTrace(nameof(OnConnectionLost), $"Re-subscription completed successfully for {_levelOneServiceManager.Count} symbol(s)."); + } + private void OnSymbolMappingConfirmation(object? _, SymbolMappingConfirmationEventArgs smce) { if (_pendingSubscriptions.TryRemove(smce.Symbol, out var symbol)) diff --git a/QuantConnect.DataBento/Models/Live/ConnectionLostEventArgs.cs b/QuantConnect.DataBento/Models/Live/ConnectionLostEventArgs.cs new file mode 100644 index 0000000..6994e98 --- /dev/null +++ b/QuantConnect.DataBento/Models/Live/ConnectionLostEventArgs.cs @@ -0,0 +1,34 @@ +namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; + +/// +/// Provides data for the event that is raised when a connection is lost. +/// +public class ConnectionLostEventArgs : EventArgs +{ + /// + /// Gets the identifier of the data set or logical stream + /// associated with the lost connection. + /// + public string DataSet { get; } + + /// + /// Gets a human-readable description of the reason + /// why the connection was lost. + /// + public string Reason { get; } + + /// + /// Initializes a new instance of the class. + /// + /// + /// The identifier of the data set or logical stream related to the connection. + /// + /// + /// A human-readable explanation describing why the connection was lost. + /// + public ConnectionLostEventArgs(string message, string reason) + { + DataSet = message; + Reason = reason; + } +} From d72717ed86afaeb9816bde3d79ef004d60a1fb29 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 29 Jan 2026 21:56:59 +0200 Subject: [PATCH 14/18] refactor: deserialization and model base types Unify market data deserialization using a new DataConverter and MarketDataBase abstract class. Remove custom JsonSettings and legacy deserialization methods. Update all model classes to inherit from MarketDataBase, make LevelOneBookLevel prices nullable, and improve error handling for unknown/invalid JSON. Move event args to .Models.Events. Expand and update tests for new d eserialization logic and edge cases. These changes improve robustness, extensibility, and maintainability of DataBento data ingestion. --- .../DataBentoHistoricalApiClientTests.cs | 19 ++- .../DataBentoJsonConverterTests.cs | 156 ++++++++++++++---- .../DataBentoLiveAPIClientTests.cs | 3 +- .../Api/HistoricalAPIClient.cs | 8 +- QuantConnect.DataBento/Api/LiveAPIClient.cs | 3 +- ...{LiveDataConverter.cs => DataConverter.cs} | 56 +++++-- .../DataBentoDataProvider.cs | 2 +- .../DataBentoHistoryProivder.cs | 7 +- QuantConnect.DataBento/Extensions.cs | 21 +-- .../ConnectionLostEventArgs.cs | 17 +- .../SymbolMappingConfirmationEventArgs.cs | 2 +- .../Models/LevelOneBookLevel.cs | 4 +- QuantConnect.DataBento/Models/LevelOneData.cs | 2 +- .../Models/Live/SymbolMappingMessage.cs | 2 +- .../{HeartbeatMessage.cs => SystemMessage.cs} | 4 +- ...{MarketDataRecord.cs => MarketDataBase.cs} | 6 +- ...cvBar.cs => OpenHighLowCloseVolumeData.cs} | 2 +- .../Models/StatisticsData.cs | 2 +- .../Serialization/JsonSettings.cs | 45 ----- 19 files changed, 236 insertions(+), 125 deletions(-) rename QuantConnect.DataBento/Converters/{LiveDataConverter.cs => DataConverter.cs} (57%) rename QuantConnect.DataBento/Models/{Live => Events}/ConnectionLostEventArgs.cs (56%) rename QuantConnect.DataBento/Models/{Live => Events}/SymbolMappingConfirmationEventArgs.cs (96%) rename QuantConnect.DataBento/Models/Live/{HeartbeatMessage.cs => SystemMessage.cs} (88%) rename QuantConnect.DataBento/Models/{MarketDataRecord.cs => MarketDataBase.cs} (85%) rename QuantConnect.DataBento/Models/{OhlcvBar.cs => OpenHighLowCloseVolumeData.cs} (96%) delete mode 100644 QuantConnect.DataBento/Serialization/JsonSettings.cs diff --git a/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs index 4623776..8c5ee9a 100644 --- a/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs @@ -51,7 +51,6 @@ public void OneTimeSetUp() [TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Hour)] [TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Minute)] [TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Second)] - //[TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Tick)] [TestCase("ESH6 C6875", "2026/01/11", "2026/01/20", Resolution.Daily)] public void CanInitializeHistoricalApiClient(string ticker, DateTime startDate, DateTime endDate, Resolution resolution) { @@ -101,4 +100,22 @@ public void ShouldFetchOpenInterest(string ticker, DateTime startDate, DateTime Log.Trace($"{nameof(CanInitializeHistoricalApiClient)}: {ticker} | [{startDate} - {endDate}] | {resolution} = {dataCounter} (bars)"); Assert.Greater(dataCounter, 0); } + + [TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Tick)] + public void ShouldFetchTicks(string ticker, DateTime startDate, DateTime endDate, Resolution resolution) + { + var dataCounter = 0; + var previousEndTime = DateTime.MinValue; + foreach (var data in _client.GetTickBars(ticker, startDate, endDate, Dataset)) + { + Assert.IsNotNull(data); + Assert.Greater(data.Price, 0m); + Assert.Greater(data.Size, 0); + Assert.AreNotEqual(default(DateTime), data.Header.UtcTime); + previousEndTime = data.Header.UtcTime; + dataCounter++; + } + Log.Trace($"{nameof(ShouldFetchTicks)}: {ticker} | [{startDate} - {endDate}] | {resolution} = {dataCounter} (ticks)"); + Assert.Greater(dataCounter, 0); + } } diff --git a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs index d29d488..3ac629f 100644 --- a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs @@ -13,10 +13,13 @@ * limitations under the License. */ +using System; +using Newtonsoft.Json; using NUnit.Framework; +using System.Collections.Generic; using QuantConnect.Lean.DataSource.DataBento.Models; -using QuantConnect.Lean.DataSource.DataBento.Models.Enums; using QuantConnect.Lean.DataSource.DataBento.Models.Live; +using QuantConnect.Lean.DataSource.DataBento.Models.Enums; namespace QuantConnect.Lean.DataSource.DataBento.Tests; @@ -39,7 +42,7 @@ public void DeserializeHistoricalOhlcvBar() ""close"": ""6355.000000000"", ""volume"": ""2"" }"; - var res = json.DeserializeKebabCase(); + var res = json.DeserializeObject(); Assert.IsNotNull(res); @@ -55,10 +58,11 @@ public void DeserializeHistoricalOhlcvBar() Assert.AreEqual(2L, res.Volume); } - [Test] - public void DeserializeHistoricalLevelOneData() + private static IEnumerable HistoricalLevelOneData { - var json = @"{ + get + { + yield return new TestCaseData(@"{ ""ts_recv"": ""1768137063449660443"", ""hd"": { ""ts_event"": ""1768137063107829777"", @@ -84,33 +88,106 @@ public void DeserializeHistoricalLevelOneData() ""ask_ct"": 1 } ] -}"; - var res = json.DeserializeKebabCase(); +}").SetArgDisplayNames("Normal"); + yield return new TestCaseData(@"{ + ""ts_recv"": ""1736722822572003465"", + ""hd"": { + ""ts_event"": ""1736722822571417935"", + ""rtype"": 1, + ""publisher_id"": 1, + ""instrument_id"": 42140878 + }, + ""action"": ""A"", + ""side"": ""N"", + ""depth"": 0, + ""price"": ""6100.000000000"", + ""size"": 3, + ""flags"": 130, + ""ts_in_delta"": 13574, + ""sequence"": 11624, + ""levels"": [ + { + ""bid_px"": null, + ""ask_px"": ""6100.000000000"", + ""bid_sz"": 0, + ""ask_sz"": 3, + ""bid_ct"": 0, + ""ask_ct"": 1 + } + ] +}").SetArgDisplayNames("BidPriceNull"); + yield return new TestCaseData(@"{ + ""ts_recv"": ""1736751414397721230"", + ""hd"": { + ""ts_event"": ""1736751414397599041"", + ""rtype"": 1, + ""publisher_id"": 1, + ""instrument_id"": 42140878 + }, + ""action"": ""C"", + ""side"": ""A"", + ""depth"": 0, + ""price"": ""6087.250000000"", + ""size"": 3, + ""flags"": 130, + ""ts_in_delta"": 13718, + ""sequence"": 934949, + ""levels"": [ + { + ""bid_px"": null, + ""ask_px"": null, + ""bid_sz"": 0, + ""ask_sz"": 0, + ""bid_ct"": 0, + ""ask_ct"": 0 + } + ] +}").SetArgDisplayNames("BidAndAskPriceNull"); + } + } + + [TestCaseSource(nameof(HistoricalLevelOneData))] + public void DeserializeHistoricalLevelOneData(string json) + { + var res = json.DeserializeObject(); Assert.IsNotNull(res); - Assert.AreEqual(1768137063449660443, res.TsRecv); + Assert.Greater(res.TsRecv, 0); - Assert.AreEqual(1768137063107829777, res.Header.TsEvent); + Assert.Greater(res.Header.TsEvent, 0); Assert.AreEqual(RecordType.MarketByPriceDepth1, res.Header.Rtype); - Assert.AreEqual(1, res.Header.PublisherId); - Assert.AreEqual(42140878, res.Header.InstrumentId); + Assert.AreEqual(res.Header.PublisherId, 1); + Assert.Greater(res.Header.InstrumentId, 0); + - Assert.AreEqual('A', res.Action); - Assert.AreEqual('N', res.Side); + Assert.IsTrue(char.IsLetter(res.Action)); + Assert.IsTrue(char.IsLetter(res.Side)); Assert.AreEqual(0, res.Depth); - Assert.AreEqual(7004.25m, res.Price); - Assert.AreEqual(15, res.Size); - Assert.AreEqual(128, res.Flags); + Assert.Greater(res.Price, 0); + Assert.Greater(res.Size, 0); + Assert.Greater(res.Flags, 0); Assert.IsNotNull(res.Levels); Assert.AreEqual(1, res.Levels.Count); var level = res.Levels[0]; - Assert.AreEqual(7004.0m, level.BidPx); - Assert.AreEqual(7004.25m, level.AskPx); - Assert.AreEqual(11, level.BidSz); - Assert.AreEqual(15, level.AskSz); - Assert.AreEqual(1, level.BidCt); - Assert.AreEqual(1, level.AskCt); + AssertPositiveOrNull(level.BidPx); + AssertPositiveOrNull(level.AskPx); + Assert.GreaterOrEqual(level.BidSz, 0); + Assert.GreaterOrEqual(level.AskSz, 0); + Assert.GreaterOrEqual(level.BidCt, 0); + Assert.GreaterOrEqual(level.AskCt, 0); + } + + private void AssertPositiveOrNull(decimal? price) + { + if (price.HasValue) + { + Assert.Greater(price.Value, 0); + } + else + { + Assert.IsNull(price); + } } [Test] @@ -135,7 +212,7 @@ public void DeserializeHistoricalStatisticsData() ""stat_flags"": 0 }"; - var res = json.DeserializeKebabCase(); + var res = json.DeserializeObject(); Assert.IsNotNull(res); @@ -148,19 +225,26 @@ public void DeserializeHistoricalStatisticsData() Assert.AreEqual(StatisticType.OpenInterest, res.StatType); } - [Test] - public void DeserializeLiveHeartbeatMessage() + private static IEnumerable SystemLiveMessages { - var json = @"{""hd"":{""ts_event"":""1769176693139629181"",""rtype"":23,""publisher_id"":0,""instrument_id"":0},""msg"":""Heartbeat""}"; + get + { + yield return new TestCaseData(@"{""hd"":{""ts_event"":""1769176693139629181"",""rtype"":23,""publisher_id"":0,""instrument_id"":0},""msg"":""Heartbeat""}").SetArgDisplayNames("HeartbeatMessage"); + yield return new TestCaseData(@"{""hd"":{ ""ts_event"":""1769712709771313573"",""rtype"":23,""publisher_id"":0,""instrument_id"":0},""msg"":""Subscription request for mbp-1 data succeeded""}").SetArgDisplayNames("SubscriptionRequestMarketByPrice1DataSucceeded"); + } + } - var res = json.DeserializeKebabCase(); + [TestCaseSource(nameof(SystemLiveMessages))] + public void DeserializeLiveSystemMessage(string json) + { + var res = json.DeserializeObject(); Assert.IsNotNull(res); - Assert.AreEqual(1769176693139629181, res.Header.TsEvent); + Assert.Greater(res.Header.TsEvent, 0); Assert.AreEqual(RecordType.System, res.Header.Rtype); Assert.AreEqual(0, res.Header.PublisherId); Assert.AreEqual(0, res.Header.InstrumentId); - Assert.AreEqual("Heartbeat", res.Msg); + Assert.IsFalse(string.IsNullOrEmpty(res.Msg)); } [TestCase("success=0|error=Unknown subscription param 'sssauth'", false)] @@ -209,7 +293,7 @@ public void DeserializeSymbolMappingMessage() ""end_ts"": ""18446744073709551615"" }"; - var marketData = json.DeserializeSnakeCaseLiveData(); + var marketData = json.DeserializeObject(); Assert.IsNotNull(marketData); Assert.AreEqual(1769546804979770503, marketData.Header.TsEvent); @@ -257,7 +341,7 @@ public void DeserializeMarketByPriceMessage() } ] }"; - var marketData = json.DeserializeSnakeCaseLiveData(); + var marketData = json.DeserializeObject(); Assert.IsNotNull(marketData); Assert.AreEqual(1769546804990833083, marketData.Header.TsEvent); @@ -284,4 +368,14 @@ public void DeserializeMarketByPriceMessage() Assert.AreEqual(8, level.BidCt); Assert.AreEqual(2, level.AskCt); } + + [Test] + public void DeserializeUnknownJsonFormatShouldThrow() + { + var json = @"{ ""some_property"": ""some_value"" }"; + Assert.Throws(() => json.DeserializeObject()); + + var json2 = @"{ ""hd"": { ""rtype"": 9999 } }"; + Assert.Throws(() => json2.DeserializeObject()); + } } diff --git a/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs index 69d5f1d..22a859d 100644 --- a/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoLiveAPIClientTests.cs @@ -20,6 +20,7 @@ using QuantConnect.Configuration; using System.Collections.Generic; using QuantConnect.Lean.DataSource.DataBento.Api; +using QuantConnect.Lean.DataSource.DataBento.Models.Events; namespace QuantConnect.Lean.DataSource.DataBento.Tests; @@ -66,7 +67,7 @@ public void ShouldReceiveSymbolMappingConfirmation() { Securities.Futures.Indices.Russell2000EMini + "H6", 0 } }; - void OnSymbolMappingConfirmation(object sender, Models.Live.SymbolMappingConfirmationEventArgs e) + void OnSymbolMappingConfirmation(object sender, SymbolMappingConfirmationEventArgs e) { if (subs.ContainsKey(e.Symbol)) { diff --git a/QuantConnect.DataBento/Api/HistoricalAPIClient.cs b/QuantConnect.DataBento/Api/HistoricalAPIClient.cs index 23c79ee..d420a88 100644 --- a/QuantConnect.DataBento/Api/HistoricalAPIClient.cs +++ b/QuantConnect.DataBento/Api/HistoricalAPIClient.cs @@ -38,7 +38,7 @@ public HistoricalAPIClient(string apiKey) ); } - public IEnumerable GetHistoricalOhlcvBars(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, Resolution resolution, string dataSet) + public IEnumerable GetHistoricalOhlcvBars(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, Resolution resolution, string dataSet) { string schema; switch (resolution) @@ -59,7 +59,7 @@ public IEnumerable GetHistoricalOhlcvBars(string symbol, DateTime star throw new ArgumentException($"Unsupported resolution {resolution} for OHLCV data."); } - return GetRange(symbol, startDateTimeUtc, endDateTimeUtc, schema, dataSet); + return GetRange(symbol, startDateTimeUtc, endDateTimeUtc, schema, dataSet); } public IEnumerable GetTickBars(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, string dataSet) @@ -78,7 +78,7 @@ public IEnumerable GetOpenInterest(string symbol, DateTime start } } - private IEnumerable GetRange(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, string schema, string dataSet, bool useLimit = false) where T : MarketDataRecord + private IEnumerable GetRange(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, string schema, string dataSet, bool useLimit = false) where T : MarketDataBase { var formData = new Dictionary { @@ -143,7 +143,7 @@ private IEnumerable GetRange(string symbol, DateTime startDateTimeUtc, Dat var data = default(T); while ((line = reader.ReadLine()) != null) { - data = line.DeserializeKebabCase(); + data = line.DeserializeObject(); yield return data; } start = data.Header.UtcTime.AddTicks(1); diff --git a/QuantConnect.DataBento/Api/LiveAPIClient.cs b/QuantConnect.DataBento/Api/LiveAPIClient.cs index 3629b95..adae1a2 100644 --- a/QuantConnect.DataBento/Api/LiveAPIClient.cs +++ b/QuantConnect.DataBento/Api/LiveAPIClient.cs @@ -18,6 +18,7 @@ using QuantConnect.Logging; using QuantConnect.Lean.DataSource.DataBento.Models; using QuantConnect.Lean.DataSource.DataBento.Models.Live; +using QuantConnect.Lean.DataSource.DataBento.Models.Events; namespace QuantConnect.Lean.DataSource.DataBento.Api; @@ -94,7 +95,7 @@ public bool Subscribe(string dataSet, string symbol) private void MessageReceived(string message) { - var data = message.DeserializeSnakeCaseLiveData(); + var data = message.DeserializeObject(); if (data == null) { diff --git a/QuantConnect.DataBento/Converters/LiveDataConverter.cs b/QuantConnect.DataBento/Converters/DataConverter.cs similarity index 57% rename from QuantConnect.DataBento/Converters/LiveDataConverter.cs rename to QuantConnect.DataBento/Converters/DataConverter.cs index 395706f..21067ab 100644 --- a/QuantConnect.DataBento/Converters/LiveDataConverter.cs +++ b/QuantConnect.DataBento/Converters/DataConverter.cs @@ -15,19 +15,30 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using QuantConnect.Logging; using QuantConnect.Lean.DataSource.DataBento.Models; -using QuantConnect.Lean.DataSource.DataBento.Models.Enums; using QuantConnect.Lean.DataSource.DataBento.Models.Live; +using QuantConnect.Lean.DataSource.DataBento.Models.Enums; namespace QuantConnect.Lean.DataSource.DataBento.Converters; -public class LiveDataConverter : JsonConverter +public class DataConverter : JsonConverter { + /// + /// JSON property name used to identify the message header section. + /// private const string headerIdentifier = "hd"; + /// + /// JSON property name that specifies the market data record type. + /// private const string recordTypeIdentifier = "rtype"; - private static JsonSerializer _snakeSerializer = new JsonSerializer + /// + /// Shared JSON serializer configured to use snake_case naming, + /// used for deserializing market data payloads. + /// + private readonly static JsonSerializer _snakeSerializer = new() { ContractResolver = Serialization.SnakeCaseContractResolver.Instance }; @@ -52,21 +63,46 @@ public class LiveDataConverter : JsonConverter /// The existing property value of the JSON that is being converted. /// The calling serializer. /// The object value. - public override MarketDataRecord ReadJson(JsonReader reader, Type objectType, MarketDataRecord? existingValue, bool hasExistingValue, JsonSerializer serializer) + public override MarketDataBase ReadJson(JsonReader reader, Type objectType, MarketDataBase existingValue, bool hasExistingValue, JsonSerializer serializer) { var jObject = JObject.Load(reader); - var recordType = jObject[headerIdentifier]?[recordTypeIdentifier]?.ToObject(); + var recordTypeToken = jObject[headerIdentifier]?[recordTypeIdentifier]; + if (recordTypeToken == null) + { + var msg = $"Cannot read '{recordTypeIdentifier}' from JSON"; + Log.Error($"{nameof(DataConverter)}.{nameof(ReadJson)}: {msg}. JSON: {jObject.ToString(Formatting.None)}."); + throw new JsonSerializationException(msg); + } + + var recordType = recordTypeToken.ToObject(); + var marketDataBase = default(MarketDataBase); switch (recordType) { - case RecordType.SymbolMapping: - return jObject.ToObject(_snakeSerializer); + case RecordType.OpenHighLowCloseVolume1Day: + marketDataBase = new OpenHighLowCloseVolumeData(); + break; case RecordType.MarketByPriceDepth1: - return jObject.ToObject(_snakeSerializer); + marketDataBase = new LevelOneData(); + break; + case RecordType.SymbolMapping: + marketDataBase = new SymbolMappingMessage(); + break; + case RecordType.Statistics: + marketDataBase = new StatisticsData(); + break; + case RecordType.System: + marketDataBase = new SystemMessage(); + break; default: - return null; + var msg = $"Unsupported RecordType '{recordType}'"; + Log.Error($"{nameof(DataConverter)}.{nameof(ReadJson)}: {msg}. JSON: {jObject.ToString(Formatting.None)}."); + throw new NotSupportedException(msg); } + + _snakeSerializer.Populate(jObject.CreateReader(), marketDataBase); + return marketDataBase; } /// @@ -75,7 +111,7 @@ public override MarketDataRecord ReadJson(JsonReader reader, Type objectType, Ma /// The to write to. /// The value. /// The calling serializer. - public override void WriteJson(JsonWriter writer, MarketDataRecord? value, JsonSerializer serializer) + public override void WriteJson(JsonWriter writer, MarketDataBase? value, JsonSerializer serializer) { throw new NotImplementedException(); } diff --git a/QuantConnect.DataBento/DataBentoDataProvider.cs b/QuantConnect.DataBento/DataBentoDataProvider.cs index 23fd465..a704ac5 100644 --- a/QuantConnect.DataBento/DataBentoDataProvider.cs +++ b/QuantConnect.DataBento/DataBentoDataProvider.cs @@ -31,7 +31,7 @@ using QuantConnect.Brokerages.LevelOneOrderBook; using QuantConnect.Lean.DataSource.DataBento.Api; using QuantConnect.Lean.DataSource.DataBento.Models; -using QuantConnect.Lean.DataSource.DataBento.Models.Live; +using QuantConnect.Lean.DataSource.DataBento.Models.Events; namespace QuantConnect.Lean.DataSource.DataBento; diff --git a/QuantConnect.DataBento/DataBentoHistoryProivder.cs b/QuantConnect.DataBento/DataBentoHistoryProivder.cs index fabf42e..09920eb 100644 --- a/QuantConnect.DataBento/DataBentoHistoryProivder.cs +++ b/QuantConnect.DataBento/DataBentoHistoryProivder.cs @@ -253,7 +253,12 @@ private IEnumerable GetQuotes(HistoryRequest request, string brokerage var time = quoteBar.Header.UtcTime.ConvertFromUtc(request.DataTimeZone); foreach (var level in quoteBar.Levels) { - yield return new Tick(time, request.Symbol, level.BidSz, level.BidPx, level.AskSz, level.AskPx); + if (level.BidPx == null && level.AskPx == null) + { + continue; + } + + yield return new Tick(time, request.Symbol, level.BidSz, level.BidPx ?? 0m, level.AskSz, level.AskPx ?? 0m); } } } diff --git a/QuantConnect.DataBento/Extensions.cs b/QuantConnect.DataBento/Extensions.cs index 7240d6a..62f30db 100644 --- a/QuantConnect.DataBento/Extensions.cs +++ b/QuantConnect.DataBento/Extensions.cs @@ -14,8 +14,7 @@ */ using Newtonsoft.Json; -using QuantConnect.Lean.DataSource.DataBento.Models; -using QuantConnect.Lean.DataSource.DataBento.Serialization; +using QuantConnect.Logging; namespace QuantConnect.Lean.DataSource.DataBento; @@ -28,22 +27,8 @@ public static class Extensions /// The target type of the deserialized object. /// The JSON string to deserialize. /// The deserialized object of type . - public static T? DeserializeKebabCase(this string json) + public static T? DeserializeObject(this string json) { - return JsonConvert.DeserializeObject(json, JsonSettings.SnakeCase); - } - - /// - /// Deserializes the specified JSON string to a - /// using snake-case property name resolution. - /// - /// The JSON string to deserialize. - /// - /// The deserialized object, - /// or null if deserialization fails or the input is null or empty. - /// - public static MarketDataRecord? DeserializeSnakeCaseLiveData(this string json) - { - return JsonConvert.DeserializeObject(json, JsonSettings.LiveDataSnakeCase); + return JsonConvert.DeserializeObject(json); } } diff --git a/QuantConnect.DataBento/Models/Live/ConnectionLostEventArgs.cs b/QuantConnect.DataBento/Models/Events/ConnectionLostEventArgs.cs similarity index 56% rename from QuantConnect.DataBento/Models/Live/ConnectionLostEventArgs.cs rename to QuantConnect.DataBento/Models/Events/ConnectionLostEventArgs.cs index 6994e98..9838433 100644 --- a/QuantConnect.DataBento/Models/Live/ConnectionLostEventArgs.cs +++ b/QuantConnect.DataBento/Models/Events/ConnectionLostEventArgs.cs @@ -1,4 +1,19 @@ -namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models.Events; /// /// Provides data for the event that is raised when a connection is lost. diff --git a/QuantConnect.DataBento/Models/Live/SymbolMappingConfirmationEventArgs.cs b/QuantConnect.DataBento/Models/Events/SymbolMappingConfirmationEventArgs.cs similarity index 96% rename from QuantConnect.DataBento/Models/Live/SymbolMappingConfirmationEventArgs.cs rename to QuantConnect.DataBento/Models/Events/SymbolMappingConfirmationEventArgs.cs index 461d0a2..2b398af 100644 --- a/QuantConnect.DataBento/Models/Live/SymbolMappingConfirmationEventArgs.cs +++ b/QuantConnect.DataBento/Models/Events/SymbolMappingConfirmationEventArgs.cs @@ -13,7 +13,7 @@ * limitations under the License. */ -namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; +namespace QuantConnect.Lean.DataSource.DataBento.Models.Events; /// /// Provides data for the symbol-mapping confirmation event. diff --git a/QuantConnect.DataBento/Models/LevelOneBookLevel.cs b/QuantConnect.DataBento/Models/LevelOneBookLevel.cs index 3eb51f5..2e46737 100644 --- a/QuantConnect.DataBento/Models/LevelOneBookLevel.cs +++ b/QuantConnect.DataBento/Models/LevelOneBookLevel.cs @@ -20,12 +20,12 @@ public sealed class LevelOneBookLevel /// /// Bid price at this book level. /// - public decimal BidPx { get; set; } + public decimal? BidPx { get; set; } /// /// Ask price at this book level. /// - public decimal AskPx { get; set; } + public decimal? AskPx { get; set; } /// /// Total bid size at this level. diff --git a/QuantConnect.DataBento/Models/LevelOneData.cs b/QuantConnect.DataBento/Models/LevelOneData.cs index cc68dd4..db70204 100644 --- a/QuantConnect.DataBento/Models/LevelOneData.cs +++ b/QuantConnect.DataBento/Models/LevelOneData.cs @@ -18,7 +18,7 @@ namespace QuantConnect.Lean.DataSource.DataBento.Models; /// /// Represents a level-one market data update containing best bid and ask information. /// -public sealed class LevelOneData : MarketDataRecord +public sealed class LevelOneData : MarketDataBase { /// /// The capture-server-received timestamp expressed as the number of nanoseconds since the UNIX epoch. diff --git a/QuantConnect.DataBento/Models/Live/SymbolMappingMessage.cs b/QuantConnect.DataBento/Models/Live/SymbolMappingMessage.cs index f0ffdfc..738c5d5 100644 --- a/QuantConnect.DataBento/Models/Live/SymbolMappingMessage.cs +++ b/QuantConnect.DataBento/Models/Live/SymbolMappingMessage.cs @@ -16,7 +16,7 @@ namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; -public class SymbolMappingMessage : MarketDataRecord +public class SymbolMappingMessage : MarketDataBase { /// /// The input symbol from the subscription diff --git a/QuantConnect.DataBento/Models/Live/HeartbeatMessage.cs b/QuantConnect.DataBento/Models/Live/SystemMessage.cs similarity index 88% rename from QuantConnect.DataBento/Models/Live/HeartbeatMessage.cs rename to QuantConnect.DataBento/Models/Live/SystemMessage.cs index 4f1b5b7..73499eb 100644 --- a/QuantConnect.DataBento/Models/Live/HeartbeatMessage.cs +++ b/QuantConnect.DataBento/Models/Live/SystemMessage.cs @@ -16,7 +16,7 @@ namespace QuantConnect.Lean.DataSource.DataBento.Models.Live; -public sealed class HeartbeatMessage : MarketDataRecord +public sealed class SystemMessage : MarketDataBase { - public required string Msg { get; set; } + public string Msg { get; set; } } diff --git a/QuantConnect.DataBento/Models/MarketDataRecord.cs b/QuantConnect.DataBento/Models/MarketDataBase.cs similarity index 85% rename from QuantConnect.DataBento/Models/MarketDataRecord.cs rename to QuantConnect.DataBento/Models/MarketDataBase.cs index 31501d2..4cbe37a 100644 --- a/QuantConnect.DataBento/Models/MarketDataRecord.cs +++ b/QuantConnect.DataBento/Models/MarketDataBase.cs @@ -14,17 +14,19 @@ */ using Newtonsoft.Json; +using QuantConnect.Lean.DataSource.DataBento.Converters; namespace QuantConnect.Lean.DataSource.DataBento.Models; /// /// Base class for all market data records containing a standard metadata header. /// -public abstract class MarketDataRecord +[JsonConverter(typeof(DataConverter))] +public abstract class MarketDataBase { /// /// Gets or sets the standard metadata header for this market data record. /// [JsonProperty("hd")] - public required Header Header { get; set; } + public Header Header { get; set; } } diff --git a/QuantConnect.DataBento/Models/OhlcvBar.cs b/QuantConnect.DataBento/Models/OpenHighLowCloseVolumeData.cs similarity index 96% rename from QuantConnect.DataBento/Models/OhlcvBar.cs rename to QuantConnect.DataBento/Models/OpenHighLowCloseVolumeData.cs index a7ce501..e42d2a5 100644 --- a/QuantConnect.DataBento/Models/OhlcvBar.cs +++ b/QuantConnect.DataBento/Models/OpenHighLowCloseVolumeData.cs @@ -19,7 +19,7 @@ namespace QuantConnect.Lean.DataSource.DataBento.Models; /// Open-High-Low-Close-Volume (OHLCV) bar representing aggregated market data /// for a specific instrument and time interval. /// -public sealed class OhlcvBar : MarketDataRecord +public sealed class OpenHighLowCloseVolumeData : MarketDataBase { /// /// Opening price of the bar. diff --git a/QuantConnect.DataBento/Models/StatisticsData.cs b/QuantConnect.DataBento/Models/StatisticsData.cs index 749127e..11c73f3 100644 --- a/QuantConnect.DataBento/Models/StatisticsData.cs +++ b/QuantConnect.DataBento/Models/StatisticsData.cs @@ -17,7 +17,7 @@ namespace QuantConnect.Lean.DataSource.DataBento.Models; -public sealed class StatisticsData : MarketDataRecord +public sealed class StatisticsData : MarketDataBase { /// /// Quantity or value associated with the statistic. diff --git a/QuantConnect.DataBento/Serialization/JsonSettings.cs b/QuantConnect.DataBento/Serialization/JsonSettings.cs deleted file mode 100644 index 4badd80..0000000 --- a/QuantConnect.DataBento/Serialization/JsonSettings.cs +++ /dev/null @@ -1,45 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -using Newtonsoft.Json; - -namespace QuantConnect.Lean.DataSource.DataBento.Serialization; - -/// -/// Provides globally accessible instances of -/// preconfigured with custom contract resolvers, such as snake-case formatting. -/// -public static class JsonSettings -{ - /// - /// Gets a reusable instance of that uses - /// for snake-case property name formatting. - /// - public static readonly JsonSerializerSettings SnakeCase = new() - { - ContractResolver = SnakeCaseContractResolver.Instance - }; - - /// - /// Gets a reusable instance of that uses - /// for snake-case property name formatting - /// and custom converter. - /// - public static readonly JsonSerializerSettings LiveDataSnakeCase = new() - { - ContractResolver = SnakeCaseContractResolver.Instance, - Converters = { new Converters.LiveDataConverter() } - }; -} From 0afd949b94c393eaf8e5b469d80f246b05220367 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 30 Jan 2026 15:03:36 +0200 Subject: [PATCH 15/18] feat: improve DataBento API error handling and test coverage - Add ErrorResponse model for structured API error parsing - Refactor HistoricalAPIClient to auto-correct invalid date ranges - Expand unit tests for error scenarios and data bounds - Add tests for error response deserialization - Support additional OHLCV record types in DataConverter - Use snake-case JSON settings for consistent deserialization - Improve logging and handling of empty/malformed responses --- .../DataBentoHistoricalApiClientTests.cs | 56 +++++++-- .../DataBentoJsonConverterTests.cs | 81 +++++++++++++ .../Api/HistoricalAPIClient.cs | 84 ++++++++++--- .../Converters/DataConverter.cs | 3 + QuantConnect.DataBento/Extensions.cs | 11 +- .../Models/ErrorResponse.cs | 110 ++++++++++++++++++ .../Serialization/JsonSettings.cs | 34 ++++++ 7 files changed, 350 insertions(+), 29 deletions(-) create mode 100644 QuantConnect.DataBento/Models/ErrorResponse.cs create mode 100644 QuantConnect.DataBento/Serialization/JsonSettings.cs diff --git a/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs index 8c5ee9a..eb4d6a9 100644 --- a/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs @@ -14,8 +14,10 @@ */ using System; +using System.Linq; using NUnit.Framework; using QuantConnect.Logging; +using System.Collections.Generic; using QuantConnect.Configuration; using QuantConnect.Lean.DataSource.DataBento.Api; @@ -47,16 +49,43 @@ public void OneTimeSetUp() _client = new HistoricalAPIClient(apiKey); } - [TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Daily)] - [TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Hour)] - [TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Minute)] - [TestCase("ESH6", "2025/01/11", "2026/01/20", Resolution.Second)] - [TestCase("ESH6 C6875", "2026/01/11", "2026/01/20", Resolution.Daily)] - public void CanInitializeHistoricalApiClient(string ticker, DateTime startDate, DateTime endDate, Resolution resolution) + private static IEnumerable GetTestCases + { + get + { + var todayDate = DateTime.UtcNow.Date; + var tomorrowDate = todayDate.AddDays(1); + + var ESH6 = Securities.Futures.Indices.SP500EMini + "H6"; // March 2026 future contract symbol + + yield return new TestCaseData(ESH6, new DateTime(2010, 01, 11), tomorrowDate, Resolution.Daily, true) + .SetArgDisplayNames("Invalid: Data Start/End Before/After Available Start/End"); + yield return new TestCaseData(ESH6, new DateTime(2010, 01, 01), new DateTime(2015, 01, 01), Resolution.Daily, false) + .SetArgDisplayNames("Invalid: Data Start Before Available Start"); + yield return new TestCaseData(ESH6, new DateTime(2010, 06, 06), tomorrowDate, Resolution.Daily, true) + .SetArgDisplayNames("Invalid: Data End After Available End"); + yield return new TestCaseData(ESH6, new DateTime(2020, 01, 11), new DateTime(2026, 01, 20), Resolution.Hour, true) + .SetArgDisplayNames("Valid: Hourly Data Within Available Range (2020/01/11 - 2026/01/20)"); + yield return new TestCaseData(ESH6, new DateTime(2026, 01, 11), new DateTime(2026, 01, 20), Resolution.Minute, true) + .SetArgDisplayNames("Valid: Minute Data Within Available Range"); + yield return new TestCaseData(ESH6, new DateTime(2025, 01, 11), new DateTime(2026, 01, 20), Resolution.Second, true) + .SetArgDisplayNames("Valid: Second Data Within Available Range"); + yield return new TestCaseData(ESH6, new DateTime(2026, 01, 24, 2, 10, 0), new DateTime(2026, 01, 24, 2, 12, 00), Resolution.Minute, false) + .SetArgDisplayNames("Invalid: Minute Data at Saturday"); + + var ESH6_C6875 = Securities.Futures.Indices.SP500EMini + "H6 C6875"; // March 2026 future contract call option symbol + yield return new TestCaseData(ESH6_C6875, new DateTime(2010, 01, 11), new DateTime(2026, 01, 20), Resolution.Daily, true) + .SetArgDisplayNames("Invalid: Data Start/End Before/After Available Start/End"); + } + } + + [TestCaseSource(nameof(GetTestCases))] + public void GetHistoricalOHLCVBars(string ticker, DateTime startDate, DateTime endDate, Resolution resolution, bool isDataReceived) { var dataCounter = 0; var previousEndTime = DateTime.MinValue; - foreach (var data in _client.GetHistoricalOhlcvBars(ticker, startDate, endDate, resolution, Dataset)) + var bars = _client.GetHistoricalOhlcvBars(ticker, startDate, endDate, resolution, Dataset).ToList(); + foreach (var data in bars) { Assert.IsNotNull(data); @@ -74,8 +103,15 @@ public void CanInitializeHistoricalApiClient(string ticker, DateTime startDate, dataCounter++; } - Log.Trace($"{nameof(CanInitializeHistoricalApiClient)}: {ticker} | [{startDate} - {endDate}] | {resolution} = {dataCounter} (bars)"); - Assert.Greater(dataCounter, 0); + Log.Trace($"{nameof(GetHistoricalOHLCVBars)}: {ticker} | [{startDate} - {endDate}] | {resolution} = {dataCounter} (bars)"); + if (isDataReceived) + { + Assert.Greater(dataCounter, 0); + } + else + { + Assert.AreEqual(0, dataCounter); + } } [TestCase("ESH6 C6875", "2026/01/11", "2026/01/20", Resolution.Daily)] @@ -97,7 +133,7 @@ public void ShouldFetchOpenInterest(string ticker, DateTime startDate, DateTime dataCounter++; } - Log.Trace($"{nameof(CanInitializeHistoricalApiClient)}: {ticker} | [{startDate} - {endDate}] | {resolution} = {dataCounter} (bars)"); + Log.Trace($"{nameof(GetHistoricalOHLCVBars)}: {ticker} | [{startDate} - {endDate}] | {resolution} = {dataCounter} (bars)"); Assert.Greater(dataCounter, 0); } diff --git a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs index 3ac629f..cb98ec7 100644 --- a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs @@ -14,6 +14,7 @@ */ using System; +using System.Linq; using Newtonsoft.Json; using NUnit.Framework; using System.Collections.Generic; @@ -378,4 +379,84 @@ public void DeserializeUnknownJsonFormatShouldThrow() var json2 = @"{ ""hd"": { ""rtype"": 9999 } }"; Assert.Throws(() => json2.DeserializeObject()); } + + private static IEnumerable ErrorResponses + { + get + { + yield return new TestCaseData(@"{ + ""detail"": { + ""case"": ""data_end_after_available_end"", + ""message"": ""The dataset GLBX.MDP3 has data available up to '2026-01-29 20:00:00+00:00'. The `end` in the query ('2026-01-29 20:23:08.167605900+00:00') is after the available range. Try requesting with an earlier `end`."", + ""status_code"": 422, + ""docs"": ""https://databento.com/docs/api-reference-historical/basics/datasets"", + ""payload"": { + ""dataset"": ""GLBX.MDP3"", + ""start"": ""2026-01-28T18:00:00.000000000Z"", + ""end"": ""2026-01-29T20:23:08.167605900Z"", + ""available_start"": ""2010-06-06T00:00:00.000000000Z"", + ""available_end"": ""2026-01-29T20:00:00.000000000Z"" + } + } +}").SetArgDisplayNames("DataEndAfterAvailableEnd"); + + yield return new TestCaseData(@"{ + ""detail"": { + ""case"": ""data_start_before_available_start"", + ""message"": ""`start` in query ('2010-01-11 00:00:00+00:00') was before the available start of dataset GLBX.MDP3 ('2010-06-06 00:00:00+00:00'). Try requesting with a later `start`."", + ""status_code"": 422, + ""docs"": ""https://databento.com/docs/api-reference-historical/basics/datasets"", + ""payload"": { + ""dataset"": ""GLBX.MDP3"", + ""start"": ""2010-01-11T00:00:00.000000000Z"", + ""end"": ""2026-01-30T00:00:00.000000000Z"", + ""available_start"": ""2010-06-06T00:00:00.000000000Z"", + ""available_end"": ""2026-01-29T00:00:00.000000000Z"" + } + } +}").SetArgDisplayNames("DataStartBeforeAvailableStart"); + + yield return new TestCaseData(@"{ + ""detail"": { + ""case"": ""data_time_range_start_on_or_after_end"", + ""message"": ""Invalid time range query, `start` 2026-01-30 00:10:00+00:00 cannot be on or after `end` 2026-01-30 00:05:00+00:00."", + ""status_code"": 422, + ""docs"": ""https://databento.com/docs/standards-and-conventions/common-fields-enums-types"", + ""payload"": { + ""start"": ""2026-01-30T00:10:00.000000000Z"", + ""end"": ""2026-01-30T00:05:00.000000000Z"" + } + } +}").SetArgDisplayNames("DataTimeRangeStartOnOrAfterEnd"); + } + } + + [TestCaseSource(nameof(ErrorResponses))] + public void DeserializeErrorResponse(string json) + { + var error = json.DeserializeObject(); + + Assert.IsNotNull(error); + Assert.IsNotNull(error.Detail); + Assert.That(error.Detail.Case, Is.Not.Null.And.Not.Empty); + var validCases = new[] + { + "data_end_after_available_end", + "data_start_before_available_start" + }; + Assert.IsTrue(validCases.Any(x => x.Equals(error.Detail.Case, StringComparison.InvariantCultureIgnoreCase))); + + Assert.That(error.Detail.Message, Is.Not.Null.And.Not.Empty); + Assert.Greater(error.Detail.StatusCode, 0); + Assert.That(error.Detail.Docs, Is.Not.Null.And.Not.Empty); + + Assert.IsNotNull(error.Detail.Payload); + Assert.AreEqual(422, error.Detail.StatusCode); + Assert.AreEqual("GLBX.MDP3", error.Detail.Payload.Dataset); + + Assert.AreNotEqual(default(DateTime), error.Detail.Payload.Start); + Assert.AreNotEqual(default(DateTime), error.Detail.Payload.End); + Assert.AreNotEqual(default(DateTime), error.Detail.Payload.AvailableStart); + Assert.AreNotEqual(default(DateTime), error.Detail.Payload.AvailableEnd); + } } diff --git a/QuantConnect.DataBento/Api/HistoricalAPIClient.cs b/QuantConnect.DataBento/Api/HistoricalAPIClient.cs index d420a88..67510f0 100644 --- a/QuantConnect.DataBento/Api/HistoricalAPIClient.cs +++ b/QuantConnect.DataBento/Api/HistoricalAPIClient.cs @@ -83,7 +83,6 @@ private IEnumerable GetRange(string symbol, DateTime startDateTimeUtc, Dat var formData = new Dictionary { { "dataset", dataSet }, - { "end", Time.DateTimeToUnixTimeStampNanoseconds(endDateTimeUtc).ToStringInvariant() }, { "symbols", symbol }, { "schema", schema }, { "encoding", "json" }, @@ -97,10 +96,12 @@ private IEnumerable GetRange(string symbol, DateTime startDateTimeUtc, Dat } var start = startDateTimeUtc; + var end = endDateTimeUtc; var httpStatusCode = default(HttpStatusCode); do { formData["start"] = Time.DateTimeToUnixTimeStampNanoseconds(start).ToStringInvariant(); + formData["end"] = Time.DateTimeToUnixTimeStampNanoseconds(end).ToStringInvariant(); using var content = new FormUrlEncodedContent(formData); @@ -111,19 +112,13 @@ private IEnumerable GetRange(string symbol, DateTime startDateTimeUtc, Dat using var response = _httpClient.Send(requestMessage); - if (response.Headers.TryGetValues("x-warning", out var warnings)) - { - foreach (var warning in warnings) - { - Log.Trace($"{nameof(HistoricalAPIClient)}.{nameof(GetRange)}: {warning}"); - } - } + LogWarnings(response); using var stream = response.Content.ReadAsStream(); if (stream.Length == 0) { - continue; + yield break; } using var reader = new StreamReader(stream); @@ -132,26 +127,79 @@ private IEnumerable GetRange(string symbol, DateTime startDateTimeUtc, Dat if (response.StatusCode == HttpStatusCode.UnprocessableContent) { line = reader.ReadLine(); - Log.Trace($"{nameof(HistoricalAPIClient)}.{nameof(GetRange)}.Response: {line}. " + - $"Request: [{response.RequestMessage?.Method}]({response.RequestMessage?.RequestUri}), " + - $"Payload: {string.Join(", ", formData.Select(kvp => $"{kvp.Key}: {kvp.Value}"))}"); - yield break; + if (line == null) + { + yield break; + } + + var error = line.DeserializeObject(); + + switch (error?.Detail?.Case) + { + case ErrorCases.DataStartBeforeAvailableStart: + start = error.Detail.Payload.AvailableStart.UtcDateTime; + if (end > error.Detail.Payload.AvailableEnd.UtcDateTime) + { + end = error.Detail.Payload.AvailableEnd.UtcDateTime; + + } + Log.Trace($"{nameof(HistoricalAPIClient)}.{nameof(GetRange)}: {ErrorCases.DataStartBeforeAvailableStart}, " + + $"Start {startDateTimeUtc:O}->{start:O}, End {endDateTimeUtc:O}->{end:O}"); + continue; + case ErrorCases.DataEndAfterAvailableEnd: + end = error.Detail.Payload.AvailableEnd.UtcDateTime; + var startBound = end - endDateTimeUtc.Subtract(startDateTimeUtc); + start = startBound < error.Detail.Payload.AvailableStart.UtcDateTime + ? error.Detail.Payload.AvailableStart.UtcDateTime + : startBound; + Log.Trace($"{nameof(HistoricalAPIClient)}.{nameof(GetRange)}: {ErrorCases.DataEndAfterAvailableEnd}, " + + $"Start {startDateTimeUtc:O}->{start:O}, End {endDateTimeUtc:O}->{end:O}"); + continue; + case ErrorCases.DataTimeRangeStartOnOrAfterEnd: + Log.Error($"{nameof(HistoricalAPIClient)}.{nameof(GetRange)}: {error.Detail.Message}"); + yield break; + default: + Log.Trace($"{nameof(HistoricalAPIClient)}.{nameof(GetRange)}.Response: {line}. " + + $"Request: [{response.RequestMessage?.Method}]({response.RequestMessage?.RequestUri}), " + + $"Payload: {string.Join(", ", formData.Select(kvp => $"{kvp.Key}: {kvp.Value}"))}"); + yield break; + } } httpStatusCode = response.EnsureSuccessStatusCode().StatusCode; - var data = default(T); + var lastEmitted = default(T); while ((line = reader.ReadLine()) != null) { - data = line.DeserializeObject(); - yield return data; + lastEmitted = line.DeserializeObject(); + + if (lastEmitted == null) + { + continue; + } + + yield return lastEmitted; } - start = data.Header.UtcTime.AddTicks(1); - } while (httpStatusCode == HttpStatusCode.PartialContent); + // Advance start by one tick to move the time window forward without duplication. + // The API range is inclusive, so this ensures the next request starts + // strictly after the last emitted record and avoids re-fetching it. + start = lastEmitted!.Header.UtcTime.AddTicks(1); + } while (httpStatusCode != HttpStatusCode.OK); } public void Dispose() { _httpClient?.DisposeSafely(); } + + private static void LogWarnings(HttpResponseMessage response) + { + if (response.Headers.TryGetValues("x-warning", out var warnings)) + { + foreach (var warning in warnings) + { + Log.Trace($"{nameof(HistoricalAPIClient)}.{nameof(LogWarnings)}: {warning}"); + } + } + } } diff --git a/QuantConnect.DataBento/Converters/DataConverter.cs b/QuantConnect.DataBento/Converters/DataConverter.cs index 21067ab..ae98f46 100644 --- a/QuantConnect.DataBento/Converters/DataConverter.cs +++ b/QuantConnect.DataBento/Converters/DataConverter.cs @@ -81,6 +81,9 @@ public override MarketDataBase ReadJson(JsonReader reader, Type objectType, Mark switch (recordType) { case RecordType.OpenHighLowCloseVolume1Day: + case RecordType.OpenHighLowCloseVolume1Hour: + case RecordType.OpenHighLowCloseVolume1Minute: + case RecordType.OpenHighLowCloseVolume1Second: marketDataBase = new OpenHighLowCloseVolumeData(); break; case RecordType.MarketByPriceDepth1: diff --git a/QuantConnect.DataBento/Extensions.cs b/QuantConnect.DataBento/Extensions.cs index 62f30db..9e1f8ae 100644 --- a/QuantConnect.DataBento/Extensions.cs +++ b/QuantConnect.DataBento/Extensions.cs @@ -15,6 +15,7 @@ using Newtonsoft.Json; using QuantConnect.Logging; +using QuantConnect.Lean.DataSource.DataBento.Serialization; namespace QuantConnect.Lean.DataSource.DataBento; @@ -29,6 +30,14 @@ public static class Extensions /// The deserialized object of type . public static T? DeserializeObject(this string json) { - return JsonConvert.DeserializeObject(json); + try + { + return JsonConvert.DeserializeObject(json, JsonSettings.SnakeCase); + } + catch (Exception ex) + { + Log.Error(ex, $"Failed to deserialize JSON into {typeof(T).Name}. JSON: {json}"); + throw; + } } } diff --git a/QuantConnect.DataBento/Models/ErrorResponse.cs b/QuantConnect.DataBento/Models/ErrorResponse.cs new file mode 100644 index 0000000..8b775a8 --- /dev/null +++ b/QuantConnect.DataBento/Models/ErrorResponse.cs @@ -0,0 +1,110 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models; + +/// +/// Represents a structured error response returned by the Historical DataBento API. +/// +public sealed class ErrorResponse +{ + /// + /// Detailed information about the error. + /// + public ErrorDetail? Detail { get; set; } +} + +/// +/// Describes a DataBento API error. +/// +public sealed class ErrorDetail +{ + /// + /// Machine-readable error identifier (e.g. data_end_after_available_end). + /// + public string? Case { get; set; } + + /// + /// Human-readable explanation of the error. + /// + public string? Message { get; set; } + + /// + /// HTTP status code associated with the error. + /// + public int StatusCode { get; set; } + + /// + /// Link to the relevant DataBento API documentation. + /// + public string? Docs { get; set; } + + /// + /// Request-specific data providing additional error context. + /// + public ErrorPayload? Payload { get; set; } +} + +/// +/// Contains request parameters and dataset limits related to the error. +/// +public sealed class ErrorPayload +{ + /// + /// Dataset identifier used in the request. + /// + public string Dataset { get; set; } + + /// + /// Requested start timestamp. + /// + public DateTimeOffset Start { get; set; } + + /// + /// Requested end timestamp. + /// + public DateTimeOffset End { get; set; } + + /// + /// Earliest timestamp available for the dataset. + /// + public DateTimeOffset AvailableStart { get; set; } + + /// + /// Latest timestamp available for the dataset. + /// + public DateTimeOffset AvailableEnd { get; set; } +} + +/// +/// Contains known DataBento Historical API error case identifiers. +/// +public static class ErrorCases +{ + /// + /// The requested end timestamp exceeds the dataset's available end. + /// + public const string DataEndAfterAvailableEnd = "data_end_after_available_end"; + + /// + /// The requested start timestamp exceeds the dataset's available start. + /// + public const string DataStartBeforeAvailableStart = "data_start_before_available_start"; + + /// + /// The requested start timestamp is greater than or equal to the requested end timestamp. + /// + public const string DataTimeRangeStartOnOrAfterEnd = "data_time_range_start_on_or_after_end"; +} diff --git a/QuantConnect.DataBento/Serialization/JsonSettings.cs b/QuantConnect.DataBento/Serialization/JsonSettings.cs new file mode 100644 index 0000000..5247d97 --- /dev/null +++ b/QuantConnect.DataBento/Serialization/JsonSettings.cs @@ -0,0 +1,34 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using Newtonsoft.Json; + +namespace QuantConnect.Lean.DataSource.DataBento.Serialization; + +/// +/// Provides globally accessible instances of +/// preconfigured with custom contract resolvers, such as snake-case formatting. +/// +public static class JsonSettings +{ + /// + /// Gets a reusable instance of that uses + /// for snake-case property name formatting. + /// + public static readonly JsonSerializerSettings SnakeCase = new() + { + ContractResolver = SnakeCaseContractResolver.Instance + }; +} From 55fa8d7742cebc5387bd6a914f383d7693904caa Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 30 Jan 2026 17:01:37 +0200 Subject: [PATCH 16/18] refactor: dataset mapping to use DataSetSpecifications Introduce DataSetSpecifications and PredefinedDataSets for richer dataset metadata, including delay info and documentation links. Refactor DataBentoSymbolMapper to map markets to DataSetSpecifications. Update provider logic to log delay warnings and extract DataSetID as needed. Add support for new error case DataStartAfterAvailableEnd and corresponding test. Remove obsolete methods and simplify subscription/history logic. --- .../DataBentoJsonConverterTests.cs | 16 +++ .../Api/HistoricalAPIClient.cs | 6 + .../DataBentoDataProvider.cs | 19 +-- .../DataBentoHistoryProivder.cs | 26 ++-- .../DataBentoSymbolMapper.cs | 17 +-- .../Models/DataSetSpecifications.cs | 129 ++++++++++++++++++ .../Models/ErrorResponse.cs | 5 + 7 files changed, 175 insertions(+), 43 deletions(-) create mode 100644 QuantConnect.DataBento/Models/DataSetSpecifications.cs diff --git a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs index cb98ec7..a26f80b 100644 --- a/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoJsonConverterTests.cs @@ -428,6 +428,22 @@ private static IEnumerable ErrorResponses } } }").SetArgDisplayNames("DataTimeRangeStartOnOrAfterEnd"); + + yield return new TestCaseData(@"{ + ""detail"": { + ""case"": ""data_start_after_available_end"", + ""message"": ""`start` in query ('2026-01-30 13:16:22+00:00') was after the available end of dataset GLBX.MDP3 ('2026-01-30 13:00:00+00:00'). Try requesting with an earlier `start`."", + ""status_code"": 422, + ""docs"": ""https://databento.com/docs/api-reference-historical/basics/datasets"", + ""payload"": { + ""dataset"": ""GLBX.MDP3"", + ""start"": ""2026-01-30T13:16:22.000000000Z"", + ""end"": ""2026-01-30T13:16:32.149178800Z"", + ""available_start"": ""2010-06-06T00:00:00.000000000Z"", + ""available_end"": ""2026-01-30T13:00:00.000000000Z"" + } + } +}").SetArgDisplayNames("DataStartAfterAvailableEnd"); } } diff --git a/QuantConnect.DataBento/Api/HistoricalAPIClient.cs b/QuantConnect.DataBento/Api/HistoricalAPIClient.cs index 67510f0..22402a9 100644 --- a/QuantConnect.DataBento/Api/HistoricalAPIClient.cs +++ b/QuantConnect.DataBento/Api/HistoricalAPIClient.cs @@ -158,6 +158,12 @@ private IEnumerable GetRange(string symbol, DateTime startDateTimeUtc, Dat case ErrorCases.DataTimeRangeStartOnOrAfterEnd: Log.Error($"{nameof(HistoricalAPIClient)}.{nameof(GetRange)}: {error.Detail.Message}"); yield break; + case ErrorCases.DataStartAfterAvailableEnd: + end = error.Detail.Payload.AvailableEnd.UtcDateTime; + start = end - endDateTimeUtc.Subtract(startDateTimeUtc); + Log.Trace($"{nameof(HistoricalAPIClient)}.{nameof(GetRange)}: {ErrorCases.DataStartAfterAvailableEnd}, " + + $"Start {startDateTimeUtc:O}->{start:O}, End {endDateTimeUtc:O}->{end:O}"); + continue; default: Log.Trace($"{nameof(HistoricalAPIClient)}.{nameof(GetRange)}.Response: {line}. " + $"Request: [{response.RequestMessage?.Method}]({response.RequestMessage?.RequestUri}), " + diff --git a/QuantConnect.DataBento/DataBentoDataProvider.cs b/QuantConnect.DataBento/DataBentoDataProvider.cs index a704ac5..8456fc1 100644 --- a/QuantConnect.DataBento/DataBentoDataProvider.cs +++ b/QuantConnect.DataBento/DataBentoDataProvider.cs @@ -157,21 +157,6 @@ private void HandleLevelOneData(LevelOneData levelOneData) } } - /// - /// Attempts to resolve the DataBento dataset for the specified symbol based on its Lean market. - /// - /// The symbol whose market is used to determine the DataBento dataset. - /// - /// When this method returns true, contains the resolved DataBento dataset; otherwise, null. - /// - /// - /// true if a DataBento dataset mapping exists for the symbol's market; otherwise, false. - /// - private bool TryGetDataBentoDataSet(Symbol symbol, out string? dataSet) - { - return _symbolMapper.DataBentoDataSetByLeanMarket.TryGetValue(symbol.ID.Market, out dataSet); - } - /// /// Logic to subscribe to the specified symbols /// @@ -179,7 +164,7 @@ public bool Subscribe(IEnumerable symbols) { foreach (var symbol in symbols) { - if (!TryGetDataBentoDataSet(symbol, out var dataSet)) + if (!_symbolMapper.DataBentoDataSetByLeanMarket.TryGetValue(symbol.ID.Market, out var dataSetSpecifications)) { throw new ArgumentException($"No DataBento dataset mapping found for symbol {symbol} in market {symbol.ID.Market}. Cannot subscribe."); } @@ -188,7 +173,7 @@ public bool Subscribe(IEnumerable symbols) _pendingSubscriptions[brokerageSymbol] = symbol; - _liveApiClient.Subscribe(dataSet, brokerageSymbol); + _liveApiClient.Subscribe(dataSetSpecifications.DataSetID, brokerageSymbol); } return true; diff --git a/QuantConnect.DataBento/DataBentoHistoryProivder.cs b/QuantConnect.DataBento/DataBentoHistoryProivder.cs index 09920eb..d2618d4 100644 --- a/QuantConnect.DataBento/DataBentoHistoryProivder.cs +++ b/QuantConnect.DataBento/DataBentoHistoryProivder.cs @@ -118,7 +118,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) return null; } - if (!TryGetDataBentoDataSet(historyRequest.Symbol, out var dataSet) || dataSet == null) + if (!_symbolMapper.DataBentoDataSetByLeanMarket.TryGetValue(historyRequest.Symbol.ID.Market, out var dataSetSpecifications) || dataSetSpecifications == null) { if (!_dataBentoDatasetErrorFired) { @@ -130,6 +130,12 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) return null; } + if (dataSetSpecifications.TryGetDelayWarningMessage(out var message)) + { + Log.Trace(message); + } + var dataSet = dataSetSpecifications.DataSetID; + var history = default(IEnumerable); var brokerageSymbol = _symbolMapper.GetBrokerageSymbol(historyRequest.Symbol); switch (historyRequest.TickType) @@ -155,23 +161,7 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) return null; } - return FilterHistory(history, historyRequest, historyRequest.StartTimeLocal, historyRequest.EndTimeLocal); - } - - private static IEnumerable FilterHistory(IEnumerable history, HistoryRequest request, DateTime startTimeLocal, DateTime endTimeLocal) - { - // cleaning the data before returning it back to user - foreach (var bar in history) - { - if (bar.Time >= startTimeLocal && bar.EndTime <= endTimeLocal) - { - if (request.ExchangeHours.IsOpen(bar.Time, bar.EndTime, request.IncludeExtendedMarketHours)) - { - Interlocked.Increment(ref _dataPointCount); - yield return bar; - } - } - } + return history; } private IEnumerable GetOpenInterestBars(HistoryRequest request, string brokerageSymbol, string dataBentoDataSet) diff --git a/QuantConnect.DataBento/DataBentoSymbolMapper.cs b/QuantConnect.DataBento/DataBentoSymbolMapper.cs index a0b8913..f17546e 100644 --- a/QuantConnect.DataBento/DataBentoSymbolMapper.cs +++ b/QuantConnect.DataBento/DataBentoSymbolMapper.cs @@ -15,6 +15,7 @@ using QuantConnect.Brokerages; using System.Collections.Frozen; +using QuantConnect.Lean.DataSource.DataBento.Models; namespace QuantConnect.Lean.DataSource.DataBento; @@ -27,17 +28,17 @@ public class DataBentoSymbolMapper : ISymbolMapper /// Dataset for CME Globex futures /// https://databento.com/docs/venues-and-datasets has more information on datasets through DataBento /// - public FrozenDictionary DataBentoDataSetByLeanMarket = new Dictionary + public FrozenDictionary DataBentoDataSetByLeanMarket = new Dictionary { - { Market.EUREX, "XEUR.EOBI" }, + { Market.EUREX, PredefinedDataSets.XEUR_EOBI }, - { Market.CBOT, "GLBX.MDP3" }, - { Market.CME, "GLBX.MDP3" }, - { Market.COMEX, "GLBX.MDP3" }, - { Market.NYMEX, "GLBX.MDP3" }, + { Market.CBOT, PredefinedDataSets.GLBX_MDP3 }, + { Market.CME, PredefinedDataSets.GLBX_MDP3 }, + { Market.COMEX, PredefinedDataSets.GLBX_MDP3 }, + { Market.NYMEX, PredefinedDataSets.GLBX_MDP3 }, - { Market.ICE, "IFUS.IMPACT" }, - { Market.NYSELIFFE, "IFUS.IMPACT" } + { Market.ICE, PredefinedDataSets.IFUS_IMPACT }, + { Market.NYSELIFFE, PredefinedDataSets.IFUS_IMPACT } }.ToFrozenDictionary(); /// diff --git a/QuantConnect.DataBento/Models/DataSetSpecifications.cs b/QuantConnect.DataBento/Models/DataSetSpecifications.cs new file mode 100644 index 0000000..1782043 --- /dev/null +++ b/QuantConnect.DataBento/Models/DataSetSpecifications.cs @@ -0,0 +1,129 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2026 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Lean.DataSource.DataBento.Models; + +/// +/// Provides pre-defined historical dataset specifications for easy access. +/// +public static class PredefinedDataSets +{ + /// + /// Historical dataset: XEUR.EOBI + /// + public static readonly DataSetSpecifications XEUR_EOBI = new( + "XEUR.EOBI", + "15 mins", + "Next day 00:00 CET/CEST", + "https://databento.com/catalog/xeur/XEUR.EOBI" + ); + + /// + /// Historical dataset: GLBX.MDP3 + /// + public static readonly DataSetSpecifications GLBX_MDP3 = new( + "GLBX.MDP3", + "20 minutes", + "8 hours", + "https://databento.com/catalog/cme/GLBX.MDP3" + ); + + /// + /// Historical dataset: IFUS.IMPACT + /// + public static readonly DataSetSpecifications IFUS_IMPACT = new( + "IFUS.IMPACT", + "15 minutes", + "Same day 18:05 EST/EDT", + "https://databento.com/catalog/ifus/IFUS.IMPACT"); +} + +/// +/// Represents specifications and metadata for a data set, including historical delay and access details. +/// +public class DataSetSpecifications +{ + /// + /// Internal flag to ensure the delay warning message is only generated once. + /// + private bool _delayWarningFired; + + /// + /// The unique identifier or name of the dataset. + /// + public string DataSetID { get; } + + /// + /// User-friendly historical delay string for datasets requiring a license. + /// Examples: "15 mins", "Delayed 20 minutes". + /// + public string HistoricalDelayWithLicense { get; } + + /// + /// User-friendly historical delay string for datasets NOT requiring a license. + /// Examples: "8 hours", "Next day 00:00 CET/CEST". + /// + public string HistoricalDelayWithoutLicense { get; } + + /// + /// Optional link to the dataset specifications page. + /// + public string Link { get; } + + /// + /// Initializes a new instance of the class + /// with dataset name and historical delay information for users with and without a license. + /// + /// The unique identifier or name of the dataset. + /// + /// The historical delay for users who have a live license. + /// Example: "15 minutes". + /// + /// + /// The historical delay for users who do NOT have a license. + /// Example: "8 hours" or "Next day 00:00 CET/CEST". + /// + public DataSetSpecifications(string dataSetID, string historicalDelayWithLicense, string historicalDelayWithoutLicense, string link) + { + DataSetID = dataSetID; + HistoricalDelayWithLicense = historicalDelayWithLicense; + HistoricalDelayWithoutLicense = historicalDelayWithoutLicense; + Link = link; + } + + /// + /// Attempts to generate a user-friendly delay warning message. + /// The message will only be returned the first time this method is called. + /// + /// Outputs the generated warning message if not previously fired; otherwise null. + /// True if the message was generated; false if it was already generated before. + public bool TryGetDelayWarningMessage(out string? message) + { + message = null; + if (_delayWarningFired) + { + return false; + } + message = $"Dataset [{DataSetID}] historical data information:\n" + + $"- Users with a live license: delayed by approximately {HistoricalDelayWithLicense}." + + $"For access to more recent data, use the intraday replay feature of the live data client.\n" + + $"- Users without a license: delayed by {HistoricalDelayWithoutLicense}.\n" + + $"- More information: {Link} (go to the 'Specifications' tab)"; + + _delayWarningFired = true; + return true; + } +} diff --git a/QuantConnect.DataBento/Models/ErrorResponse.cs b/QuantConnect.DataBento/Models/ErrorResponse.cs index 8b775a8..2aee33a 100644 --- a/QuantConnect.DataBento/Models/ErrorResponse.cs +++ b/QuantConnect.DataBento/Models/ErrorResponse.cs @@ -103,6 +103,11 @@ public static class ErrorCases /// public const string DataStartBeforeAvailableStart = "data_start_before_available_start"; + /// + /// The requested start timestamp is after the dataset's available end. + /// + public const string DataStartAfterAvailableEnd = "data_start_after_available_end"; + /// /// The requested start timestamp is greater than or equal to the requested end timestamp. /// From c9a5c817b3b3407db4d9d73e75b1c0b544d4d897 Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 30 Jan 2026 18:34:59 +0200 Subject: [PATCH 17/18] refactor: schema usage and add large-range tests Replaced hardcoded schema strings with documented constants in HistoricalAPIClient for maintainability. Refactored GetRange to set record limits by schema, preventing timeouts on large requests. Added tests for 11-year minute and second data ranges. Updated all schema usages to use new constants. --- .../DataBentoHistoricalApiClientTests.cs | 4 ++ .../Api/HistoricalAPIClient.cs | 61 ++++++++++++++++--- 2 files changed, 56 insertions(+), 9 deletions(-) diff --git a/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs b/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs index eb4d6a9..cf63ca6 100644 --- a/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs +++ b/QuantConnect.DataBento.Tests/DataBentoHistoricalApiClientTests.cs @@ -72,6 +72,10 @@ private static IEnumerable GetTestCases .SetArgDisplayNames("Valid: Second Data Within Available Range"); yield return new TestCaseData(ESH6, new DateTime(2026, 01, 24, 2, 10, 0), new DateTime(2026, 01, 24, 2, 12, 00), Resolution.Minute, false) .SetArgDisplayNames("Invalid: Minute Data at Saturday"); + yield return new TestCaseData(ESH6, new DateTime(2015, 01, 01), new DateTime(2026, 01, 29), Resolution.Minute, true) + .SetArgDisplayNames("Valid: Minute Data within 11 years range"); + yield return new TestCaseData(ESH6, new DateTime(2015, 01, 01), new DateTime(2026, 01, 29), Resolution.Second, true) + .SetArgDisplayNames("Valid: Second Data within 11 years range"); var ESH6_C6875 = Securities.Futures.Indices.SP500EMini + "H6 C6875"; // March 2026 future contract call option symbol yield return new TestCaseData(ESH6_C6875, new DateTime(2010, 01, 11), new DateTime(2026, 01, 20), Resolution.Daily, true) diff --git a/QuantConnect.DataBento/Api/HistoricalAPIClient.cs b/QuantConnect.DataBento/Api/HistoricalAPIClient.cs index 22402a9..4a1d213 100644 --- a/QuantConnect.DataBento/Api/HistoricalAPIClient.cs +++ b/QuantConnect.DataBento/Api/HistoricalAPIClient.cs @@ -24,6 +24,42 @@ namespace QuantConnect.Lean.DataSource.DataBento.Api; public class HistoricalAPIClient : IDisposable { + /// + /// OHLCV bars aggregated to 1-second intervals. + /// + /// Docs: + private const string OHLCV1sSchema = "ohlcv-1s"; + + /// + /// OHLCV bars aggregated to 1-minute intervals. + /// + /// Docs: + private const string OHLCV1mSchema = "ohlcv-1m"; + + /// + /// OHLCV bars aggregated to 1-hour intervals. + /// + /// Docs: + private const string OHLCV1hSchema = "ohlcv-1h"; + + /// + /// OHLCV bars aggregated to 1-day intervals. + /// + /// Docs: + private const string OHLCV1dSchema = "ohlcv-1d"; + + /// + /// Market-by-price data with top-of-book depth (MBP-1). + /// + /// Docs: + private const string MBP1Schema = "mbp-1"; + + /// + /// Market statistics data (e.g. volume, trades, session statistics). + /// + /// Docs: + private const string StatisticsSchema = "statistics"; + private readonly HttpClient _httpClient = new() { BaseAddress = new Uri("https://hist.databento.com") @@ -44,16 +80,16 @@ public IEnumerable GetHistoricalOhlcvBars(string sym switch (resolution) { case Resolution.Second: - schema = "ohlcv-1s"; + schema = OHLCV1sSchema; break; case Resolution.Minute: - schema = "ohlcv-1m"; + schema = OHLCV1mSchema; break; case Resolution.Hour: - schema = "ohlcv-1h"; + schema = OHLCV1hSchema; break; case Resolution.Daily: - schema = "ohlcv-1d"; + schema = OHLCV1dSchema; break; default: throw new ArgumentException($"Unsupported resolution {resolution} for OHLCV data."); @@ -64,12 +100,12 @@ public IEnumerable GetHistoricalOhlcvBars(string sym public IEnumerable GetTickBars(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, string dataSet) { - return GetRange(symbol, startDateTimeUtc, endDateTimeUtc, "mbp-1", dataSet, useLimit: true); + return GetRange(symbol, startDateTimeUtc, endDateTimeUtc, MBP1Schema, dataSet); } public IEnumerable GetOpenInterest(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, string dataSet) { - foreach (var statistics in GetRange(symbol, startDateTimeUtc, endDateTimeUtc, "statistics", dataSet)) + foreach (var statistics in GetRange(symbol, startDateTimeUtc, endDateTimeUtc, StatisticsSchema, dataSet)) { if (statistics.StatType == Models.Enums.StatisticType.OpenInterest) { @@ -78,7 +114,7 @@ public IEnumerable GetOpenInterest(string symbol, DateTime start } } - private IEnumerable GetRange(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, string schema, string dataSet, bool useLimit = false) where T : MarketDataBase + private IEnumerable GetRange(string symbol, DateTime startDateTimeUtc, DateTime endDateTimeUtc, string schema, string dataSet) where T : MarketDataBase { var formData = new Dictionary { @@ -90,9 +126,16 @@ private IEnumerable GetRange(string symbol, DateTime startDateTimeUtc, Dat { "pretty_px", "true" }, }; - if (useLimit) + // Prevent HTTP client timeouts for large historical range requests. + // Explicitly cap the number of returned records per request based on the schema + switch (schema) { - formData["limit"] = "10000"; + case MBP1Schema: + formData["limit"] = "10000"; + break; + case OHLCV1sSchema: + formData["limit"] = "432000"; // 5 days + break; } var start = startDateTimeUtc; From a3c5d02f79375f4ecb6d822c2983903bb991fffb Mon Sep 17 00:00:00 2001 From: Romazes Date: Fri, 30 Jan 2026 20:43:09 +0200 Subject: [PATCH 18/18] refactor: HistoryProvider to use MappedSynchronizingHistoryProvider Refactored DataBentoProvider to inherit from MappedSynchronizingHistoryProvider, removing custom map file handling and multi-request GetHistory logic. Cleaned up unused usings and simplified history request processing by leveraging the new base class for map file resolution and request splitting. --- .../DataBentoDataProvider.cs | 5 --- .../DataBentoHistoryProivder.cs | 36 ++----------------- 2 files changed, 2 insertions(+), 39 deletions(-) diff --git a/QuantConnect.DataBento/DataBentoDataProvider.cs b/QuantConnect.DataBento/DataBentoDataProvider.cs index 8456fc1..16f1d6f 100644 --- a/QuantConnect.DataBento/DataBentoDataProvider.cs +++ b/QuantConnect.DataBento/DataBentoDataProvider.cs @@ -42,11 +42,6 @@ namespace QuantConnect.Lean.DataSource.DataBento; /// public partial class DataBentoProvider : IDataQueueHandler { - /// - /// Resolves map files to correctly handle current and historical ticker symbols. - /// - private readonly IMapFileProvider _mapFileProvider = Composer.Instance.GetPart(); - private HistoricalAPIClient _historicalApiClient; private readonly DataBentoSymbolMapper _symbolMapper = new(); diff --git a/QuantConnect.DataBento/DataBentoHistoryProivder.cs b/QuantConnect.DataBento/DataBentoHistoryProivder.cs index d2618d4..5af7a6c 100644 --- a/QuantConnect.DataBento/DataBentoHistoryProivder.cs +++ b/QuantConnect.DataBento/DataBentoHistoryProivder.cs @@ -14,14 +14,11 @@ * */ -using NodaTime; using QuantConnect.Data; using QuantConnect.Util; using QuantConnect.Logging; -using QuantConnect.Securities; using QuantConnect.Data.Market; using QuantConnect.Data.Consolidators; -using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.Lean.Engine.HistoricalData; namespace QuantConnect.Lean.DataSource.DataBento; @@ -30,7 +27,7 @@ namespace QuantConnect.Lean.DataSource.DataBento; /// Implements a history provider for DataBento historical data. /// Uses consolidators to produce the requested resolution when necessary. /// -public partial class DataBentoProvider : SynchronizingHistoryProvider +public partial class DataBentoProvider : MappedSynchronizingHistoryProvider { private static int _dataPointCount; @@ -62,41 +59,12 @@ public override void Initialize(HistoryProviderInitializeParameters parameters) { } - /// - /// Gets the history for the requested securities - /// - /// The historical data requests - /// The time zone used when time stamping the slice instances - /// An enumerable of the slices of data covering the span specified in each request - public override IEnumerable? GetHistory(IEnumerable requests, DateTimeZone sliceTimeZone) - { - var subscriptions = new List(); - foreach (var request in requests) - { - var history = request.SplitHistoryRequestWithUpdatedMappedSymbol(_mapFileProvider).SelectMany(x => GetHistory(x) ?? []); - - var subscription = CreateSubscription(request, history); - if (!subscription.MoveNext()) - { - continue; - } - - subscriptions.Add(subscription); - } - - if (subscriptions.Count == 0) - { - return null; - } - return CreateSliceEnumerableFromSubscriptions(subscriptions, sliceTimeZone); - } - /// /// Gets the history for the requested security /// /// The historical data request /// An enumerable of BaseData points - public IEnumerable? GetHistory(HistoryRequest historyRequest) + public override IEnumerable? GetHistory(HistoryRequest historyRequest) { if (!CanSubscribe(historyRequest.Symbol)) {