Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Historical data when symbol was restructured (renamed) #10

Merged
merged 5 commits into from
Mar 18, 2024
Merged

Historical data when symbol was restructured (renamed) #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1e93925
feat: `volatile` for prevent spamming flags
Romazes Mar 15, 2024
1d949b3
test:remove: extra param in test data
Romazes Mar 15, 2024
c255169
refactor: DataDownloader provider to general structure
Romazes Mar 15, 2024
ef15c36
fix: convert DateTime to TZ in GetHistory
Romazes Mar 15, 2024
ae839a3
feat: get relevant ticker for each historical request
Romazes Mar 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 43 additions & 9 deletions QuantConnect.IEX.Tests/IEXDataHistoryTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
*/

using System;
using NodaTime;
using System.Linq;
using NUnit.Framework;
using QuantConnect.Data;
Expand Down Expand Up @@ -65,7 +66,7 @@ internal static IEnumerable<TestCaseData> ValidDataTestParameters
.SetDescription("Valid parameters - Minute resolution, 5 days period.")
.SetCategory("Valid");

yield return new TestCaseData(Symbols.SPY, Resolution.Minute, TickType.Trade, TimeSpan.FromDays(45), true)
yield return new TestCaseData(Symbols.SPY, Resolution.Minute, TickType.Trade, TimeSpan.FromDays(45))
.SetDescription("Valid parameters - Beyond 45 days, Minute resolution.")
.SetCategory("Valid");
}
Expand Down Expand Up @@ -159,6 +160,22 @@ public void IEXCloudGetHistoryWithValidDataTestParameters(Symbol symbol, Resolut
Assert.That(slices, Is.Ordered.By("Time"));
}

[Explicit("This tests require a iexcloud.io api key")]
[TestCase("GOOGL", Resolution.Daily, "2013/1/3", "2015/12/29", Description = "October 2, 2015. [GOOG -> GOOGL]")]
[TestCase("META", Resolution.Daily, "2020/1/3", "2023/12/29", Description = "October 28, 2021. [FB -> META]")]
public void GetAncientEquityHistoricalData(string ticker, Resolution resolution, DateTime startDate, DateTime endDate)
{
var symbol = Symbol.Create(ticker, SecurityType.Equity, Market.USA);

var request = CreateHistoryRequest(symbol, resolution, TickType.Trade, startDate, endDate);

var slices = iexDataProvider.GetHistory(new[] { request }, TimeZones.NewYork)?.ToList();

Assert.Greater(slices.Count, 1);
Assert.That(slices.First().Time.Date, Is.EqualTo(startDate));
Assert.That(slices.Last().Time.Date, Is.LessThanOrEqualTo(endDate));
}

internal static void AssertTradeBar(Symbol expectedSymbol, Resolution resolution, BaseData baseData, Symbol actualSymbol = null)
{
if (actualSymbol != null)
Expand Down Expand Up @@ -235,25 +252,42 @@ private Slice[] GetHistory(Symbol symbol, Resolution resolution, TickType tickTy

internal static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution resolution, TickType tickType, TimeSpan period)
{
var utcNow = DateTime.UtcNow;
var end = new DateTime(2024, 3, 15, 16, 0, 0);

var dataType = LeanData.GetDataType(resolution, tickType);
if (resolution == Resolution.Daily)
{
end = end.Date.AddDays(1);
}

return CreateHistoryRequest(symbol, resolution, tickType, end.Subtract(period), end);
}

internal static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution resolution, TickType tickType, DateTime startDateTime, DateTime endDateTime,
SecurityExchangeHours exchangeHours = null, DateTimeZone dataTimeZone = null)
{
if (exchangeHours == null)
{
exchangeHours = SecurityExchangeHours.AlwaysOpen(TimeZones.NewYork);
}

var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType);
if (dataTimeZone == null)
{
dataTimeZone = TimeZones.NewYork;
}

var dataType = LeanData.GetDataType(resolution, tickType);
return new HistoryRequest(
startTimeUtc: utcNow.Add(-period),
endTimeUtc: utcNow,
startTimeUtc: startDateTime,
endTimeUtc: endDateTime,
dataType: dataType,
symbol: symbol,
resolution: resolution,
exchangeHours: exchangeHours,
dataTimeZone: dataTimeZone,
fillForwardResolution: resolution,
fillForwardResolution: null,
includeExtendedMarketHours: true,
isCustomData: false,
DataNormalizationMode.Raw,
dataNormalizationMode: DataNormalizationMode.Adjusted,
tickType: tickType
);
}
Expand Down
35 changes: 15 additions & 20 deletions QuantConnect.IEX/IEXDataDownloader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
*/

using QuantConnect.Data;
using QuantConnect.Util;
using QuantConnect.Securities;
using QuantConnect.Data.Market;

namespace QuantConnect.Lean.DataSource.IEX
{
Expand Down Expand Up @@ -44,30 +44,25 @@ public void Dispose()
public IEnumerable<BaseData>? Get(DataDownloaderGetParameters dataDownloaderGetParameters)
{
var symbol = dataDownloaderGetParameters.Symbol;
var resolution = dataDownloaderGetParameters.Resolution;
var startUtc = dataDownloaderGetParameters.StartUtc;
var endUtc = dataDownloaderGetParameters.EndUtc;
var tickType = dataDownloaderGetParameters.TickType;

var exchangeHours = _marketHoursDatabase.GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
var dataTimeZone = _marketHoursDatabase.GetDataTimeZone(symbol.ID.Market, symbol, symbol.SecurityType);

var historyRequests = new[] {
new HistoryRequest(startUtc,
endUtc,
typeof(TradeBar),
symbol,
resolution,
exchangeHours,
dataTimeZone,
resolution,
true,
false,
DataNormalizationMode.Raw,
TickType.Trade)
};
var historyRequests = new HistoryRequest(
dataDownloaderGetParameters.StartUtc,
dataDownloaderGetParameters.EndUtc,
LeanData.GetDataType(dataDownloaderGetParameters.Resolution, dataDownloaderGetParameters.TickType),
symbol,
dataDownloaderGetParameters.Resolution,
exchangeHours,
dataTimeZone,
dataDownloaderGetParameters.Resolution,
true,
false,
DataNormalizationMode.Raw,
dataDownloaderGetParameters.TickType);

return _handler.GetHistory(historyRequests, TimeZones.EasternStandard)?.Select(slice => (BaseData)slice[symbol]);
return _handler.ProcessHistoryRequests(historyRequests);
}
}
}
172 changes: 89 additions & 83 deletions QuantConnect.IEX/IEXDataProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,22 +56,28 @@ public class IEXDataProvider : SynchronizingHistoryProvider, IDataQueueHandler
/// <summary>
/// Flag indicating whether a warning about unsupported data types in user history should be suppressed to prevent spam.
/// </summary>
private static bool _invalidHistoryDataTypeWarningFired;
private volatile bool _invalidHistoryDataTypeWarningFired;

/// <summary>
/// Indicates whether the warning for invalid <see cref="SecurityType"/> has been fired.
/// </summary>
private bool _invalidSecurityTypeWarningFired;
private volatile bool _invalidSecurityTypeWarningFired;

/// <summary>
/// Indicates whether a warning for an invalid start time has been fired, where the start time is greater than or equal to the end time in UTC.
/// </summary>
private bool _invalidStartTimeWarningFired;
private volatile bool _invalidStartTimeWarningFired;

/// <summary>
/// Indicates whether a warning for an invalid <see cref="Resolution"/> has been fired, where the resolution is neither daily nor minute-based.
/// </summary>
private bool _invalidResolutionWarningFired;
private volatile bool _invalidResolutionWarningFired;

/// <summary>
/// Indicates whether a warning has been triggered for reaching the limit of <see cref="Resolution.Minute"/> resolution,
/// where the startDateTime is not greater than 2 years.
/// </summary>
private volatile bool _limitMinuteResolutionWarningFired;

/// <summary>
/// Represents two clients: one for the trade channel and another for the top-of-book channel.
Expand Down Expand Up @@ -512,49 +518,13 @@ public override void Initialize(HistoryProviderInitializeParameters parameters)
var subscriptions = new List<Subscription>();
foreach (var request in requests)
{
// IEX does return historical TradeBar - give one time warning if inconsistent data type was requested
if (request.TickType != TickType.Trade)
{
if (!_invalidHistoryDataTypeWarningFired)
{
Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Not supported data type - {request.DataType.Name}. " +
"Currently available support only for historical of type - TradeBar");
_invalidHistoryDataTypeWarningFired = true;
}
continue;
}

if (!CanSubscribe(request.Symbol))
{
if (!_invalidSecurityTypeWarningFired)
{
Log.Trace($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Unsupported SecurityType '{request.Symbol.SecurityType}' for symbol '{request.Symbol}'");
_invalidSecurityTypeWarningFired = true;
}
continue;
}

if (request.StartTimeUtc >= request.EndTimeUtc)
{
if (!_invalidStartTimeWarningFired)
{
Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Error - The start date in the history request must come before the end date. No historical data will be returned.");
_invalidStartTimeWarningFired = true;
}
continue;
}
var history = ProcessHistoryRequests(request);

if (request.Resolution != Resolution.Daily && request.Resolution != Resolution.Minute)
if (history == null)
{
if (!_invalidResolutionWarningFired)
{
Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: History calls for IEX only support daily & minute resolution.");
_invalidResolutionWarningFired = true;
}
continue;
}

var history = ProcessHistoryRequests(request);
var subscription = CreateSubscription(request, history);
subscriptions.Add(subscription);
}
Expand All @@ -570,23 +540,75 @@ public override void Initialize(HistoryProviderInitializeParameters parameters)
/// <summary>
/// Populate request data
/// </summary>
private IEnumerable<BaseData>? ProcessHistoryRequests(Data.HistoryRequest request)
public IEnumerable<BaseData>? ProcessHistoryRequests(Data.HistoryRequest request)
{
var ticker = request.Symbol.ID.Symbol;
var start = ConvertTickTimeBySymbol(request.Symbol, request.StartTimeUtc);
var end = ConvertTickTimeBySymbol(request.Symbol, request.EndTimeUtc);
// IEX does return historical TradeBar - give one time warning if inconsistent data type was requested
if (request.TickType != TickType.Trade)
{
if (!_invalidHistoryDataTypeWarningFired)
{
_invalidHistoryDataTypeWarningFired = true;
Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Not supported data type - {request.DataType.Name}. " +
"Currently available support only for historical of type - TradeBar");
}
return null;
}

if (!CanSubscribe(request.Symbol))
{
if (!_invalidSecurityTypeWarningFired)
{
_invalidSecurityTypeWarningFired = true;
Log.Trace($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Unsupported SecurityType '{request.Symbol.SecurityType}' for symbol '{request.Symbol}'");
}
return null;
}

Log.Trace($"{nameof(IEXDataProvider)}.{nameof(ProcessHistoryRequests)}: {request.Symbol.SecurityType}.{ticker}, Resolution: {request.Resolution}, DateTime: [{start} - {end}].");
if (request.StartTimeUtc >= request.EndTimeUtc)
{
if (!_invalidStartTimeWarningFired)
{
_invalidStartTimeWarningFired = true;
Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: Error - The start date in the history request must come before the end date. No historical data will be returned.");
}
return null;
}

if (request.Resolution != Resolution.Daily && request.Resolution != Resolution.Minute)
{
if (!_invalidResolutionWarningFired)
{
_invalidResolutionWarningFired = true;
Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: History calls for IEX only support daily & minute resolution.");
}
return null;
}

// Always obtain the most relevant ticker symbol based on the current time.
var ticker = SecurityIdentifier.Ticker(request.Symbol, DateTime.UtcNow);
var startExchangeDateTime = ConvertTickTimeBySymbol(request.Symbol, request.StartTimeUtc);
var endExchangeDateTime = ConvertTickTimeBySymbol(request.Symbol, request.EndTimeUtc);

if (request.Resolution == Resolution.Minute && startExchangeDateTime <= DateTime.Today.AddYears(-2))
{
if (!_limitMinuteResolutionWarningFired)
{
_limitMinuteResolutionWarningFired = true;
Log.Error($"{nameof(IEXDataProvider)}.{nameof(GetHistory)}: History calls with minute resolution for IEX available only not more 2 years.");
}
return null;
}

Log.Trace($"{nameof(IEXDataProvider)}.{nameof(ProcessHistoryRequests)}: {request.Symbol.SecurityType}.{ticker}, Resolution: {request.Resolution}, DateTime: [{startExchangeDateTime} - {endExchangeDateTime}].");

var span = end - start;
var urls = new List<string>();

switch (request.Resolution)
{
case Resolution.Minute:
{
var begin = start;
while (begin < end)
var begin = startExchangeDateTime;
while (begin < endExchangeDateTime)
{
var url = $"{BaseUrl}/{ticker}/chart/date/{begin.ToStringInvariant("yyyyMMdd")}?token={_apiKey}";
urls.Add(url);
Expand All @@ -597,35 +619,19 @@ public override void Initialize(HistoryProviderInitializeParameters parameters)
}
case Resolution.Daily:
{
string suffix;
if (span.Days < 30)
{
suffix = "1m";
}
else if (span.Days < 3 * 30)
{
suffix = "3m";
}
else if (span.Days < 6 * 30)
{
suffix = "6m";
}
else if (span.Days < 12 * 30)
{
suffix = "1y";
}
else if (span.Days < 24 * 30)
{
suffix = "2y";
}
else if (span.Days < 60 * 30)
{
suffix = "5y";
}
else
// To retrieve a specific start-to-end dateTime range, calculate the duration between the current time and the startExchangeDateTime.
var span = DateTime.Now - startExchangeDateTime;

string suffix = span.Days switch
{
suffix = "max"; // max is 15 years
}
< 30 => "1m",
< 3 * 30 => "3m",
< 6 * 30 => "6m",
< 12 * 30 => "1y",
< 24 * 30 => "2y",
< 60 * 30 => "5y",
_ => "max" // max is 15 years
};

var url = $"{BaseUrl}/{ticker}/chart/{suffix}?token={_apiKey}";
urls.Add(url);
Expand All @@ -634,7 +640,9 @@ public override void Initialize(HistoryProviderInitializeParameters parameters)
}
}

return GetHistoryRequestByUrls(urls, start, end, request.Resolution, request.DataNormalizationMode, request.Symbol);
Log.Debug($"{nameof(IEXDataProvider)}.{nameof(ProcessHistoryRequests)}: {string.Join("\n", urls)}");

return GetHistoryRequestByUrls(urls, startExchangeDateTime, endExchangeDateTime, request.Resolution, request.DataNormalizationMode, request.Symbol);
}

/// <summary>
Expand Down Expand Up @@ -678,7 +686,7 @@ private IEnumerable<BaseData> GetHistoryRequestByUrls(List<string> urls, DateTim
period = TimeSpan.FromDays(1);
}

if (date < startDateTime || date > endDateTime)
if (!(date >= startDateTime && date < endDateTime))
{
continue;
}
Expand Down Expand Up @@ -710,9 +718,7 @@ private IEnumerable<BaseData> GetHistoryRequestByUrls(List<string> urls, DateTim
volume = item["volume"].Value<int>();
}

var tradeBar = new TradeBar(date, symbol, open, high, low, close, volume, period);

yield return tradeBar;
yield return new TradeBar(ConvertTickTimeBySymbol(symbol, date), symbol, open, high, low, close, volume, period);
}
}
}
Expand Down
Loading