Market-hours aware intraday consolidation

Algorithm.CSharp/MarketHourAwareIntradayConsolidationRegressionAlgorithm.cs

@@ -0,0 +1,140 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using QuantConnect.Data;
+using QuantConnect.Data.Common;
+using QuantConnect.Data.Market;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+using QuantConnect.Securities.Future;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Regression algorithm asserting that a <see cref="MarketHourAwareConsolidator"/> with an intraday period
+    /// anchors each bar to the market open and never lets a bar extend past the market close.
+    /// </summary>
+    public class MarketHourAwareIntradayConsolidationRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        private readonly TimeSpan _period = TimeSpan.FromMinutes(7);
+        private Future _future;
+        private SecurityExchangeHours _hours;
+        private int _consolidatedBarCount;
+
+        public override void Initialize()
+        {
+            SetStartDate(2013, 10, 06);
+            SetEndDate(2013, 10, 11);
+
+            _future = AddFuture(Futures.Indices.SP500EMini, Resolution.Minute, extendedMarketHours: true);
+            _hours = _future.Exchange.Hours;
+
+            var consolidator = new MarketHourAwareConsolidator(false, _period, typeof(TradeBar), TickType.Trade, extendedMarketHours: true);
+            consolidator.DataConsolidated += OnSevenMinuteBar;
+            SubscriptionManager.AddConsolidator(_future.Symbol, consolidator);
+        }
+
+        private void OnSevenMinuteBar(object sender, IBaseData consolidated)
+        {
+            var bar = (TradeBar)consolidated;
+            var marketOpen = _hours.GetPreviousMarketOpen(bar.Time.AddTicks(1), extendedMarketHours: true);
+            var marketClose = _hours.GetNextMarketClose(marketOpen, extendedMarketHours: true);
+
+            // the bar must be anchored to the market open
+            if ((bar.Time - marketOpen).Ticks % _period.Ticks != 0)
+            {
+                throw new RegressionTestException($"Bar starting at {bar.Time} is not anchored to the market open {marketOpen}");
+            }
+
+            // the bar must not extend past the market close
+            if (bar.EndTime > marketClose)
+            {
+                throw new RegressionTestException($"Bar ending at {bar.EndTime} extends past the market close {marketClose}");
+            }
+
+            _consolidatedBarCount++;
+        }
+
+        public override void OnEndOfAlgorithm()
+        {
+            if (_consolidatedBarCount == 0)
+            {
+                throw new RegressionTestException("The consolidator did not produce any bar");
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public List<Language> Languages { get; } = new() { Language.CSharp, Language.Python };
+
+        /// <summary>
+        /// Data Points count of all timeslices of algorithm
+        /// </summary>
+        public long DataPoints => 41486;
+
+        /// <summary>
+        /// Data Points count of the algorithm history
+        /// </summary>
+        public int AlgorithmHistoryDataPoints => 0;
+
+        /// <summary>
+        /// Final status of the algorithm
+        /// </summary>
+        public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Orders", "0"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "0%"},
+            {"Drawdown", "0%"},
+            {"Expectancy", "0"},
+            {"Start Equity", "100000"},
+            {"End Equity", "100000"},
+            {"Net Profit", "0%"},
+            {"Sharpe Ratio", "0"},
+            {"Sortino Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "0%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "-2.564"},
+            {"Tracking Error", "0.214"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "$0.00"},
+            {"Estimated Strategy Capacity", "$0"},
+            {"Lowest Capacity Asset", ""},
+            {"Portfolio Turnover", "0%"},
+            {"Drawdown Recovery", "0"},
+            {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"}
+        };
+    }
+}

Algorithm.Python/MarketHourAwareIntradayConsolidationRegressionAlgorithm.py

@@ -0,0 +1,53 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+### <summary>
+### Regression algorithm asserting that a MarketHourAwareConsolidator with an intraday period
+### anchors each bar to the market open and never lets a bar extend past the market close.
+### </summary>
+class MarketHourAwareIntradayConsolidationRegressionAlgorithm(QCAlgorithm):
+
+    def initialize(self):
+        self.set_start_date(2013, 10, 6)
+        self.set_end_date(2013, 10, 11)
+
+        self._period = timedelta(minutes=7)
+        self._consolidated_bar_count = 0
+
+        self._future = self.add_future(Futures.Indices.SP_500_E_MINI, Resolution.MINUTE, extended_market_hours=True)
+        self._hours = self._future.exchange.hours
+
+        consolidator = MarketHourAwareConsolidator(False, self._period, TradeBar, TickType.TRADE, True)
+        consolidator.data_consolidated += self._on_seven_minute_bar
+        self.subscription_manager.add_consolidator(self._future.symbol, consolidator)
+
+    def _on_seven_minute_bar(self, sender, consolidated):
+        bar = consolidated
+        market_open = self._hours.get_previous_market_open(bar.time + timedelta(microseconds=1), True)
+        market_close = self._hours.get_next_market_close(market_open, True)
+
+        # the bar must be anchored to the market open
+        if (bar.time - market_open) % self._period != timedelta(0):
+            raise RegressionTestException(f"Bar starting at {bar.time} is not anchored to the market open {market_open}")
+
+        # the bar must not extend past the market close
+        if bar.end_time > market_close:
+            raise RegressionTestException(f"Bar ending at {bar.end_time} extends past the market close {market_close}")
+
+        self._consolidated_bar_count += 1
+
+    def on_end_of_algorithm(self):
+        if self._consolidated_bar_count == 0:
+            raise RegressionTestException("The consolidator did not produce any bar")

Common/AlgorithmImports.py

@@ -68,6 +68,7 @@
 from QuantConnect.Securities.Equity import *
 from QuantConnect.Securities.Future import *
 from QuantConnect.Data.Consolidators import *
+from QuantConnect.Data.Common import *
 from QuantConnect.Orders.TimeInForces import *
 from QuantConnect.Algorithm.Framework import *
 from QuantConnect.Algorithm.Selection import *

Common/Data/Consolidators/MarketHourAwareConsolidator.cs

@@ -119,6 +119,43 @@ public MarketHourAwareConsolidator(bool dailyStrictEndTimeEnabled, Resolution re
             Consolidator.DataConsolidated += ForwardConsolidatedBar;
         }
 
+        /// <summary>
+        /// Initializes a new instance of the <see cref="MarketHourAwareConsolidator"/> class for an arbitrary period.
+        /// Intraday periods are anchored to the market open without extending past the close.
+        /// </summary>
+        /// <param name="dailyStrictEndTimeEnabled">True if daily strict end times should be enabled</param>
+        /// <param name="period">The consolidation period</param>
+        /// <param name="dataType">The target data type</param>
+        /// <param name="tickType">The target tick type</param>
+        /// <param name="extendedMarketHours">True if extended market hours should be consolidated</param>
+        public MarketHourAwareConsolidator(bool dailyStrictEndTimeEnabled, TimeSpan period, Type dataType, TickType tickType, bool extendedMarketHours)
+        {
+            _dailyStrictEndTimeEnabled = dailyStrictEndTimeEnabled;
+            Period = period;
+            _extendedMarketHours = extendedMarketHours;
+            Func<DateTime, CalendarInfo> calendar = period < Time.OneDay ? IntradayCalendar : DailyStrictEndTime;
+
+            if (dataType == typeof(Tick))
+            {
+                Consolidator = tickType == TickType.Trade
+                    ? new TickConsolidator(calendar)
+                    : new TickQuoteBarConsolidator(calendar);
+            }
+            else if (dataType == typeof(TradeBar))
+            {
+                Consolidator = new TradeBarConsolidator(calendar);
+            }
+            else if (dataType == typeof(QuoteBar))
+            {
+                Consolidator = new QuoteBarConsolidator(calendar);
+            }
+            else
+            {
+                throw new ArgumentNullException(nameof(dataType), $"{dataType.Name} not supported");
+            }
+            Consolidator.DataConsolidated += ForwardConsolidatedBar;
+        }
+
         /// <summary>
         /// Event handler that fires when a new piece of data is produced
         /// </summary>
@@ -200,6 +237,19 @@ protected virtual CalendarInfo DailyStrictEndTime(DateTime dateTime)
             return LeanData.GetDailyCalendar(dateTime, ExchangeHours, _extendedMarketHours);
         }
 
+        /// <summary>
+        /// Determines a bar start time and period for intraday consolidation, anchored to the market open
+        /// without extending past the market close so a bar never spans across closed market hours
+        /// </summary>
+        protected virtual CalendarInfo IntradayCalendar(DateTime dateTime)
+        {
+            if (ExchangeHours == null || ExchangeHours.IsMarketAlwaysOpen)
+            {
+                return new(dateTime.RoundDown(Period), Period);
+            }
+            return LeanData.GetIntradayCalendar(dateTime, Period, ExchangeHours, _extendedMarketHours);
+        }
+
         /// <summary>
         /// Useful for testing
         /// </summary>

Common/Util/LeanData.cs

@@ -1476,6 +1476,33 @@ public static CalendarInfo GetDailyCalendar(DateTime exchangeTimeZoneDate, Secur
             return new CalendarInfo(startTime, period);
         }
 
+        /// <summary>
+        /// Helper method to return the intraday bar start time and period, anchored to the market open, without extending past the close
+        /// </summary>
+        /// <param name="exchangeTimeZoneDate">The point in time we want to get the bar information about</param>
+        /// <param name="period">The intraday consolidation period</param>
+        /// <param name="exchangeHours">The associated exchange hours</param>
+        /// <param name="extendedMarketHours">True if extended market hours should be taken into consideration</param>
+        /// <returns>The calendar information that holds a start time and a period</returns>
+        public static CalendarInfo GetIntradayCalendar(DateTime exchangeTimeZoneDate, TimeSpan period, SecurityExchangeHours exchangeHours, bool extendedMarketHours)
+        {
+            var marketOpen = exchangeHours.IsOpen(exchangeTimeZoneDate, extendedMarketHours)
+                ? exchangeHours.GetPreviousMarketOpen(exchangeTimeZoneDate.AddTicks(1), extendedMarketHours)
+                : exchangeHours.GetPreviousMarketOpen(exchangeTimeZoneDate, extendedMarketHours);
+
+            var intervalsPassed = (long)Math.Floor((exchangeTimeZoneDate - marketOpen).Ticks / (double)period.Ticks);
+            var startTime = marketOpen.AddTicks(intervalsPassed * period.Ticks);
+
+            // keep the last bar from extending past the market close
+            var endTime = startTime + period;
+            var marketClose = exchangeHours.GetNextMarketClose(marketOpen, extendedMarketHours);
+            if (endTime > marketClose)
+            {
+                endTime = marketClose;
+            }
+            return new CalendarInfo(startTime, endTime - startTime);
+        }
+
         /// <summary>
         /// Helper method to get the next daily end time, taking into account strict end times if appropriate
         /// </summary>

Tests/Common/Data/MarketHourAwareConsolidatorTests.cs

@@ -21,6 +21,7 @@
 using QuantConnect.Data.Consolidators;
 using System.Collections.Generic;
 using QuantConnect.Data;
+using QuantConnect.Securities;
 
 namespace QuantConnect.Tests.Common.Data
 {
@@ -294,6 +295,64 @@ public void WorksWithDailyResolutionAndPreciseEndTimeFalse()
             Assert.AreEqual(100, consolidatedData.High);
         }
 
+        [Test]
+        public void IntradayConsolidatorIsAnchoredToMarketOpen()
+        {
+            var symbol = Symbols.Future_ESZ18_Dec2018;
+            var exchangeHours = MarketHoursDatabase.FromDataFolder().GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType);
+            var marketOpen = exchangeHours.GetNextMarketOpen(new DateTime(2024, 11, 30, 12, 0, 0), extendedMarketHours: true);
+
+            using var consolidator = new MarketHourAwareConsolidator(false, TimeSpan.FromMinutes(7), typeof(TradeBar), TickType.Trade, extendedMarketHours: true);
+            var bars = new List<TradeBar>();
+            consolidator.DataConsolidated += (_, b) => bars.Add((TradeBar)b);
+
+            // feed the first 30 minutes after the open, one bar per minute
+            for (var i = 0; i < 30; i++)
+            {
+                var t = marketOpen.AddMinutes(i);
+                consolidator.Update(new TradeBar { Time = t, Period = Time.OneMinute, Symbol = symbol, Open = 1, High = 1, Low = 1, Close = 1, Volume = 1 });
+            }
+
+            Assert.GreaterOrEqual(bars.Count, 3);
+            Assert.AreEqual(marketOpen, bars[0].Time);
+            Assert.AreEqual(marketOpen.AddMinutes(7), bars[0].EndTime);
+            Assert.AreEqual(marketOpen.AddMinutes(14), bars[1].EndTime);
+            Assert.AreEqual(marketOpen.AddMinutes(21), bars[2].EndTime);
+        }
+
+        [Test]
+        public void IntradayConsolidatorLastBarEndsAtMarketClose()
+        {
+            var symbol = Symbols.SPY;
+            using var consolidator = new MarketHourAwareConsolidator(false, TimeSpan.FromMinutes(7), typeof(TradeBar), TickType.Trade, extendedMarketHours: false);
+            var bars = new List<TradeBar>();
+            consolidator.DataConsolidated += (_, b) => bars.Add((TradeBar)b);
+
+            void Feed(DateTime from, int minutes)
+            {
+                for (var i = 0; i < minutes; i++)
+                {
+                    var t = from.AddMinutes(i);
+                    consolidator.Update(new TradeBar { Time = t, Period = Time.OneMinute, Symbol = symbol, Open = 1, High = 1, Low = 1, Close = 1, Volume = 1 });
+                }
+            }
+
+            // feed the last 10 minutes of day 1 (up to the 16:00 close) and the first 10 of day 2
+            Feed(new DateTime(2015, 04, 13, 15, 50, 0), 10);
+            Feed(new DateTime(2015, 04, 14, 9, 30, 0), 10);
+
+            // the last bar of day 1 should end at the 16:00 close
+            var lastDay1 = bars.FindLast(b => b.Time.Date == new DateTime(2015, 04, 13));
+            Assert.IsNotNull(lastDay1);
+            Assert.AreEqual(new DateTime(2015, 04, 13, 16, 0, 0), lastDay1.EndTime);
+
+            // the first bar of day 2 should start at the 9:30 open, ending at 9:37
+            var day2Open = new DateTime(2015, 04, 14, 9, 30, 0);
+            var firstDay2 = bars.Find(b => b.Time == day2Open);
+            Assert.IsNotNull(firstDay2);
+            Assert.AreEqual(day2Open.AddMinutes(7), firstDay2.EndTime);
+        }
+
         protected override IDataConsolidator CreateConsolidator()
         {
             return new MarketHourAwareConsolidator(true, Resolution.Hour, typeof(TradeBar), TickType.Trade, false);