Add instrument_ids and bar_types for BacktestDataConfig (#2478)

Author: faysou

examples/backtest/databento_option_greeks.py

@@ -198,6 +198,8 @@ def on_stop(self):
 
 # for saving and loading custom data greeks, use True, False then False, True below
 stream_data, load_greeks = False, False
+# stream_data, load_greeks = True, False
+# stream_data, load_greeks = False, True
 
 actors = [
     ImportableActorConfig(
@@ -248,17 +250,21 @@ def on_stop(self):
 # BacktestRunConfig
 
 data = [
+    # TODO using instrument_id and bar_spec only, or instrument_ids and bar_spec only, or bar_types only
     BacktestDataConfig(
         data_cls=Bar,
         catalog_path=catalog.path,
         instrument_id=InstrumentId.from_str(f"{future_symbols[0]}.GLBX"),
+        # instrument_ids=[InstrumentId.from_str(f"{future_symbols[0]}.GLBX")],
         bar_spec="1-MINUTE-LAST",
+        # bar_types=[f"{future_symbols[0]}.GLBX-1-MINUTE-LAST-EXTERNAL"],
         # start_time=start_time,
         # end_time=end_time,
     ),
     BacktestDataConfig(
         data_cls=QuoteTick,
         catalog_path=catalog.path,
+        # instrument_ids=[InstrumentId.from_str(f"{option_symbols[0]}.GLBX"), InstrumentId.from_str(f"{option_symbols[1]}.GLBX")],
     ),
 ]
 
@@ -289,7 +295,7 @@ def on_stop(self):
         engine=engine_config,
         data=data,
         venues=venues,
-        chunk_size=None,  # use None when loading custom data
+        chunk_size=None,  # use None when loading custom data, else a value of 10_000 for example
     ),
 ]
 

nautilus_trader/backtest/config.py

@@ -29,6 +29,7 @@
 from nautilus_trader.data.config import DataEngineConfig
 from nautilus_trader.execution.config import ExecEngineConfig
 from nautilus_trader.model.data import Bar
+from nautilus_trader.model.data import BarType
 from nautilus_trader.model.identifiers import InstrumentId
 from nautilus_trader.model.identifiers import TraderId
 from nautilus_trader.risk.config import RiskEngineConfig
@@ -58,9 +59,11 @@ def parse_filters_expr(s: str | None):
     def safer_eval(input_string):
         allowed_names = {"field": field}
         code = compile(input_string, "<string>", "eval")
+
         for name in code.co_names:
             if name not in allowed_names:
                 raise NameError(f"Use of {name} not allowed")
+
         return eval(code, {}, allowed_names)  # noqa
 
     return safer_eval(s)  # Only allow use of the field object
@@ -173,6 +176,13 @@ class BacktestDataConfig(NautilusConfig, frozen=True):
         The metadata for the data catalog query.
     bar_spec : str, optional
         The bar specification for the data catalog query.
+    instrument_ids : list[str], optional
+        The instrument IDs for the data catalog query.
+        Can be used if instrument_id is not specified.
+        If bar_spec is specified an equivalent list of bar_types will be constructed.
+    bar_types : list[str], optional
+        The bar types for the data catalog query.
+        Can be used if instrument_id is not specified.
 
     """
 
@@ -187,6 +197,8 @@ class BacktestDataConfig(NautilusConfig, frozen=True):
     client_id: str | None = None
     metadata: dict | None = None
     bar_spec: str | None = None
+    instrument_ids: list[str] | None = None
+    bar_types: list[str] | None = None
 
     @property
     def data_type(self) -> type:
@@ -204,7 +216,7 @@ def data_type(self) -> type:
             return self.data_cls
 
     @property
-    def query(self) -> dict[str, Any]:
+    def query(self) -> dict[str, Any]:  # noqa: C901
         """
         Return a catalog query object for the configuration.
 
@@ -213,15 +225,48 @@ def query(self) -> dict[str, Any]:
         dict[str, Any]
 
         """
-        if self.data_cls is Bar and self.bar_spec:
-            bar_type = f"{self.instrument_id}-{self.bar_spec}-EXTERNAL"
-            filter_expr: str | None = f'field("bar_type") == "{bar_type}"'
+        filter_expr: str | None = None
+
+        if self.data_cls is Bar:
+            used_bar_types = []
+
+            if self.bar_types is None and self.instrument_ids is None:
+                assert self.instrument_id, "No `instrument_id` for Bar data config"
+                assert self.bar_spec, "No `bar_spec` for Bar data config"
+
+            if self.instrument_id is not None and self.bar_spec is not None:
+                bar_type = f"{self.instrument_id}-{self.bar_spec}-EXTERNAL"
+                used_bar_types = [bar_type]
+            elif self.bar_types is not None:
+                used_bar_types = self.bar_types
+            elif self.instrument_ids is not None and self.bar_spec is not None:
+                for instrument_id in self.instrument_ids:
+                    used_bar_types.append(f"{instrument_id}-{self.bar_spec}-EXTERNAL")
+
+            if len(used_bar_types) > 0:
+                filter_expr = f'(field("bar_type") == "{used_bar_types[0]}")'
+
+            for bar_type in used_bar_types[1:]:
+                filter_expr = f'{filter_expr} | (field("bar_type") == "{bar_type}")'
         else:
             filter_expr = self.filter_expr
 
+        used_instrument_ids = None
+
+        if self.instrument_id is not None:
+            used_instrument_ids = [self.instrument_id]
+        elif self.instrument_ids is not None:
+            used_instrument_ids = self.instrument_ids
+        elif self.bar_types is not None:
+            bar_types: list[BarType] = [
+                BarType.from_str(bar_type) if type(bar_type) is str else bar_type
+                for bar_type in self.bar_types
+            ]
+            used_instrument_ids = [bar_type.instrument_id for bar_type in bar_types]
+
         return {
             "data_cls": self.data_type,
-            "instrument_ids": [self.instrument_id] if self.instrument_id else None,
+            "instrument_ids": used_instrument_ids,
             "start": self.start_time,
             "end": self.end_time,
             "filter_expr": parse_filters_expr(filter_expr),
@@ -242,6 +287,7 @@ def start_time_nanos(self) -> int:
         """
         if self.start_time is None:
             return 0
+
         return dt_to_unix_nanos(self.start_time)
 
     @property
@@ -258,6 +304,7 @@ def end_time_nanos(self) -> int:
         """
         if self.end_time is None:
             return sys.maxsize
+
         return dt_to_unix_nanos(self.end_time)
 
 

nautilus_trader/backtest/engine.pyx

@@ -672,6 +672,7 @@ cdef class BacktestEngine:
 
         if validate:
             first = data[0]
+
             if hasattr(first, "instrument_id"):
                 Condition.is_true(
                     first.instrument_id in self.kernel.cache.instrument_ids(),