from __future__ import annotations """CSV/Parquet replay feed. Loads OHLCV data from disk and replays it into live_store at a configurable speed. Useful for offline demos or backtesting visualizations. """ import asyncio from dataclasses import dataclass from datetime import datetime from pathlib import Path from typing import Any, Dict, Iterable, List, Set, Tuple import pandas as pd from app.streaming.live_store import live_store @dataclass(frozen=True) class CSVKey: symbol: str timeframe: str class CSVFeedProvider: def __init__(self, data_dir: str | Path | None = None) -> None: self._subs: Dict[CSVKey, Set[asyncio.Queue]] = {} self._tasks: Dict[CSVKey, asyncio.Task] = {} self._pinned: Set[CSVKey] = set() self._lock = asyncio.Lock() self._data_dir = Path(data_dir or Path.cwd() / "data" / "parquet" / "live") self._speed = 1.0 # 1x realtime replay def get_status(self) -> list[dict]: out: list[dict] = [] for key, subs in self._subs.items(): out.append( { "symbol": key.symbol, "timeframe": key.timeframe, "subscribers": len(subs), "source": "csv_replay", "data_dir": str(self._data_dir), } ) return out async def subscribe(self, symbol: str, timeframe: str = "1m") -> Tuple[asyncio.Queue, Any]: key = CSVKey(symbol.upper().replace("/", ""), timeframe) queue: asyncio.Queue = asyncio.Queue(maxsize=100) async with self._lock: subs = self._subs.setdefault(key, set()) subs.add(queue) if key not in self._tasks: self._tasks[key] = asyncio.create_task(self._run_replay(key)) async def _unsubscribe() -> None: async with self._lock: s = self._subs.get(key) if s and queue in s: s.remove(queue) try: queue.put_nowait(None) except Exception: pass if s and len(s) == 0 and key not in self._pinned: task = self._tasks.pop(key, None) if task: task.cancel() self._subs.pop(key, None) return queue, _unsubscribe async def ensure_stream(self, symbol: str, timeframe: str = "1m") -> None: key = CSVKey(symbol.upper().replace("/", ""), timeframe) async with self._lock: self._pinned.add(key) self._subs.setdefault(key, set()) if key not in self._tasks: self._tasks[key] = asyncio.create_task(self._run_replay(key)) def set_speed(self, speed: float) -> None: self._speed = max(0.1, speed) async def _run_replay(self, key: CSVKey) -> None: file_path = self._resolve_file(key.symbol, key.timeframe) if not file_path.exists(): raise FileNotFoundError(f"Replay file not found: {file_path}") df = self._load_file(file_path) for row in df.itertuples(): evt = { "symbol": key.symbol, "timeframe": key.timeframe, "open_time": datetime.utcfromtimestamp(int(row.time)).isoformat(), "close_time": datetime.utcfromtimestamp(int(row.time)).isoformat(), "open": float(row.open), "high": float(row.high), "low": float(row.low), "close": float(row.close), "volume": float(getattr(row, "volume", 0.0)), "is_closed": True, "source": "csv_replay", } live_store.ingest_bar( symbol=key.symbol, timeframe=key.timeframe, bar={ "time": int(row.time), "open": evt["open"], "high": evt["high"], "low": evt["low"], "close": evt["close"], "volume": evt["volume"], }, ) subs = self._subs.get(key) or set() for queue in list(subs): try: if queue.full(): queue.get_nowait() queue.put_nowait(evt) except Exception: subs.discard(queue) await asyncio.sleep((60 / self._speed)) # default 1m bars -> 1 minute def _resolve_file(self, symbol: str, timeframe: str) -> Path: filename = f"{symbol}_{timeframe}.parquet" return self._data_dir / filename def _load_file(self, path: Path) -> pd.DataFrame: if path.suffix == ".csv": return pd.read_csv(path) return pd.read_parquet(path) csv_feed = CSVFeedProvider()