feat: Add Phase 4 advanced metrics and components

- Add advanced metrics dashboard with trade analytics
- Add new trading components (EntryTypeAnalysis, MultiDayPositionTracker, NewsEventTracker, etc.)
- Add strategy mode selector and trend confirmation
- Add risk automation panel and slippage correlation analysis
- Add daily trading plan enhancements with modal components
- Add custom hooks (useApi, useLocalStorage, useAdvancedTradeMetrics)
- Add broker service integration and trading API
- Add test setup and vitest configuration
- Include parquet data files for live market data
- Add comprehensive documentation in docs/ folder
This commit is contained in:
Krikorios
2025-11-27 10:23:58 +02:00
parent b5e2b02cb8
commit 48e60d015f
2019 changed files with 39793 additions and 257 deletions
+30
View File
@@ -0,0 +1,30 @@
from __future__ import annotations
import asyncio
from typing import Iterable, Awaitable
from app.config import settings
from app.streaming.binance_hub import hub as binance_hub
from app.streaming.data_provider import data_provider
async def bootstrap_streams() -> None:
"""Ensure configured streams are hot even before clients connect."""
if not settings.STREAM_AUTO_BOOTSTRAP:
return
symbols: Iterable[str] = settings.STREAM_WARM_SYMBOLS or []
timeframe = settings.STREAM_WARM_TIMEFRAME or "1m"
coros: list[Awaitable[None]] = []
for raw in symbols:
sym = (raw or "").strip()
if not sym:
continue
if sym.upper().startswith("XAU"):
coros.append(data_provider.ensure_stream(sym, timeframe))
else:
coros.append(binance_hub.ensure_stream(sym, timeframe))
if coros:
await asyncio.gather(*coros, return_exceptions=True)
+138
View File
@@ -0,0 +1,138 @@
from __future__ import annotations
"""CSV/Parquet replay feed.
Loads OHLCV data from disk and replays it into live_store at a configurable
speed. Useful for offline demos or backtesting visualizations.
"""
import asyncio
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, Iterable, List, Set, Tuple
import pandas as pd
from app.streaming.live_store import live_store
@dataclass(frozen=True)
class CSVKey:
symbol: str
timeframe: str
class CSVFeedProvider:
def __init__(self, data_dir: str | Path | None = None) -> None:
self._subs: Dict[CSVKey, Set[asyncio.Queue]] = {}
self._tasks: Dict[CSVKey, asyncio.Task] = {}
self._pinned: Set[CSVKey] = set()
self._lock = asyncio.Lock()
self._data_dir = Path(data_dir or Path.cwd() / "data" / "parquet" / "live")
self._speed = 1.0 # 1x realtime replay
def get_status(self) -> list[dict]:
out: list[dict] = []
for key, subs in self._subs.items():
out.append(
{
"symbol": key.symbol,
"timeframe": key.timeframe,
"subscribers": len(subs),
"source": "csv_replay",
"data_dir": str(self._data_dir),
}
)
return out
async def subscribe(self, symbol: str, timeframe: str = "1m") -> Tuple[asyncio.Queue, Any]:
key = CSVKey(symbol.upper().replace("/", ""), timeframe)
queue: asyncio.Queue = asyncio.Queue(maxsize=100)
async with self._lock:
subs = self._subs.setdefault(key, set())
subs.add(queue)
if key not in self._tasks:
self._tasks[key] = asyncio.create_task(self._run_replay(key))
async def _unsubscribe() -> None:
async with self._lock:
s = self._subs.get(key)
if s and queue in s:
s.remove(queue)
try:
queue.put_nowait(None)
except Exception:
pass
if s and len(s) == 0 and key not in self._pinned:
task = self._tasks.pop(key, None)
if task:
task.cancel()
self._subs.pop(key, None)
return queue, _unsubscribe
async def ensure_stream(self, symbol: str, timeframe: str = "1m") -> None:
key = CSVKey(symbol.upper().replace("/", ""), timeframe)
async with self._lock:
self._pinned.add(key)
self._subs.setdefault(key, set())
if key not in self._tasks:
self._tasks[key] = asyncio.create_task(self._run_replay(key))
def set_speed(self, speed: float) -> None:
self._speed = max(0.1, speed)
async def _run_replay(self, key: CSVKey) -> None:
file_path = self._resolve_file(key.symbol, key.timeframe)
if not file_path.exists():
raise FileNotFoundError(f"Replay file not found: {file_path}")
df = self._load_file(file_path)
for row in df.itertuples():
evt = {
"symbol": key.symbol,
"timeframe": key.timeframe,
"open_time": datetime.utcfromtimestamp(int(row.time)).isoformat(),
"close_time": datetime.utcfromtimestamp(int(row.time)).isoformat(),
"open": float(row.open),
"high": float(row.high),
"low": float(row.low),
"close": float(row.close),
"volume": float(getattr(row, "volume", 0.0)),
"is_closed": True,
"source": "csv_replay",
}
live_store.ingest_bar(
symbol=key.symbol,
timeframe=key.timeframe,
bar={
"time": int(row.time),
"open": evt["open"],
"high": evt["high"],
"low": evt["low"],
"close": evt["close"],
"volume": evt["volume"],
},
)
subs = self._subs.get(key) or set()
for queue in list(subs):
try:
if queue.full():
queue.get_nowait()
queue.put_nowait(evt)
except Exception:
subs.discard(queue)
await asyncio.sleep((60 / self._speed)) # default 1m bars -> 1 minute
def _resolve_file(self, symbol: str, timeframe: str) -> Path:
filename = f"{symbol}_{timeframe}.parquet"
return self._data_dir / filename
def _load_file(self, path: Path) -> pd.DataFrame:
if path.suffix == ".csv":
return pd.read_csv(path)
return pd.read_parquet(path)
csv_feed = CSVFeedProvider()
+24
View File
@@ -0,0 +1,24 @@
from __future__ import annotations
from app.config import settings
from app.streaming.local_feed import local_feed
from app.streaming.metatrader_feed import metatrader_feed
from app.streaming.csv_feed import csv_feed
from app.streaming.historical_replay import historical_replay
# Registry for future providers. For now only the local simulator is available.
_PROVIDER_REGISTRY = {
"historical_replay": historical_replay,
"local_simulator": local_feed,
"metatrader": metatrader_feed,
"csv_replay": csv_feed,
}
provider_key = settings.DATA_PROVIDER.lower().strip()
data_provider = _PROVIDER_REGISTRY.get(provider_key)
if data_provider is None:
raise ValueError(
f"Unsupported DATA_PROVIDER '{settings.DATA_PROVIDER}'. Available: {', '.join(_PROVIDER_REGISTRY)}"
)
+209
View File
@@ -0,0 +1,209 @@
from __future__ import annotations
import asyncio
import time
from dataclasses import dataclass
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, Dict, List, Tuple, Set
import pyarrow.parquet as pq
from app.config import settings
from app.streaming.live_store import live_store, _TIMEFRAME_SECONDS
@dataclass(frozen=True)
class ReplayKey:
symbol: str
timeframe: str
class HistoricalReplayProvider:
"""Streams OHLCV data by replaying parquet partitions as live candles."""
def __init__(
self,
root: str | Path | None = None,
speed: float | None = None,
days: int | None = None,
loop: bool | None = None,
) -> None:
default_root = Path(__file__).resolve().parents[3] / "data" / "parquet" / "live"
resolved_root = Path(root) if root else Path(settings.HISTORICAL_REPLAY_ROOT or default_root)
self._root = resolved_root
self._speed = max(speed or settings.HISTORICAL_REPLAY_SPEED, 0.1)
self._days_back = max(days or settings.HISTORICAL_REPLAY_DAYS, 1)
self._loop = settings.HISTORICAL_REPLAY_LOOP if loop is None else loop
self._tasks: Dict[ReplayKey, asyncio.Task] = {}
self._pinned: Set[ReplayKey] = set()
self._lock = asyncio.Lock()
self._active_counts: Dict[ReplayKey, int] = {}
self._bar_cache: Dict[ReplayKey, Tuple[float, List[Dict[str, Any]]]] = {}
self._positions: Dict[ReplayKey, int] = {}
self._last_times: Dict[ReplayKey, int] = {}
def _sleep_seconds(self, timeframe: str) -> float:
base = _TIMEFRAME_SECONDS.get(timeframe.lower(), 60)
return max(base / self._speed, 0.5)
def _ensure_history(self, key: ReplayKey) -> None:
try:
if not live_store.get_history(key.symbol, key.timeframe):
live_store.load_historical_data(key.symbol, key.timeframe, days_back=self._days_back)
except Exception:
# Best-effort warmup; ignore errors so streaming can proceed
pass
def _resolve_partitions(self, key: ReplayKey) -> List[Tuple[datetime, Path]]:
base_path = self._root / key.symbol / key.timeframe
partitions: List[Tuple[datetime, Path]] = []
if not base_path.exists():
return partitions
for part in base_path.glob("date=*"):
if not part.is_dir():
continue
_, _, date_part = part.name.partition("=")
try:
dt = datetime.strptime(date_part, "%Y-%m-%d")
except ValueError:
continue
partitions.append((dt, part))
partitions.sort(key=lambda x: x[0])
return partitions
def _load_bars(self, key: ReplayKey) -> List[Dict[str, Any]]:
cached = self._bar_cache.get(key)
now = time.time()
if cached and now - cached[0] < 300:
return cached[1]
partitions = self._resolve_partitions(key)
if not partitions:
self._bar_cache[key] = (now, [])
return []
cutoff_date = partitions[-1][0] - timedelta(days=self._days_back - 1)
eligible = [p for p in partitions if p[0] >= cutoff_date]
if not eligible:
eligible = partitions[-self._days_back :] if len(partitions) >= self._days_back else partitions
rows: List[Dict[str, Any]] = []
for _, part in eligible:
files = sorted(part.glob("*.parquet"))
for file in files:
try:
table = pq.read_table(file, columns=["time", "open", "high", "low", "close", "volume"])
except Exception:
continue
for row in table.to_pylist():
try:
rows.append(
{
"time": int(row["time"]),
"open": float(row["open"]),
"high": float(row["high"]),
"low": float(row["low"]),
"close": float(row["close"]),
"volume": float(row.get("volume") or 0.0),
}
)
except Exception:
continue
rows.sort(key=lambda r: r["time"])
self._bar_cache[key] = (now, rows)
return rows
async def subscribe(self, symbol: str, timeframe: str = "1m") -> Tuple[asyncio.Queue, Any]:
if timeframe != "1m":
raise ValueError("HistoricalReplayProvider currently supports timeframe '1m' only")
key = ReplayKey(symbol=symbol.upper().replace("/", ""), timeframe=timeframe)
queue: asyncio.Queue = asyncio.Queue(maxsize=200)
live_store.subscribe(key.symbol, key.timeframe, queue)
async with self._lock:
self._active_counts[key] = self._active_counts.get(key, 0) + 1
if key not in self._tasks:
self._tasks[key] = asyncio.create_task(self._run_replay(key))
async def _unsubscribe() -> None:
live_store.unsubscribe(key.symbol, key.timeframe, queue)
async with self._lock:
self._active_counts[key] = max(0, self._active_counts.get(key, 0) - 1)
if self._active_counts.get(key, 0) == 0 and key not in self._pinned:
task = self._tasks.pop(key, None)
if task:
task.cancel()
self._active_counts.pop(key, None)
return queue, _unsubscribe
async def ensure_stream(self, symbol: str, timeframe: str = "1m") -> None:
if timeframe != "1m":
raise ValueError("HistoricalReplayProvider currently supports timeframe '1m' only")
key = ReplayKey(symbol=symbol.upper().replace("/", ""), timeframe=timeframe)
async with self._lock:
self._pinned.add(key)
self._active_counts.setdefault(key, 0)
if key not in self._tasks:
self._tasks[key] = asyncio.create_task(self._run_replay(key))
async def release_stream(self, symbol: str, timeframe: str = "1m") -> None:
key = ReplayKey(symbol=symbol.upper().replace("/", ""), timeframe=timeframe)
async with self._lock:
self._pinned.discard(key)
if self._active_counts.get(key, 0) == 0:
task = self._tasks.pop(key, None)
if task:
task.cancel()
self._active_counts.pop(key, None)
async def _run_replay(self, key: ReplayKey) -> None:
self._ensure_history(key)
sleep_secs = self._sleep_seconds(key.timeframe)
tf_seconds = _TIMEFRAME_SECONDS.get(key.timeframe.lower(), 60)
self._positions.setdefault(key, 0)
self._last_times.setdefault(key, 0)
while True:
try:
bars = self._load_bars(key)
if not bars:
await asyncio.sleep(5.0)
continue
idx = self._positions.get(key, 0)
if idx >= len(bars):
if not self._loop:
await asyncio.sleep(sleep_secs)
continue
idx = 0
bar = bars[idx]
self._positions[key] = idx + 1
now = int(time.time())
aligned = (now // tf_seconds) * tf_seconds
last_time = self._last_times.get(key) or 0
if aligned <= last_time:
aligned = last_time + tf_seconds
payload = {
"time": aligned,
"open": bar["open"],
"high": bar["high"],
"low": bar["low"],
"close": bar["close"],
"volume": bar.get("volume", 0.0),
}
live_store.ingest_bar(symbol=key.symbol, timeframe=key.timeframe, bar=payload)
self._last_times[key] = aligned
await asyncio.sleep(sleep_secs)
except asyncio.CancelledError:
break
except Exception:
await asyncio.sleep(min(5.0, sleep_secs))
historical_replay = HistoricalReplayProvider()
+144
View File
@@ -0,0 +1,144 @@
from __future__ import annotations
import asyncio
from dataclasses import dataclass
from datetime import datetime
from typing import Any, Dict, Set, Tuple
from app.streaming.live_store import live_store
from app.services.price_simulator import gold_simulator
@dataclass(frozen=True)
class FeedKey:
symbol: str
timeframe: str
class LocalFeedProvider:
"""In-memory price feed backed by the gold price simulator."""
def __init__(self) -> None:
self._tasks: Dict[FeedKey, asyncio.Task] = {}
self._pinned: Set[FeedKey] = set()
self._lock = asyncio.Lock()
self._active_counts: Dict[FeedKey, int] = {}
def get_status(self) -> list[dict]:
out: list[dict] = []
for key, task in self._tasks.items():
hist = live_store.get_history(key.symbol, key.timeframe)
last_ts = hist[-1]["time"] if hist else None
last_iso = None
if isinstance(last_ts, (int, float)):
try:
last_iso = datetime.utcfromtimestamp(int(last_ts)).isoformat() + "Z"
except Exception:
last_iso = None
out.append(
{
"symbol": key.symbol,
"timeframe": key.timeframe,
"subscribers": self._active_counts.get(key, 0),
"last_event_time": last_iso,
}
)
return out
async def subscribe(self, symbol: str, timeframe: str = "1m") -> Tuple[asyncio.Queue, Any]:
if timeframe != "1m":
raise ValueError("LocalFeedProvider currently supports timeframe '1m' only")
key = FeedKey(symbol=symbol.upper().replace("/", ""), timeframe=timeframe)
queue: asyncio.Queue = asyncio.Queue(maxsize=100)
live_store.subscribe(key.symbol, key.timeframe, queue)
async with self._lock:
self._active_counts[key] = self._active_counts.get(key, 0) + 1
if key not in self._tasks:
self._tasks[key] = asyncio.create_task(self._run_simulator_poller(key))
async def _unsubscribe() -> None:
live_store.unsubscribe(key.symbol, key.timeframe, queue)
async with self._lock:
self._active_counts[key] = max(0, self._active_counts.get(key, 0) - 1)
if self._active_counts.get(key, 0) == 0 and key not in self._pinned:
task = self._tasks.pop(key, None)
if task:
task.cancel()
self._active_counts.pop(key, None)
return queue, _unsubscribe
async def ensure_stream(self, symbol: str, timeframe: str = "1m") -> None:
if timeframe != "1m":
raise ValueError("LocalFeedProvider currently supports timeframe '1m' only")
key = FeedKey(symbol=symbol.upper().replace("/", ""), timeframe=timeframe)
async with self._lock:
self._pinned.add(key)
if key not in self._tasks:
self._tasks[key] = asyncio.create_task(self._run_simulator_poller(key))
async def release_stream(self, symbol: str, timeframe: str = "1m") -> None:
key = FeedKey(symbol=symbol.upper().replace("/", ""), timeframe=timeframe)
async with self._lock:
self._pinned.discard(key)
if self._active_counts.get(key, 0) == 0:
task = self._tasks.pop(key, None)
if task:
task.cancel()
async def _run_simulator_poller(self, key: FeedKey) -> None:
symbol = key.symbol
simulator = gold_simulator
if simulator.current_price < 2400 or simulator.current_price > 2900:
simulator.current_price = 2650.0
simulator.base_price = 2650.0
last_ts: int | None = None
poll_interval = 3
while True:
try:
candle = simulator.get_live_candle(interval="1min")
tsec = candle.time
if last_ts is None or tsec > last_ts:
dt = datetime.utcfromtimestamp(tsec)
evt = {
"symbol": symbol,
"timeframe": key.timeframe,
"open_time": dt.isoformat(),
"close_time": dt.isoformat(),
"open": candle.open,
"high": candle.high,
"low": candle.low,
"close": candle.close,
"volume": candle.volume or 0.0,
"is_closed": True,
"source": "local_simulator",
}
try:
live_store.ingest_bar(
symbol=symbol,
timeframe="1m",
bar={
"time": tsec,
"open": evt["open"],
"high": evt["high"],
"low": evt["low"],
"close": evt["close"],
"volume": evt["volume"],
},
)
except Exception:
pass
last_ts = tsec
await asyncio.sleep(poll_interval)
except asyncio.CancelledError:
break
except Exception:
await asyncio.sleep(poll_interval)
local_feed = LocalFeedProvider()
+168
View File
@@ -0,0 +1,168 @@
from __future__ import annotations
"""MetaTrader-powered price feed (scaffold).
The actual MT5 integration lives here so we can keep the rest of the app agnostic.
Implement the ``_connect`` and ``_fetch`` routines once MetaTrader 5 is available
on the host machine.
"""
import asyncio
from dataclasses import dataclass
from datetime import datetime
from typing import Any, Dict, Tuple
try:
import MetaTrader5 as mt5 # type: ignore
except Exception: # pragma: no cover - MetaTrader library not always installed
mt5 = None
from app.streaming.live_store import live_store
@dataclass(frozen=True)
class MTKey:
symbol: str
timeframe: str
class MetaTraderFeed:
def __init__(self) -> None:
self._tasks: Dict[MTKey, asyncio.Task] = {}
self._pinned: set[MTKey] = set()
self._lock = asyncio.Lock()
self._active_counts: Dict[MTKey, int] = {}
def get_status(self) -> list[dict]:
out: list[dict] = []
for key, task in self._tasks.items():
hist = live_store.get_history(key.symbol, key.timeframe)
last_ts = hist[-1]["time"] if hist else None
last_iso = None
if isinstance(last_ts, (int, float)):
last_iso = datetime.utcfromtimestamp(int(last_ts)).isoformat() + "Z"
out.append(
{
"symbol": key.symbol,
"timeframe": key.timeframe,
"subscribers": self._active_counts.get(key, 0),
"last_event_time": last_iso,
"connected": mt5 is not None and mt5.terminal_info() is not None,
"poller_running": not task.done(),
}
)
return out
async def subscribe(self, symbol: str, timeframe: str = "1m") -> Tuple[asyncio.Queue, Any]:
key = MTKey(symbol.upper().replace("/", ""), timeframe)
queue: asyncio.Queue = asyncio.Queue(maxsize=100)
live_store.subscribe(key.symbol, key.timeframe, queue)
async with self._lock:
self._active_counts[key] = self._active_counts.get(key, 0) + 1
if key not in self._tasks:
self._tasks[key] = asyncio.create_task(self._run_mt_poller(key))
async def _unsubscribe() -> None:
live_store.unsubscribe(key.symbol, key.timeframe, queue)
async with self._lock:
self._active_counts[key] = max(0, self._active_counts.get(key, 0) - 1)
if self._active_counts.get(key, 0) == 0 and key not in self._pinned:
task = self._tasks.pop(key, None)
if task:
task.cancel()
self._active_counts.pop(key, None)
return queue, _unsubscribe
async def ensure_stream(self, symbol: str, timeframe: str = "1m") -> None:
key = MTKey(symbol.upper().replace("/", ""), timeframe)
async with self._lock:
self._pinned.add(key)
self._active_counts.setdefault(key, 0)
if key not in self._tasks:
self._tasks[key] = asyncio.create_task(self._run_mt_poller(key))
async def release_stream(self, symbol: str, timeframe: str = "1m") -> None:
"""Allow external callers to drop pinning once no longer needed."""
key = MTKey(symbol.upper().replace("/", ""), timeframe)
async with self._lock:
self._pinned.discard(key)
if self._active_counts.get(key, 0) == 0:
task = self._tasks.pop(key, None)
if task:
task.cancel()
self._active_counts.pop(key, None)
async def _run_mt_poller(self, key: MTKey) -> None:
if mt5 is None:
raise RuntimeError("MetaTrader5 package not installed. Install to enable MT feed.")
if not mt5.initialize():
raise RuntimeError(f"Unable to initialize MetaTrader5: {mt5.last_error()}")
interval = self._resolve_timeframe(key.timeframe)
last_ts: int | None = None
poll_interval = 5
while True:
try:
rates = mt5.copy_rates_from_pos(key.symbol, interval, 0, 1)
if not rates:
await asyncio.sleep(poll_interval)
continue
row = rates[0]
tsec = int(row["time"])
if last_ts is not None and tsec <= last_ts:
await asyncio.sleep(poll_interval)
continue
last_ts = tsec
evt = {
"symbol": key.symbol,
"timeframe": key.timeframe,
"open_time": datetime.utcfromtimestamp(tsec).isoformat(),
"close_time": datetime.utcfromtimestamp(tsec).isoformat(),
"open": float(row["open"]),
"high": float(row["high"]),
"low": float(row["low"]),
"close": float(row["close"]),
"volume": float(row.get("tick_volume", 0.0)),
"is_closed": True,
"source": "metatrader",
}
live_store.ingest_bar(
symbol=key.symbol,
timeframe=key.timeframe,
bar={
"time": tsec,
"open": evt["open"],
"high": evt["high"],
"low": evt["low"],
"close": evt["close"],
"volume": evt["volume"],
},
)
await asyncio.sleep(poll_interval)
except asyncio.CancelledError:
break
except Exception:
await asyncio.sleep(poll_interval)
def _resolve_timeframe(self, name: str):
if mt5 is None:
raise RuntimeError("MetaTrader5 package not available")
mapping = {
"1m": mt5.TIMEFRAME_M1,
"5m": mt5.TIMEFRAME_M5,
"15m": mt5.TIMEFRAME_M15,
"30m": mt5.TIMEFRAME_M30,
"60m": mt5.TIMEFRAME_H1,
"1h": mt5.TIMEFRAME_H1,
}
return mapping.get(name.lower(), mt5.TIMEFRAME_M1)
metatrader_feed = MetaTraderFeed()