Initial commit: Gold Trading Simulator with AI-powered analysis
This commit is contained in:
Krikorios
@@ -0,0 +1,215 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Dict, Set, Tuple, Any
|
||||
|
||||
import httpx
|
||||
import asyncio
|
||||
import time
|
||||
import random
|
||||
|
||||
from app.config import settings
|
||||
from app.streaming.live_store import live_store
|
||||
|
||||
ALPHA_BASE = "https://www.alphavantage.co/query"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AVKey:
|
||||
symbol: str # e.g., XAUUSD
|
||||
timeframe: str # '1m' only for hub
|
||||
|
||||
|
||||
class AlphaVantageHub:
|
||||
"""
|
||||
Polls Alpha Vantage FX_INTRADAY for the latest bar per (symbol, 1m),
|
||||
fans out to subscribers via asyncio.Queue, and ingests into live_store.
|
||||
Ensures a single poller per (symbol,timeframe).
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._subs: Dict[AVKey, Set[asyncio.Queue]] = {}
|
||||
self._tasks: Dict[AVKey, asyncio.Task] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
# Conditional request caches
|
||||
self._etags: Dict[AVKey, str] = {}
|
||||
self._last_mod: Dict[AVKey, str] = {}
|
||||
# Global token-bucket (Alpha free tier ~5 req/min)
|
||||
self._rate_lock = asyncio.Lock()
|
||||
self._tokens: float = 5.0
|
||||
self._max_tokens: float = 5.0
|
||||
self._refill_rate_per_sec: float = 5.0 / 60.0
|
||||
self._last_refill_ts: float = time.time()
|
||||
|
||||
async def _acquire_token(self) -> None:
|
||||
# Simple async token bucket
|
||||
while True:
|
||||
async with self._rate_lock:
|
||||
now = time.time()
|
||||
elapsed = now - self._last_refill_ts
|
||||
if elapsed > 0:
|
||||
self._tokens = min(self._max_tokens, self._tokens + elapsed * self._refill_rate_per_sec)
|
||||
self._last_refill_ts = now
|
||||
if self._tokens >= 1.0:
|
||||
self._tokens -= 1.0
|
||||
return
|
||||
# Not enough tokens, compute wait time for next token
|
||||
need = 1.0 - self._tokens
|
||||
wait = max(0.1, need / self._refill_rate_per_sec)
|
||||
await asyncio.sleep(min(wait, 5.0))
|
||||
|
||||
def get_status(self) -> list[dict]:
|
||||
out: list[dict] = []
|
||||
for key, subs in self._subs.items():
|
||||
hist = live_store.get_history(key.symbol, key.timeframe)
|
||||
last_ts = hist[-1]["time"] if hist else None
|
||||
last_iso = None
|
||||
if isinstance(last_ts, (int, float)):
|
||||
try:
|
||||
last_iso = datetime.utcfromtimestamp(int(last_ts)).isoformat() + "Z"
|
||||
except Exception:
|
||||
last_iso = None
|
||||
out.append({
|
||||
"symbol": key.symbol,
|
||||
"timeframe": key.timeframe,
|
||||
"subscribers": len(subs),
|
||||
"last_event_time": last_iso,
|
||||
})
|
||||
return out
|
||||
|
||||
async def subscribe(self, symbol: str, timeframe: str = "1m") -> Tuple[asyncio.Queue, Any]:
|
||||
if timeframe != "1m":
|
||||
raise ValueError("AlphaVantageHub currently supports timeframe '1m' only")
|
||||
key = AVKey(symbol=symbol.upper().replace("/", ""), timeframe=timeframe)
|
||||
q: asyncio.Queue = asyncio.Queue(maxsize=100)
|
||||
async with self._lock:
|
||||
subs = self._subs.get(key)
|
||||
if not subs:
|
||||
subs = set()
|
||||
self._subs[key] = subs
|
||||
subs.add(q)
|
||||
if key not in self._tasks:
|
||||
self._tasks[key] = asyncio.create_task(self._run_poller(key))
|
||||
|
||||
async def _unsubscribe() -> None:
|
||||
async with self._lock:
|
||||
s = self._subs.get(key)
|
||||
if s and q in s:
|
||||
s.remove(q)
|
||||
try:
|
||||
q.put_nowait(None)
|
||||
except Exception:
|
||||
pass
|
||||
if s is not None and len(s) == 0:
|
||||
t = self._tasks.pop(key, None)
|
||||
if t:
|
||||
t.cancel()
|
||||
self._subs.pop(key, None)
|
||||
return q, _unsubscribe
|
||||
|
||||
async def _run_poller(self, key: AVKey) -> None:
|
||||
symbol = key.symbol
|
||||
from_symbol = symbol[:3]
|
||||
to_symbol = symbol[3:]
|
||||
apikey = settings.ALPHA_VANTAGE_API_KEY or "demo"
|
||||
last_ts: int | None = None
|
||||
poll_interval = 60 # seconds
|
||||
backoff_cap = 300 # max 5 min
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
while True:
|
||||
try:
|
||||
await self._acquire_token()
|
||||
params = {
|
||||
"function": "FX_INTRADAY",
|
||||
"from_symbol": from_symbol,
|
||||
"to_symbol": to_symbol,
|
||||
"interval": "1min",
|
||||
"outputsize": "compact",
|
||||
"apikey": apikey,
|
||||
}
|
||||
headers = {}
|
||||
et = self._etags.get(key)
|
||||
lm = self._last_mod.get(key)
|
||||
if et:
|
||||
headers["If-None-Match"] = et
|
||||
if lm:
|
||||
headers["If-Modified-Since"] = lm
|
||||
r = await client.get(ALPHA_BASE, params=params, headers=headers)
|
||||
if r.status_code == 304:
|
||||
# Not modified, keep interval
|
||||
delay = poll_interval + random.uniform(0, 2)
|
||||
await asyncio.sleep(delay)
|
||||
continue
|
||||
# Raise for other non-2xx
|
||||
r.raise_for_status()
|
||||
# Store caching headers for next time
|
||||
etag = r.headers.get("ETag")
|
||||
if etag:
|
||||
self._etags[key] = etag
|
||||
last_mod = r.headers.get("Last-Modified")
|
||||
if last_mod:
|
||||
self._last_mod[key] = last_mod
|
||||
js = r.json()
|
||||
series = js.get("Time Series FX (1min)") or {}
|
||||
if series:
|
||||
latest_ts_str = max(series.keys())
|
||||
dt = datetime.fromisoformat(latest_ts_str)
|
||||
tsec = int(dt.timestamp())
|
||||
if last_ts is None or tsec > last_ts:
|
||||
row = series[latest_ts_str]
|
||||
evt = {
|
||||
"symbol": symbol,
|
||||
"timeframe": key.timeframe,
|
||||
"open_time": dt.isoformat(),
|
||||
"close_time": dt.isoformat(),
|
||||
"open": float(row["1. open"]),
|
||||
"high": float(row["2. high"]),
|
||||
"low": float(row["3. low"]),
|
||||
"close": float(row["4. close"]),
|
||||
"volume": float(row.get("5. volume", 0.0)),
|
||||
"is_closed": True,
|
||||
"source": "alpha_vantage",
|
||||
}
|
||||
try:
|
||||
live_store.ingest_bar(symbol=symbol, timeframe="1m", bar={
|
||||
"time": tsec,
|
||||
"open": evt["open"],
|
||||
"high": evt["high"],
|
||||
"low": evt["low"],
|
||||
"close": evt["close"],
|
||||
"volume": evt["volume"],
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
subs = self._subs.get(key) or set()
|
||||
for q in list(subs):
|
||||
try:
|
||||
if q.full():
|
||||
q.get_nowait()
|
||||
q.put_nowait(evt)
|
||||
except Exception:
|
||||
try:
|
||||
subs.remove(q)
|
||||
except Exception:
|
||||
pass
|
||||
last_ts = tsec
|
||||
# success -> reset interval
|
||||
poll_interval = 60
|
||||
except httpx.HTTPStatusError as e:
|
||||
status = e.response.status_code if e.response else None
|
||||
# 429 or 5xx -> exponential backoff
|
||||
if status == 429 or (status and 500 <= status < 600):
|
||||
poll_interval = min(backoff_cap, max(60, int(poll_interval * 2)))
|
||||
# else, keep interval
|
||||
except Exception:
|
||||
# network or parse error
|
||||
poll_interval = min(backoff_cap, max(60, int(poll_interval * 2)))
|
||||
# sleep with small jitter
|
||||
delay = poll_interval + random.uniform(0, 2)
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
|
||||
# Singleton hub
|
||||
alpha_hub = AlphaVantageHub()
|
||||
@@ -0,0 +1,142 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Dict, Set, Tuple, Any
|
||||
|
||||
import websockets
|
||||
|
||||
from app.streaming.live_store import live_store
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StreamKey:
|
||||
symbol: str
|
||||
timeframe: str # only '1m' supported in hub
|
||||
|
||||
|
||||
class BinanceStreamHub:
|
||||
"""
|
||||
Maintains a single upstream websocket per (symbol,timeframe) and fans out
|
||||
kline events to multiple subscribers via asyncio.Queues.
|
||||
"""
|
||||
|
||||
def __init__(self, base_ws: str | None = None) -> None:
|
||||
self.base_ws = (base_ws or os.getenv("BINANCE_WS_URL", "wss://stream.binance.com:9443/ws")).rstrip("/")
|
||||
self._subs: Dict[StreamKey, Set[asyncio.Queue]] = {}
|
||||
self._tasks: Dict[StreamKey, asyncio.Task] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
def get_status(self) -> list[dict]:
|
||||
"""Return status snapshot of active streams."""
|
||||
out: list[dict] = []
|
||||
for key, subs in self._subs.items():
|
||||
hist = live_store.get_history(key.symbol, key.timeframe)
|
||||
last_ts = hist[-1]["time"] if hist else None
|
||||
last_iso = None
|
||||
if isinstance(last_ts, (int, float)):
|
||||
try:
|
||||
last_iso = datetime.utcfromtimestamp(int(last_ts)).isoformat() + "Z"
|
||||
except Exception:
|
||||
last_iso = None
|
||||
out.append({
|
||||
"symbol": key.symbol,
|
||||
"timeframe": key.timeframe,
|
||||
"subscribers": len(subs),
|
||||
"last_event_time": last_iso,
|
||||
})
|
||||
return out
|
||||
|
||||
async def subscribe(self, symbol: str, timeframe: str = "1m") -> Tuple[asyncio.Queue, Any]:
|
||||
"""Subscribe to a stream. Returns (queue, unsubscribe_cb)."""
|
||||
if timeframe != "1m":
|
||||
raise ValueError("BinanceStreamHub currently supports timeframe '1m' only")
|
||||
key = StreamKey(symbol=symbol.upper().replace("/", ""), timeframe=timeframe)
|
||||
q: asyncio.Queue = asyncio.Queue(maxsize=1000)
|
||||
async with self._lock:
|
||||
subs = self._subs.get(key)
|
||||
if not subs:
|
||||
subs = set()
|
||||
self._subs[key] = subs
|
||||
subs.add(q)
|
||||
if key not in self._tasks:
|
||||
self._tasks[key] = asyncio.create_task(self._run_stream(key))
|
||||
|
||||
async def _unsubscribe() -> None:
|
||||
async with self._lock:
|
||||
s = self._subs.get(key)
|
||||
if s and q in s:
|
||||
s.remove(q)
|
||||
# Close queue to unblock listeners
|
||||
try:
|
||||
q.put_nowait(None)
|
||||
except Exception:
|
||||
pass
|
||||
if s is not None and len(s) == 0:
|
||||
# cancel task and cleanup
|
||||
t = self._tasks.pop(key, None)
|
||||
if t:
|
||||
t.cancel()
|
||||
self._subs.pop(key, None)
|
||||
return q, _unsubscribe
|
||||
|
||||
async def _run_stream(self, key: StreamKey) -> None:
|
||||
symbol = key.symbol
|
||||
stream = f"{symbol.lower()}@kline_{key.timeframe}"
|
||||
url = self.base_ws.replace("/ws", "/stream") + f"?streams={stream}"
|
||||
# Reconnect loop
|
||||
while True:
|
||||
try:
|
||||
async with websockets.connect(url, ping_interval=20, ping_timeout=20) as ws:
|
||||
async for message in ws:
|
||||
try:
|
||||
data = json.loads(message)
|
||||
k = (data.get("data") or {}).get("k") or {}
|
||||
if not k:
|
||||
continue
|
||||
# Normalize event
|
||||
evt = {
|
||||
"symbol": symbol,
|
||||
"timeframe": key.timeframe,
|
||||
"open_time": datetime.fromtimestamp(k["t"] / 1000.0).isoformat(),
|
||||
"close_time": datetime.fromtimestamp(k["T"] / 1000.0).isoformat(),
|
||||
"open": float(k["o"]),
|
||||
"high": float(k["h"]),
|
||||
"low": float(k["l"]),
|
||||
"close": float(k["c"]),
|
||||
"volume": float(k.get("v", 0.0)),
|
||||
"is_closed": bool(k.get("x", False)),
|
||||
"source": "binance",
|
||||
}
|
||||
# Update live store (1m bar)
|
||||
try:
|
||||
tsec = int(k["T"] // 1000)
|
||||
live_store.ingest_bar(symbol=symbol, timeframe=key.timeframe, bar={
|
||||
"time": tsec, "open": evt["open"], "high": evt["high"], "low": evt["low"], "close": evt["close"], "volume": evt["volume"],
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
# Fan-out to subscribers
|
||||
subs = self._subs.get(key) or set()
|
||||
for q in list(subs):
|
||||
try:
|
||||
if q.full():
|
||||
q.get_nowait()
|
||||
q.put_nowait(evt)
|
||||
except Exception:
|
||||
# Drop failed subscriber
|
||||
try:
|
||||
subs.remove(q)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
continue
|
||||
except Exception:
|
||||
await asyncio.sleep(1.5)
|
||||
|
||||
|
||||
# Singleton hub instance
|
||||
hub = BinanceStreamHub()
|
||||
@@ -0,0 +1,173 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Tuple, Any
|
||||
import glob
|
||||
import shutil
|
||||
|
||||
import pyarrow as pa
|
||||
import pyarrow.parquet as pq
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Series:
|
||||
bars: List[Dict[str, Any]]
|
||||
last_flushed_ts: int
|
||||
|
||||
|
||||
class LiveStore:
|
||||
def __init__(self, max_bars: int = 5000, root: str = "data/parquet/live") -> None:
|
||||
self._series: Dict[Tuple[str, str], _Series] = {}
|
||||
self._max_bars = max_bars
|
||||
self._root = root
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
@property
|
||||
def root(self) -> str:
|
||||
return self._root
|
||||
|
||||
def _get_series(self, symbol: str, timeframe: str) -> _Series:
|
||||
key = (symbol, timeframe)
|
||||
s = self._series.get(key)
|
||||
if not s:
|
||||
s = _Series(bars=[], last_flushed_ts=0)
|
||||
self._series[key] = s
|
||||
return s
|
||||
|
||||
def get_history(self, symbol: str, timeframe: str) -> List[Dict[str, Any]]:
|
||||
s = self._get_series(symbol, timeframe)
|
||||
return list(s.bars)
|
||||
|
||||
def ingest_bar(self, symbol: str, timeframe: str, bar: Dict[str, Any]) -> None:
|
||||
s = self._get_series(symbol, timeframe)
|
||||
if s.bars and s.bars[-1]["time"] == bar["time"]:
|
||||
# update last
|
||||
last = s.bars[-1]
|
||||
last["high"] = max(last["high"], bar["high"])
|
||||
last["low"] = min(last["low"], bar["low"])
|
||||
last["close"] = bar["close"]
|
||||
last["volume"] = last.get("volume", 0.0) + bar.get("volume", 0.0)
|
||||
else:
|
||||
s.bars.append(bar)
|
||||
if len(s.bars) > self._max_bars:
|
||||
s.bars.pop(0)
|
||||
|
||||
async def flush_parquet(self) -> None:
|
||||
# Write new bars since last flush, partitioned by date
|
||||
async with self._lock:
|
||||
for (symbol, timeframe), s in self._series.items():
|
||||
new_rows = [b for b in s.bars if b["time"] > s.last_flushed_ts]
|
||||
if not new_rows:
|
||||
continue
|
||||
# Partition by date
|
||||
rows_by_date: Dict[str, List[Dict[str, Any]]] = defaultdict(list)
|
||||
for r in new_rows:
|
||||
dt = datetime.utcfromtimestamp(int(r["time"]))
|
||||
rows_by_date[dt.strftime("%Y-%m-%d")].append(r)
|
||||
for date_str, rows in rows_by_date.items():
|
||||
table = pa.Table.from_pylist(rows)
|
||||
base = os.path.join(self._root, symbol, timeframe)
|
||||
out = os.path.join(base, f"date={date_str}")
|
||||
os.makedirs(out, exist_ok=True)
|
||||
# write one file per flush to this partition
|
||||
pq.write_table(table, os.path.join(out, f"part-{int(datetime.utcnow().timestamp())}.parquet"))
|
||||
s.last_flushed_ts = max(b["time"] for b in new_rows)
|
||||
|
||||
|
||||
# Singleton store
|
||||
live_store = LiveStore()
|
||||
|
||||
|
||||
async def periodic_flush(interval_sec: int = 60):
|
||||
while True:
|
||||
try:
|
||||
await live_store.flush_parquet()
|
||||
except Exception:
|
||||
pass
|
||||
await asyncio.sleep(interval_sec)
|
||||
|
||||
|
||||
def _iter_partitions(root: str):
|
||||
"""Yield (symbol, timeframe, partition_path, date_str) for existing partitions."""
|
||||
# root/symbol/timeframe/date=YYYY-MM-DD
|
||||
for sym_dir in glob.glob(f"{root}/*"):
|
||||
if not os.path.isdir(sym_dir):
|
||||
continue
|
||||
symbol = os.path.basename(sym_dir)
|
||||
for tf_dir in glob.glob(f"{sym_dir}/*"):
|
||||
if not os.path.isdir(tf_dir):
|
||||
continue
|
||||
timeframe = os.path.basename(tf_dir)
|
||||
for part_dir in glob.glob(f"{tf_dir}/date=*" ):
|
||||
if not os.path.isdir(part_dir):
|
||||
continue
|
||||
date_str = os.path.basename(part_dir).split("=", 1)[-1]
|
||||
yield (symbol, timeframe, part_dir, date_str)
|
||||
|
||||
|
||||
def prune_old_partitions(root: str, retention_days: int = 7) -> int:
|
||||
"""Delete partition directories older than retention_days. Returns count deleted."""
|
||||
now = datetime.utcnow()
|
||||
deleted = 0
|
||||
for symbol, timeframe, part_dir, date_str in list(_iter_partitions(root)):
|
||||
try:
|
||||
y, m, d = map(int, date_str.split("-"))
|
||||
dt = datetime(y, m, d)
|
||||
if now - dt > timedelta(days=retention_days):
|
||||
shutil.rmtree(part_dir, ignore_errors=True)
|
||||
deleted += 1
|
||||
except Exception:
|
||||
# Skip unparsable date partitions
|
||||
continue
|
||||
return deleted
|
||||
|
||||
|
||||
def compact_partition(part_dir: str, max_files_threshold: int = 20) -> bool:
|
||||
"""If too many small part files exist, compact them into a single file.
|
||||
Returns True if compaction performed.
|
||||
"""
|
||||
part_files = sorted(glob.glob(os.path.join(part_dir, "part-*.parquet")))
|
||||
if len(part_files) < max_files_threshold:
|
||||
return False
|
||||
try:
|
||||
tables: List[pa.Table] = []
|
||||
for p in part_files:
|
||||
tables.append(pq.read_table(p))
|
||||
if not tables:
|
||||
return False
|
||||
combined = pa.concat_tables(tables, promote=True)
|
||||
out_file = os.path.join(part_dir, f"compact-{int(datetime.utcnow().timestamp())}.parquet")
|
||||
pq.write_table(combined, out_file)
|
||||
# remove old parts
|
||||
for p in part_files:
|
||||
try:
|
||||
os.remove(p)
|
||||
except Exception:
|
||||
pass
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def compact_all(root: str, max_files_threshold: int = 20) -> int:
|
||||
"""Run compaction across all partitions. Returns number of partitions compacted."""
|
||||
compacted = 0
|
||||
for _, _, part_dir, _ in list(_iter_partitions(root)):
|
||||
if compact_partition(part_dir, max_files_threshold=max_files_threshold):
|
||||
compacted += 1
|
||||
return compacted
|
||||
|
||||
|
||||
async def periodic_maintenance(retention_days: int = 7, compact_threshold_files: int = 20, interval_sec: int = 900):
|
||||
"""Periodically prune old partitions and compact small files."""
|
||||
while True:
|
||||
try:
|
||||
prune_old_partitions(live_store.root, retention_days=retention_days)
|
||||
compact_all(live_store.root, max_files_threshold=compact_threshold_files)
|
||||
except Exception:
|
||||
pass
|
||||
await asyncio.sleep(interval_sec)
|
||||
Reference in New Issue
Block a user