robinhood/backend/generate_historical_data.py

#!/usr/bin/env python3
"""
Generate realistic historical OHLCV data for XAUUSD (Gold vs USD)
Saves data as parquet files in the live store format for testing.
"""

import os
import random
import math
from datetime import datetime, timedelta
from typing import List, Dict, Any
import pyarrow as pa
import pyarrow.parquet as pq


def generate_gold_prices(start_date: datetime, end_date: datetime, base_price: float = 2000.0) -> List[Dict[str, Any]]:
    """
    Generate realistic 1-minute OHLCV data for gold prices.

    Gold price characteristics:
    - Base price around $2000/oz
    - Daily volatility ~0.5-1.5%
    - Higher volatility during market hours (London/New York overlap)
    - Weekend gaps
    - Trend following with mean reversion
    """

    data = []
    current_price = base_price
    trend = 0.0  # Trend direction (-1 to 1)
    volatility = 0.008  # Base volatility (0.8%)

    current = start_date
    while current <= end_date:
        # Skip weekends (Saturday=5, Sunday=6)
        if current.weekday() >= 5:
            current += timedelta(minutes=1)
            continue

        # Market hours: 00:00-23:59 UTC (24/7 for forex, but lower volume on weekends)
        hour = current.hour

        # Adjust volatility based on market session
        # London: 08:00-16:00 UTC, New York: 14:30-21:00 UTC
        # Overlap (high volume): 14:30-16:00 UTC
        if 14 <= hour < 16:
            session_volatility = volatility * 1.5  # Higher volatility during overlap
        elif 8 <= hour < 21:
            session_volatility = volatility * 1.2  # Active session
        else:
            session_volatility = volatility * 0.7  # Low volume

        # Random walk with trend and mean reversion
        trend_change = random.gauss(0, 0.001)  # Slow trend changes
        trend = max(-0.5, min(0.5, trend + trend_change))

        # Mean reversion to base price
        reversion = (base_price - current_price) / base_price * 0.001

        # Price change
        price_change_pct = random.gauss(trend * 0.0001 + reversion, session_volatility)
        price_change = current_price * price_change_pct
        new_price = current_price + price_change

        # Ensure reasonable bounds
        new_price = max(1500, min(3000, new_price))

        # Generate OHLC for 1-minute candle
        high = new_price + abs(random.gauss(0, new_price * session_volatility * 0.5))
        low = new_price - abs(random.gauss(0, new_price * session_volatility * 0.5))
        open_price = current_price + random.gauss(0, new_price * session_volatility * 0.3)
        close_price = new_price

        # Ensure OHLC relationships
        high = max(high, open_price, close_price)
        low = min(low, open_price, close_price)

        # Volume (simulated, higher during active hours)
        base_volume = random.randint(50, 200)
        if 14 <= hour < 16:
            volume = int(base_volume * 2.5)
        elif 8 <= hour < 21:
            volume = int(base_volume * 1.8)
        else:
            volume = base_volume

        bar = {
            "time": int(current.timestamp()),
            "open": round(open_price, 2),
            "high": round(high, 2),
            "low": round(low, 2),
            "close": round(close_price, 2),
            "volume": volume
        }

        data.append(bar)
        current_price = close_price
        current += timedelta(minutes=1)

    return data


def save_to_parquet(data: List[Dict[str, Any]], symbol: str, timeframe: str, base_dir: str = "data/parquet/live"):
    """Save OHLCV data to parquet files partitioned by date."""

    # Group by date
    from collections import defaultdict
    data_by_date = defaultdict(list)

    for bar in data:
        dt = datetime.utcfromtimestamp(bar["time"])
        date_str = dt.strftime("%Y-%m-%d")
        data_by_date[date_str].append(bar)

    # Save each date partition
    for date_str, bars in data_by_date.items():
        # Sort by time
        bars.sort(key=lambda x: x["time"])

        # Create directory
        partition_dir = os.path.join(base_dir, symbol, timeframe, f"date={date_str}")
        os.makedirs(partition_dir, exist_ok=True)

        # Convert to pyarrow table
        table = pa.Table.from_pylist(bars)

        # Save as parquet
        file_path = os.path.join(partition_dir, f"historical-{int(datetime.utcnow().timestamp())}.parquet")
        pq.write_table(table, file_path)
        print(f"Saved {len(bars)} bars for {date_str} to {file_path}")


def main():
    """Generate 3 months of historical data."""

    # Generate data for the last 90 days
    end_date = datetime.utcnow()
    start_date = end_date - timedelta(days=90)

    print(f"Generating historical data from {start_date.date()} to {end_date.date()}")

    # Generate data
    data = generate_gold_prices(start_date, end_date)

    print(f"Generated {len(data)} 1-minute bars")

    # Save to parquet
    save_to_parquet(data, "XAUUSD", "1m")

    print("Historical data generation complete!")


if __name__ == "__main__":
    main()