diff --git a/.claude/skills/reclamm-pair-onboarding/SKILL.md b/.claude/skills/reclamm-pair-onboarding/SKILL.md index 880930c..8b75f84 100644 --- a/.claude/skills/reclamm-pair-onboarding/SKILL.md +++ b/.claude/skills/reclamm-pair-onboarding/SKILL.md @@ -20,12 +20,22 @@ and cached automatically under `results/mm_noise/_sim_arrays/`. parquet per pool token (`TOKEN_MAP` in `quantammsim/calibration/market_features.py` maps WBTC→BTC, WETH→ETH, USDT→USDC, …). +`scripts/download_data.py` handles both sources — one command, any token: + - **Binance-listed token**: `python scripts/download_data.py `. -- **Not on Binance**: copy `scripts/prepare_bold_usdc_data.py` (CoinGecko): - daily close+volume → minute grid by forward-fill, daily volume spread /1440 - (so the daily resample recovers real volume), stables as flat $1.00 peg. - CoinGecko free tier = **last 365 days only** — this bounds the earliest - simulation start and usually forces a per-pair train window (Step 4). +- **Not on Binance**: same script auto-falls back to CoinGecko. If the token is + in the built-in registry (`COINGECKO_IDS` in + `quantammsim/utils/data_processing/coingecko_data.py`) just run + `python scripts/download_data.py `; otherwise pass the id explicitly. + For a stable-paired volatile token, add `--peg ` to write/extend the + stablecoin's flat-$1 parquet over the same window. BOLD/USDC example: + `python scripts/download_data.py BOLD --cg-id liquity-bold-2 --peg USDC`. + (`--source coingecko` forces CoinGecko even for a Binance-listed token.) + + CoinGecko builds the minute grid by forward-fill with daily volume spread /1440 + (so the daily resample recovers real volume). Free tier = **last 365 days + only** — this bounds the earliest simulation start and usually forces a + per-pair train window (Step 4). Check coverage: the parquet must span train start → test end. diff --git a/quantammsim/utils/data_processing/coingecko_data.py b/quantammsim/utils/data_processing/coingecko_data.py new file mode 100644 index 0000000..4c8ca7c --- /dev/null +++ b/quantammsim/utils/data_processing/coingecko_data.py @@ -0,0 +1,284 @@ +"""CoinGecko data source for the historic-data pipeline. + +Generalizes the one-off ``scripts/prepare_bold_usdc_data.py`` so that any token +listed on CoinGecko can be fetched through the same waterfall as Binance/Coinbase. + +Two public entry points: + +- :func:`get_coingecko_data` — fetch a token's daily price + volume from CoinGecko + and expand it to a complete 1-minute grid in the pipeline's canonical schema + (unix-indexed; columns ``date, symbol, open, high, low, close, Volume USD, + Volume ``). Daily USD volume is spread evenly across the 1440 minutes of + each day so a daily resample-sum downstream recovers the true daily volume. +- :func:`rebuild_stable_peg` — write/extend ``_USD.parquet`` at a flat + $1.00 peg over the union of any existing coverage and a requested window. + +CoinGecko free/demo tier caps ``market_chart`` history at the last **365 days**, +which bounds the earliest possible simulation start for CoinGecko-only tokens. +""" + +from __future__ import annotations + +import time +from pathlib import Path + +import numpy as np +import pandas as pd +import requests + +# --------------------------------------------------------------------------- +# Symbol -> CoinGecko id registry (single source of truth; imported by +# scripts/fetch_token_mcaps.py). Covers the tokens in our matched pools plus +# common Balancer tokens. Pass --cg-id on the command line for anything absent. +# --------------------------------------------------------------------------- +COINGECKO_IDS = { + # Blue-chip / wrapped natives + "WETH": "ethereum", + "ETH": "ethereum", + "WBTC": "wrapped-bitcoin", + "BTC": "bitcoin", + "cbBTC": "bitcoin", # Coinbase wrapped BTC — use BTC mcap + "USDC": "usd-coin", + "USDT": "tether", + "DAI": "dai", + "wstETH": "wrapped-steth", + "stETH": "staked-ether", + "rETH": "rocket-pool-eth", + "cbETH": "coinbase-wrapped-staked-eth", + "WMATIC": "polygon-ecosystem-token", + "MATIC": "polygon-ecosystem-token", + "POL": "polygon-ecosystem-token", + "WAVAX": "avalanche-2", + "AVAX": "avalanche-2", + "GNO": "gnosis", + "WXDAI": "dai", # Wrapped xDAI ≈ DAI + "xDAI": "dai", + "S": "sonic-3", + "wS": "sonic-3", + # Mid-cap DeFi + "AAVE": "aave", + "LINK": "chainlink", + "UNI": "uniswap", + "BAL": "balancer", + "MKR": "maker", + "CRV": "curve-dao-token", + "COMP": "compound-governance-token", + "SNX": "havven", + "LDO": "lido-dao", + "RPL": "rocket-pool", + "SUSHI": "sushi", + "YFI": "yearn-finance", + "1INCH": "1inch", + "ENS": "ethereum-name-service", + "ARB": "arbitrum", + "OP": "optimism", + "PENDLE": "pendle", + "ENA": "ethena", + "EIGEN": "eigenlayer", + "COW": "cow-protocol", + "SAFE": "safe", + # Smaller / specific tokens in our pools + "ACX": "across-protocol", + "ALCX": "alchemix", + "QI": "benqi", + "QNT": "quant-network", + "RDNT": "radiant-capital", + # TREE not on CoinGecko — handled as fallback below + "XAI": "xai-blockchain", + # Wrapped aTokens — use underlying + "waEthLidoWETH": "ethereum", + "waEthLidowstETH": "wrapped-steth", + "waBasWETH": "ethereum", + "waBasUSDC": "usd-coin", + "waEthUSDC": "usd-coin", + "waGnoGNO": "gnosis", + "waGnowstETH": "wrapped-steth", + # Additional tokens from expanded pool set + "wPOL": "polygon-ecosystem-token", + "stS": "sonic-3", # Staked Sonic — use S mcap + "JitoSOL": "jito-governance-token", + "scUSD": "usd-coin", # Rings scUSD stablecoin — use USDC mcap as proxy + "DOLA": "dola-usd", + # CoinGecko-only tokens (not on Binance/Coinbase) + "BOLD": "liquity-bold-2", +} + +# Asset type classification +STABLECOINS = { + "USDC", "USDT", "DAI", "WXDAI", "xDAI", "GHO", "LUSD", "crvUSD", + "FRAX", "sDAI", "scUSD", "DOLA", + "waBasUSDC", "waEthUSDC", +} +NATIVE_LST = { + "WETH", "ETH", "wstETH", "stETH", "rETH", "cbETH", + "WBTC", "BTC", "cbBTC", + "WMATIC", "MATIC", "POL", "wPOL", + "WAVAX", "AVAX", + "GNO", "S", "wS", "stS", + "JitoSOL", + "waEthLidoWETH", "waEthLidowstETH", + "waBasWETH", "waGnoGNO", "waGnowstETH", +} + +_CG_BASE = "https://api.coingecko.com/api/v3" + + +def resolve_cg_id(token: str, override: str | None = None) -> str: + """Resolve a token symbol to its CoinGecko id. + + ``override`` (the ``--cg-id`` flag) wins; otherwise look up the built-in + registry. Raises a clear error if neither is available. + """ + if override: + return override + cg_id = COINGECKO_IDS.get(token) or COINGECKO_IDS.get(token.upper()) + if cg_id: + return cg_id + raise ValueError( + f"No CoinGecko id known for token '{token}'. Pass it explicitly with " + f"--cg-id (find the id in the token's CoinGecko URL, " + f"e.g. https://www.coingecko.com/en/coins/)." + ) + + +def fetch_coingecko_daily(cg_id: str, days: str = "365") -> tuple[list, list]: + """Return ``(prices, total_volumes)`` as ``[[unix_ms, value], ...]``. + + Retries with a back-off on transient failures and honours 429 rate limits. + Free/demo tier caps ``days`` at 365. + """ + url = f"{_CG_BASE}/coins/{cg_id}/market_chart" + params = {"vs_currency": "usd", "days": str(days), "interval": "daily"} + for attempt in range(4): + try: + resp = requests.get(url, params=params, timeout=40) + if resp.status_code == 429: + print(" rate-limited, sleeping 60s …") + time.sleep(60) + continue + resp.raise_for_status() + j = resp.json() + return j.get("prices", []), j.get("total_volumes", []) + except Exception as exc: + print(f" CoinGecko attempt {attempt + 1} failed: {exc}") + time.sleep(10) + raise RuntimeError(f"CoinGecko fetch failed for id '{cg_id}'") + + +def get_coingecko_data( + token: str, + root: str, + cg_id: str | None = None, + days: str = "365", + numeraire: str = "USD", +) -> pd.DataFrame | None: + """Fetch ``token`` from CoinGecko as a unix-indexed 1-minute canonical frame. + + Matches the schema the exchange helpers feed into ``update_historic_data``: + unix (int64 ms) index; columns ``date, symbol, open, high, low, close, + Volume USD, Volume ``. Daily close is forward-filled across each day; + daily USD volume is spread ``/1440`` so a daily resample-sum recovers the + real daily volume. ``root`` is accepted for signature parity with the other + ``get_*_data`` source helpers (CoinGecko needs no local cache dir). + + Returns ``None`` if CoinGecko yields too little history to be usable. + """ + resolved_id = resolve_cg_id(token, cg_id) + print(f"Fetching {token} ({resolved_id}) daily data from CoinGecko …") + prices, volumes = fetch_coingecko_daily(resolved_id, days=days) + if len(prices) < 2: + print(f"CoinGecko returned too little data for {token} ({len(prices)} points)") + return None + + # Daily frame: close price + daily USD volume, indexed by midnight-UTC day. + pdf = pd.DataFrame(prices, columns=["unix", "close"]) + vdf = pd.DataFrame(volumes, columns=["unix", "vol_usd"]) + pdf["day"] = pd.to_datetime(pdf["unix"], unit="ms", utc=True).dt.normalize() + vdf["day"] = pd.to_datetime(vdf["unix"], unit="ms", utc=True).dt.normalize() + daily = ( + pdf.groupby("day")["close"].last().to_frame() + .join(vdf.groupby("day")["vol_usd"].last()) + ) + daily["vol_usd"] = daily["vol_usd"].fillna(0.0) + daily = daily.dropna(subset=["close"]).sort_index() + + start = daily.index.min() + end = daily.index.max() + print( + f" {token} daily: {start.date()} → {end.date()} ({len(daily)} days), " + f"close ${daily['close'].min():.4f}–${daily['close'].max():.4f}" + ) + + # Expand to a complete 1-minute grid covering full days [start, end+1day). + epoch = pd.Timestamp("1970-01-01", tz="UTC") + grid_end = end + pd.Timedelta(days=1) - pd.Timedelta(minutes=1) + min_range = pd.date_range(start=start, end=grid_end, freq="1min") + unix_idx = ((min_range - epoch).total_seconds() * 1000).astype(np.int64) + + out = pd.DataFrame(index=unix_idx) + out.index.name = "unix" + out["date"] = min_range.tz_localize(None).strftime("%Y-%m-%d %H:%M:%S") + day_key = min_range.normalize() + + close_map = daily["close"].to_dict() + vol_map = (daily["vol_usd"] / 1440.0).to_dict() + out["close"] = pd.Series(day_key.map(close_map), index=unix_idx).astype(float) + out["Volume USD"] = pd.Series(day_key.map(vol_map), index=unix_idx).astype(float) + out["close"] = out["close"].ffill().bfill() + out["Volume USD"] = out["Volume USD"].fillna(0.0) + out["open"] = out["high"] = out["low"] = out["close"] + out[f"Volume {token}"] = out["Volume USD"] / out["close"].clip(lower=1e-9) + out["symbol"] = f"{token}/{numeraire}" + + out = out[ + ["date", "symbol", "open", "high", "low", "close", + "Volume USD", f"Volume {token}"] + ] + return out + + +def rebuild_stable_peg( + stable_token: str, + root: str, + cover_start_ms: int, + cover_end_ms: int, +) -> None: + """Write/extend ``_USD.parquet`` at a flat $1.00 peg. + + Covers the union of any existing coverage and ``[cover_start_ms, + cover_end_ms]`` on a 1-minute grid. Volume columns are zero. ``root`` is the + data directory (with trailing slash), matching ``update_historic_data``. + """ + path = Path(root) / f"{stable_token}_USD.parquet" + start_ms = int(cover_start_ms) + end_ms = int(cover_end_ms) + if path.exists(): + existing = pd.read_parquet(path) + idx_vals = ( + existing.index if existing.index.name == "unix" else existing["unix"] + ) + start_ms = min(start_ms, int(np.asarray(idx_vals).min())) + end_ms = max(end_ms, int(np.asarray(idx_vals).max())) + + idx = np.arange(start_ms, end_ms + 60_000, 60_000, dtype=np.int64) + n = len(idx) + out = pd.DataFrame( + { + "unix": idx, + "date": pd.to_datetime(idx, unit="ms", utc=True) + .tz_localize(None) + .strftime("%Y-%m-%d %H:%M:%S"), + "symbol": f"{stable_token}/USD", + "open": np.ones(n), + "high": np.ones(n), + "low": np.ones(n), + "close": np.ones(n), + "Volume USD": np.zeros(n), + f"Volume {stable_token}": np.zeros(n), + } + ) + out.to_parquet(path, engine="pyarrow", index=False) + print( + f" Saved {n:,} rows → {path} (peg $1.00, " + f"{out['date'].iloc[0]} → {out['date'].iloc[-1]})" + ) diff --git a/quantammsim/utils/data_processing/historic_data_utils.py b/quantammsim/utils/data_processing/historic_data_utils.py index 63e73c4..d306b33 100644 --- a/quantammsim/utils/data_processing/historic_data_utils.py +++ b/quantammsim/utils/data_processing/historic_data_utils.py @@ -33,6 +33,9 @@ from quantammsim.utils.data_processing.aerodrome_data_utils import ( fill_missing_rows_with_aerodrome_data, ) +from quantammsim.utils.data_processing.coingecko_data import ( + get_coingecko_data, +) from quantammsim.utils.data_processing.minute_daily_conversion_utils import ( calculate_annualised_daily_volatility_from_minute_data, expand_daily_to_minute_data, @@ -797,12 +800,17 @@ def get_coinbase_live_data(token, numeraire): return standardized_df -def update_historic_data(token, root): +def update_historic_data(token, root, source="auto", cg_id=None, cg_days="365"): """Update historic data for a given token, handling reruns gracefully. Args: token (str): Token symbol (e.g., 'BTC', 'ETH') root (str): Root directory path + source (str): "auto" (default) runs the exchange waterfall and falls back + to CoinGecko only if nothing is found; "coingecko" fetches straight + from CoinGecko; "binance" runs the waterfall with no CoinGecko fallback. + cg_id (str, optional): CoinGecko id override for CoinGecko fetches. + cg_days (str): CoinGecko history window in days (free tier caps at 365). """ outputPath = root + "combined_data/" parquetPath = outputPath + token + "_USD.parquet" @@ -818,6 +826,19 @@ def update_historic_data(token, root): quote_currency_candidates = get_quote_currency_candidates(token) primary_numeraire = quote_currency_candidates[0] concated_df = None + + # Explicit CoinGecko source: skip the exchange waterfall entirely. + if source == "coingecko": + print(f"Fetching {token} directly from CoinGecko (source=coingecko)") + concated_df = get_coingecko_data(token, root, cg_id=cg_id, days=cg_days) + if concated_df is None or concated_df.empty: + raise FileNotFoundError(f"No CoinGecko data found for {token}") + filled_timestamps["CoinGecko"] = concated_df.index.tolist() + return _finalize_historic_data( + token, concated_df, filled_timestamps, root, outputPath, + parquetPath, minutePath, dailyPath, hourlyPath, + ) + filled_timestamps["Binance Vision"] = [] # Try binance.vision data first, using a USD quote for assets like USDT. @@ -994,10 +1015,33 @@ def update_historic_data(token, root): filled_timestamps["Aerodrome"] = filled_aerodrome_unix_values print("Filled aerodrome data") print(len(filled_aerodrome_unix_values)) + # Fall back to CoinGecko when the exchange waterfall found nothing. + if concated_df.empty and source == "auto": + print(f"No exchange data for {token}; falling back to CoinGecko") + cg_df = get_coingecko_data(token, root, cg_id=cg_id, days=cg_days) + if cg_df is not None and not cg_df.empty: + concated_df = cg_df + filled_timestamps["CoinGecko"] = concated_df.index.tolist() if concated_df.empty: raise FileNotFoundError( f"No market data found for {token} using quote currencies {quote_currency_candidates}" ) + return _finalize_historic_data( + token, concated_df, filled_timestamps, root, outputPath, + parquetPath, minutePath, dailyPath, hourlyPath, + ) + + +def _finalize_historic_data( + token, concated_df, filled_timestamps, root, outputPath, + parquetPath, minutePath, dailyPath, hourlyPath, +): + """Sort/dedup, enforce the 1-minute grid, and write CSVs/plots/parquet. + + Shared tail of :func:`update_historic_data` — reused by every source + (exchange waterfall and CoinGecko) so all tokens get identical outputs. + ``concated_df`` must be unix-indexed on entry. + """ # Ensure data is properly sorted and has no duplicates concated_df = concated_df.sort_index() concated_df = concated_df[~concated_df.index.duplicated(keep="first")] @@ -1034,6 +1078,7 @@ def update_historic_data(token, root): "Binance CDD": "orange", "Binance Vision": "cyan", "Candles": "magenta", + "CoinGecko": "black", } for source, timestamps in filled_timestamps.items(): if timestamps: diff --git a/scripts/download_data.py b/scripts/download_data.py index c004bcb..62e40af 100644 --- a/scripts/download_data.py +++ b/scripts/download_data.py @@ -1,12 +1,12 @@ import os -import hashlib -import zipfile import argparse from pathlib import Path -from tqdm import tqdm from quantammsim.utils.data_processing.historic_data_utils import ( update_historic_data, ) +from quantammsim.utils.data_processing.coingecko_data import ( + rebuild_stable_peg, +) # Get absolute paths relative to this script SCRIPT_DIR = Path(__file__).resolve().parent @@ -17,6 +17,15 @@ TICKER_FILE = SCRIPT_DIR / "ticker_list.txt" + +def _token_unix_span(parquet_path): + """Return (min_unix_ms, max_unix_ms) for a written token parquet.""" + import pandas as pd + + df = pd.read_parquet(parquet_path, columns=["unix"]) + return int(df["unix"].min()), int(df["unix"].max()) + + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Download historic data for tickers.") parser.add_argument( @@ -24,6 +33,40 @@ nargs="*", help="List of tickers to process. If provided, overrides ticker_list.txt", ) + parser.add_argument( + "--source", + choices=["auto", "binance", "coingecko"], + default="auto", + help=( + "auto (default): try Binance/Coinbase/etc, fall back to CoinGecko if " + "nothing is found. binance: exchange waterfall only. coingecko: fetch " + "straight from CoinGecko." + ), + ) + parser.add_argument( + "--cg-id", + default=None, + help=( + "CoinGecko id override for a token not in the built-in registry " + "(e.g. --cg-id liquity-bold-2). Single-token runs only." + ), + ) + parser.add_argument( + "--cg-days", + default="365", + help="CoinGecko history window in days (free tier caps at 365).", + ) + parser.add_argument( + "--peg", + action="append", + default=[], + metavar="STABLE", + help=( + "After downloading, write/extend _USD.parquet at a flat $1.00 " + "peg covering the downloaded token's window (repeatable). " + "e.g. BOLD --cg-id liquity-bold-2 --peg USDC" + ), + ) args = parser.parse_args() if args.tickers: @@ -32,16 +75,34 @@ with open(TICKER_FILE, "r") as f: tickers = f.readlines() + if (args.cg_id or args.peg) and len(tickers) != 1: + parser.error("--cg-id and --peg require exactly one ticker.") + for token in tickers: token = token.strip().upper() print(f"Processing {token}") - update_historic_data(token, DATA_DIR_STR) + update_historic_data( + token, + DATA_DIR_STR, + source=args.source, + cg_id=args.cg_id, + cg_days=args.cg_days, + ) print(f"Finished processing {token}") + token_parquet = DATA_DIR_STR + token + "_USD.parquet" os.rename( DATA_DIR_STR + "combined_data/" + token + "_USD.parquet", - DATA_DIR_STR + token + "_USD.parquet", + token_parquet, ) os.rename( - DATA_DIR_STR + "combined_data/" + token.strip().upper() + "_USD_daily.csv", - DATA_DIR_STR + token.strip().upper() + "_USD_daily.csv", + DATA_DIR_STR + "combined_data/" + token + "_USD_daily.csv", + DATA_DIR_STR + token + "_USD_daily.csv", ) + + # Rebuild/extend any requested stablecoin pegs to cover this token's window. + if args.peg: + start_ms, end_ms = _token_unix_span(token_parquet) + for stable in args.peg: + stable = stable.strip().upper() + print(f"Rebuilding {stable} peg to cover {token} window") + rebuild_stable_peg(stable, DATA_DIR_STR, start_ms, end_ms) diff --git a/scripts/fetch_token_mcaps.py b/scripts/fetch_token_mcaps.py index 8e71d6f..9053ce2 100644 --- a/scripts/fetch_token_mcaps.py +++ b/scripts/fetch_token_mcaps.py @@ -19,95 +19,14 @@ "local_data", "noise_calibration", "token_mcaps.json", ) -# Token symbol -> CoinGecko ID mapping -# Covers all tokens appearing in our 26 matched pools + common Balancer tokens -COINGECKO_IDS = { - # Blue-chip / wrapped natives - "WETH": "ethereum", - "ETH": "ethereum", - "WBTC": "wrapped-bitcoin", - "BTC": "bitcoin", - "cbBTC": "bitcoin", # Coinbase wrapped BTC — use BTC mcap - "USDC": "usd-coin", - "USDT": "tether", - "DAI": "dai", - "wstETH": "wrapped-steth", - "stETH": "staked-ether", - "rETH": "rocket-pool-eth", - "cbETH": "coinbase-wrapped-staked-eth", - "WMATIC": "polygon-ecosystem-token", - "MATIC": "polygon-ecosystem-token", - "POL": "polygon-ecosystem-token", - "WAVAX": "avalanche-2", - "AVAX": "avalanche-2", - "GNO": "gnosis", - "WXDAI": "dai", # Wrapped xDAI ≈ DAI - "xDAI": "dai", - "S": "sonic-3", - "wS": "sonic-3", - # Mid-cap DeFi - "AAVE": "aave", - "LINK": "chainlink", - "UNI": "uniswap", - "BAL": "balancer", - "MKR": "maker", - "CRV": "curve-dao-token", - "COMP": "compound-governance-token", - "SNX": "havven", - "LDO": "lido-dao", - "RPL": "rocket-pool", - "SUSHI": "sushi", - "YFI": "yearn-finance", - "1INCH": "1inch", - "ENS": "ethereum-name-service", - "ARB": "arbitrum", - "OP": "optimism", - "PENDLE": "pendle", - "ENA": "ethena", - "EIGEN": "eigenlayer", - "COW": "cow-protocol", - "SAFE": "safe", - # Smaller / specific tokens in our pools - "ACX": "across-protocol", - "ALCX": "alchemix", - "QI": "benqi", - "QNT": "quant-network", - "RDNT": "radiant-capital", - # TREE not on CoinGecko — handled as fallback below - "XAI": "xai-blockchain", - # Wrapped aTokens — use underlying - "waEthLidoWETH": "ethereum", - "waEthLidowstETH": "wrapped-steth", - "waBasWETH": "ethereum", - "waBasUSDC": "usd-coin", - "waEthUSDC": "usd-coin", - "waGnoGNO": "gnosis", - "waGnowstETH": "wrapped-steth", - # Additional tokens from expanded pool set - "wPOL": "polygon-ecosystem-token", - "stS": "sonic-3", # Staked Sonic — use S mcap - "JitoSOL": "jito-governance-token", - "scUSD": "usd-coin", # Rings scUSD stablecoin — use USDC mcap as proxy - "DOLA": "dola-usd", -} - -# Asset type classification -STABLECOINS = { - "USDC", "USDT", "DAI", "WXDAI", "xDAI", "GHO", "LUSD", "crvUSD", - "FRAX", "sDAI", "scUSD", "DOLA", - "waBasUSDC", "waEthUSDC", -} -NATIVE_LST = { - "WETH", "ETH", "wstETH", "stETH", "rETH", "cbETH", - "WBTC", "BTC", "cbBTC", - "WMATIC", "MATIC", "POL", "wPOL", - "WAVAX", "AVAX", - "GNO", "S", "wS", "stS", - "JitoSOL", - "waEthLidoWETH", "waEthLidowstETH", - "waBasWETH", "waGnoGNO", "waGnowstETH", -} -# Everything else is VOLATILE (asset_type=2) +# Token symbol -> CoinGecko ID mapping and asset-type classification live in the +# shared data-processing module so the download pipeline and this script stay in +# sync. Everything not in STABLECOINS/NATIVE_LST is VOLATILE (asset_type=2). +from quantammsim.utils.data_processing.coingecko_data import ( # noqa: E402 + COINGECKO_IDS, + STABLECOINS, + NATIVE_LST, +) def fetch_mcaps(): diff --git a/scripts/prepare_bold_usdc_data.py b/scripts/prepare_bold_usdc_data.py deleted file mode 100644 index bb547dc..0000000 --- a/scripts/prepare_bold_usdc_data.py +++ /dev/null @@ -1,154 +0,0 @@ -"""Prepare price data for BOLD/USDC reCLAMM simulations. - -1. Fetches BOLD/USD daily price + traded volume from CoinGecko (free tier = - last 365 days), expands to a minute grid by forward-fill, and writes - quantammsim/data/BOLD_USD.parquet. The real daily traded volume is spread - evenly across the day so that market_features' daily resample recovers the - true daily USD volume (used by the organic-volume noise model). - -2. Rebuilds quantammsim/data/USDC_USD.parquet pegged at $1.00 over the union - of its existing coverage and the BOLD grid, so both the BOLD/USDC window - and the earlier Monad window stay covered. - -BOLD (Liquity V2, CoinGecko id "liquity-bold-2") is not on Binance/Coinbase, -so CoinGecko is the only source. Free-tier history is capped at 365 days — -this bounds the earliest possible simulation start date. - -Usage (from repo root): - python scripts/prepare_bold_usdc_data.py -""" - -from __future__ import annotations - -import time -from pathlib import Path - -import numpy as np -import pandas as pd -import requests - -REPO_ROOT = Path(__file__).resolve().parent.parent -DATA_DIR = REPO_ROOT / "quantammsim" / "data" - -CG_ID = "liquity-bold-2" -_CG_URL = f"https://api.coingecko.com/api/v3/coins/{CG_ID}/market_chart" - - -def _fetch_coingecko_daily() -> tuple[list, list]: - """Return (prices, total_volumes) as [[unix_ms, value], ...] for last 365d.""" - params = {"vs_currency": "usd", "days": "365", "interval": "daily"} - for attempt in range(4): - try: - resp = requests.get(_CG_URL, params=params, timeout=40) - if resp.status_code == 429: - print(" rate-limited, sleeping 60s …") - time.sleep(60) - continue - resp.raise_for_status() - j = resp.json() - return j.get("prices", []), j.get("total_volumes", []) - except Exception as exc: - print(f" CoinGecko attempt {attempt+1} failed: {exc}") - time.sleep(10) - raise RuntimeError("CoinGecko fetch failed for BOLD") - - -def build_bold_parquet() -> pd.DataFrame: - prices, volumes = _fetch_coingecko_daily() - if len(prices) < 100: - raise RuntimeError(f"BOLD price series too short: {len(prices)}") - - # Daily frame: price (close) + daily USD volume, indexed by midnight-UTC day. - pdf = pd.DataFrame(prices, columns=["unix", "close"]) - vdf = pd.DataFrame(volumes, columns=["unix", "vol_usd"]) - pdf["day"] = pd.to_datetime(pdf["unix"], unit="ms", utc=True).dt.normalize() - vdf["day"] = pd.to_datetime(vdf["unix"], unit="ms", utc=True).dt.normalize() - daily = (pdf.groupby("day")["close"].last() - .to_frame() - .join(vdf.groupby("day")["vol_usd"].last())) - daily["vol_usd"] = daily["vol_usd"].fillna(0.0) - daily = daily.dropna(subset=["close"]).sort_index() - - start = daily.index.min() - end = daily.index.max() - print(f" BOLD daily: {start.date()} → {end.date()} ({len(daily)} days)") - print(f" price ${daily['close'].min():.4f}–${daily['close'].max():.4f}, " - f"last ${daily['close'].iloc[-1]:.4f}") - print(f" daily vol (USD): median ${daily['vol_usd'].median():,.0f}, " - f"mean ${daily['vol_usd'].mean():,.0f}") - - # Expand to a complete minute grid covering full days [start, end+1day). - epoch = pd.Timestamp("1970-01-01", tz="UTC") - grid_end = end + pd.Timedelta(days=1) - pd.Timedelta(minutes=1) - min_range = pd.date_range(start=start, end=grid_end, freq="1min") - min_idx = ((min_range - epoch).total_seconds() * 1000).astype(np.int64) - - out = pd.DataFrame(index=min_idx) - out.index.name = "unix" - out["date"] = min_range - out["day"] = min_range.normalize() - - # Map close + per-minute volume (daily vol spread across 1440 minutes so the - # daily resample-sum in market_features recovers the real daily volume). - close_map = daily["close"].to_dict() - vol_map = (daily["vol_usd"] / 1440.0).to_dict() - out["close"] = out["day"].map(close_map).astype(float) - out["Volume USD"] = out["day"].map(vol_map).astype(float) - out["close"] = out["close"].ffill().bfill() - out["Volume USD"] = out["Volume USD"].fillna(0.0) - out["open"] = out["high"] = out["low"] = out["close"] - out["Volume BOLD"] = out["Volume USD"] / out["close"].clip(lower=1e-9) - out["symbol"] = "BOLD" - out = out.drop(columns=["day"]) - out = out[["date", "symbol", "open", "high", "low", "close", - "Volume USD", "Volume BOLD"]] - - path = DATA_DIR / "BOLD_USD.parquet" - out.to_parquet(path, engine="pyarrow") - print(f" Saved {len(out):,} rows → {path}") - return out - - -def rebuild_usdc_parquet(bold_df: pd.DataFrame) -> None: - """USDC pegged at $1.00 over the union of its current grid and BOLD's.""" - path = DATA_DIR / "USDC_USD.parquet" - start_ms = int(bold_df.index.min()) - end_ms = int(bold_df.index.max()) - if path.exists(): - existing = pd.read_parquet(path) - start_ms = min(start_ms, int(existing.index.min())) - end_ms = max(end_ms, int(existing.index.max())) - - idx = np.arange(start_ms, end_ms + 60_000, 60_000, dtype=np.int64) - n = len(idx) - out = pd.DataFrame({ - "date": pd.to_datetime(idx, unit="ms", utc=True), - "symbol": "USDC", - "open": np.ones(n), "high": np.ones(n), - "low": np.ones(n), "close": np.ones(n), - "Volume USD": np.zeros(n), "Volume USDC": np.zeros(n), - }, index=idx) - out.index.name = "unix" - out.to_parquet(path, engine="pyarrow") - print(f" Saved {n:,} rows → {path} (peg $1.00, " - f"{out['date'].iloc[0].date()} → {out['date'].iloc[-1].date()})") - - -def main() -> None: - DATA_DIR.mkdir(parents=True, exist_ok=True) - print("Fetching BOLD/USD from CoinGecko …") - bold = build_bold_parquet() - print("Rebuilding USDC peg parquet …") - rebuild_usdc_parquet(bold) - - print("\n--- Data summary ---") - for token in ["BOLD", "USDC"]: - df = pd.read_parquet(DATA_DIR / f"{token}_USD.parquet") - dates = pd.to_datetime(df.index, unit="ms", utc=True) - print(f" {token:5s}: {dates.min().date()} → {dates.max().date()} " - f"({len(df):,} rows) close=[{df['close'].min():.4f}, " - f"{df['close'].max():.4f}]") - - -if __name__ == "__main__": - main()