diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..53958f1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,20 @@ +name: research-artifact-ci + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + - run: python -m pip install --upgrade pip + - run: python -m pip install -r requirements.txt -r requirements-dev.txt + - run: python -m ruff check src tests + - run: python -m pytest + - run: python -m src.experiments --seeds 2 --jobs 40 --hours 72 --results-dir scratch/results --figures-dir scratch/figures diff --git a/.gitignore b/.gitignore index e086637..72cfbc4 100644 --- a/.gitignore +++ b/.gitignore @@ -14,17 +14,12 @@ venv/ .ipynb_checkpoints/ # Large local data files -data/*.csv -data/*.xlsx -data/*.parquet -data/*.feather -data/*.h5 -data/*.zip +data/raw/ data/external/ data/processed/ -# Local outputs -models/ +# Local scratch outputs. Curated results in results/ are versioned. +scratch/ # OS files .DS_Store diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..4a7d1b2 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,15 @@ +cff-version: 1.2.0 +message: "If you use this research artifact, cite the software and the specific external datasets and papers used in your study." +title: "Carbon and Water Aware Compute Scheduling Research Artifact" +type: software +authors: + - name: "Personal Project contributors" +version: 0.2.0 +date-released: 2026-08-30 +repository-code: "https://github.com/drona23/Personal_Project" +keywords: + - carbon-aware computing + - water-aware scheduling + - data centers + - reproducible research + - sustainable computing diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..51ff94c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,24 @@ +# Contributing + +Changes should preserve the research boundary and reproducibility guarantees. + +Before submitting a change: + +```bash +python -m ruff check src tests +python -m pytest +python -m src.experiments +``` + +For a method or metric change, include: + +- a test that fails under the previous behavior; +- an update to `docs/METHODOLOGY.md`; +- regenerated seed-level and aggregate outputs; +- an update to `docs/RESULTS.md` if interpretation changes; +- a citation for any borrowed model, factor, or dataset; +- a clear statement of new limitations. + +Do not add a dashboard or service layer to this branch. Keep application code in +a separate repository or branch so the research artifact remains small and +auditable. diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index a69e52a..0000000 --- a/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -# ─── FastAPI Backend Dockerfile ─────────────────────────────────────────────── -# Base image: Python 3.11 slim (matches requirements.txt pin comment) -FROM python:3.11-slim - -# Set working directory inside the container -WORKDIR /app - -# Install Python dependencies first (layer-cached unless requirements.txt changes) -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - -# Copy application source, models, and data -COPY src/ ./src/ -COPY models/ ./models/ -COPY data/ ./data/ - -# Expose the port uvicorn will listen on -EXPOSE 8000 - -# Run the FastAPI app as a package (src.api) so relative imports work -CMD ["uvicorn", "src.api:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md index 2fc0dbe..01d8386 100644 --- a/README.md +++ b/README.md @@ -1,320 +1,149 @@ -# Making AI Less Thirsty +# Carbon and Water Aware Compute Scheduling -> Predicting how much carbon and water a data center will use — then automatically sending AI jobs to the cleanest one available. +This repository is a research-only artifact for studying when and where to run +deadline-flexible compute jobs. It contains no web interface, API service, or +deployment stack. The focus is a reproducible scheduling experiment with clear +assumptions, paired baselines, uncertainty tests, and traceable outputs. ---- +## Research question -## The Problem in Plain English +Can a scheduler reduce operational carbon emissions and onsite water use for +non-preemptive compute jobs by shifting them across time and location, while +preserving job deadlines and data center capacity constraints? -When you ask an AI model a question, a computer somewhere runs the calculation. That computer uses electricity and water to stay cool. The problem: +The problem matters because U.S. data centers consumed an estimated 176 TWh in +2023, and the demand range projected for 2028 is substantially higher +[[R1](docs/REFERENCES.md#r1)]. Grid carbon intensity and cooling water use also +vary by time and location [[R2](docs/REFERENCES.md#r2), +[R4](docs/REFERENCES.md#r4)]. -- **Different power sources produce different carbon** — a data center running on solar emits far less CO₂ than one running on coal -- **Carbon output changes by the hour** — at night in Texas, wind covers most of the grid; midday on a cloudy day, coal picks up the slack -- **Nobody is optimising for this** — jobs get sent to the nearest or cheapest server, not the cleanest one +## Intended beneficiaries -This project builds a system that watches those changes in real time and routes computing jobs toward whichever data center is cleanest *right now*. - ---- - -## What It Actually Does - -``` -Step 1 — Collect data - Download hourly electricity generation (EIA) and weather (Open-Meteo) - for 28 U.S. data center cities. Free government data, no API keys needed. - -Step 2 — Predict the future - Train machine learning models that forecast CO₂ intensity and water - usage for each city 48 hours ahead. Uses XGBoost + Prophet + hybrid. - -Step 3 — Schedule jobs - Given a batch of AI workloads and the 48-hour forecast, find the - routing plan that minimises carbon + water while respecting deadlines. - -Step 4 — Show the reasoning - For any routing decision, explain which features (temperature, time of - day, fuel mix) pushed the prediction up or down — using SHAP values. - -Step 5 — Visualise it - Interactive U.S. map showing routing flows, environmental trade-offs, - and a baseline comparison so you can see the improvement. -``` - ---- - -## Results (from real model output) - -### CO₂ varies 3.2× across cities - -| City | Avg CO₂ Intensity | -|---|---| -| Albany, NY (cleanest) | 0.206 kg CO₂/kWh | -| Dallas, TX | 0.320 kg CO₂/kWh | -| Dona Ana County, NM (dirtiest) | 0.667 kg CO₂/kWh | - -**What this means:** Sending the same job to Albany instead of Dona Ana County produces one-third the carbon. The scheduler exploits this gap automatically. - -### Water usage also varies by city - -| City | WUE (litres per kWh) | -|---|---| -| Northern Indiana (most efficient) | 1.117 | -| Shackelford County, TX (least efficient) | 1.186 | - -### Scheduler performance (50-job test) - -| Jobs submitted | Jobs scheduled | Coverage | +| Organization type | Representative examples | Decision supported | |---|---|---| -| 50 | 50 | 100% | - ---- - -## How the System Is Built - -``` -┌─────────────────────────────────────────────────────────────────────┐ -│ DATA LAYER │ -│ EIA (electricity grid) + Open-Meteo (weather) + EPA (emissions) │ -│ → build_dataset.py → one clean CSV with everything merged │ -│ │ -│ Azure VM Trace (2.2M real cloud jobs) │ -│ → workload_loader.py → realistic job queue for testing │ -└─────────────────────────────────────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────────────────────────────────────┐ -│ ML LAYER │ -│ For each city, for each hour, predict: │ -│ • CO₂ intensity (how dirty is the electricity right now?) │ -│ • WUE (how much water does cooling use?) │ -│ │ -│ Three models trained per target: │ -│ Prophet — captures time patterns (rush hour, seasonal) │ -│ XGBoost — learns from weather + fuel mix + time features │ -│ Hybrid — Prophet + XGBoost residual (best of both) │ -│ │ -│ tune.py — Optuna auto-searches for best XGBoost settings │ -│ explain.py — SHAP shows WHY a prediction is high or low │ -└─────────────────────────────────────────────────────────────────────┘ - ↓ -┌─────────────────────────────────────────────────────────────────────┐ -│ SCHEDULER + API + DASHBOARD │ -│ scheduling.py — scores every (job, city, time) combination: │ -│ score = α × carbon + β × water + γ × distance │ -│ α, β, γ are sliders you control — trade off green vs fast │ -│ │ -│ FastAPI backend — serves predictions and routing decisions │ -│ React frontend — interactive U.S. map, batch mode, SHAP chart │ -└─────────────────────────────────────────────────────────────────────┘ -``` - ---- - -## Weather Pipeline Integration - -This project connects to a **live weather data pipeline** built in the companion `weather_pipeline` project (Apache Airflow + PostgreSQL). - -When the weather pipeline is running, Capstone can pull **real-time hourly weather** for all 28 cities instead of using historical CSV data: - -```python -from src.weather_loader import load_from_pipeline_db - -# Returns live weather from the last 48 hours -df = load_from_pipeline_db(hours=48) -``` - -The pipeline runs as a Docker service on `localhost:15432`. If it's not running, the system falls back to the historical Meteostat dataset automatically. - ---- - -## Quick Start - -### Option A — Docker (recommended, no setup required) +| Cloud and hyperscale operators | AWS, Google Cloud, Microsoft Azure | Placement of deferrable batch, analytics, training, and maintenance jobs | +| AI infrastructure teams | Model training and batch inference platforms | Carbon and water tradeoff analysis before a run is admitted | +| Research computing institutions | Universities, national laboratories, HPC centers | Queue scheduling under deadlines and site power limits | +| Colocation and enterprise operators | Multi-site data center portfolios | Scenario analysis for regional load shifting | +| Utilities and sustainability researchers | Grid operators, energy institutes, public agencies | Study of flexible computing demand and forecast sensitivity | -```bash -git clone -cd Capstone_Research -docker compose up --build -``` - -Open in your browser: -- **`http://localhost:3000`** — the interactive routing dashboard -- **`http://localhost:8000/docs`** — the API with a built-in test interface - -### Option B — Run locally - -```bash -# Install Python dependencies -python3 -m venv .venv && source .venv/bin/activate -pip install -r requirements.txt - -# Add the dataset (required) -# Place base_data_with_metrics.parquet or sample_dataset.xlsx in data/ - -# Train the ML models -python3 -m src.main --data-path data/sample_dataset.xlsx - -# Start the API -python3 -m uvicorn src.api:app --reload +These are representative beneficiaries. The repository does not claim a +deployment, partnership, or validation by any named organization. -# In a second terminal, start the frontend -cd frontend && npm install && npm run dev -# Open http://localhost:5173 -``` +## What this artifact contributes ---- +Prior work already established carbon-aware scheduling, water-aware scheduling, +and joint carbon-water optimization. This project does not claim to invent those +ideas. Its contribution is a compact evaluation scaffold that combines: -## Tuning the Models (Optional) +- deadline and hourly capacity feasibility; +- physical water use reported separately from scarcity-weighted water impact; +- per-job normalization before combining quantities with different units; +- paired-job comparisons, so a policy cannot appear cleaner only by completing + fewer jobs; +- full execution-window carbon accounting with data center PUE; +- forecast-error stress tests against a perfect-forecast reference policy; +- deterministic synthetic inputs that run offline and in continuous integration. -The default XGBoost settings work well. To automatically find better ones: +See [the related-work comparison](docs/RELATED_WORK.md) for a claim-by-claim +positioning against published research. -```bash -# Search 40 parameter combinations per model (~5 minutes) -python3 -m src.tune +## Reproduced results -# Re-train using the best parameters found above -python3 -m src.main -``` +The committed results use 10 seeds, 180 jobs per seed, four synthetic zones, and +one week of hourly signals per seed. All policies completed every submitted job +in this benchmark. The values below are paired comparisons over exactly the same +completed jobs. -Results are saved to `models/best_params.json`. Training picks them up automatically next time. +| Comparison | Operational CO2e | Physical water | Scarcity-weighted water | +|---|---:|---:|---:| +| Proposed vs immediate local | 23.94% lower, CI 1.92 | 28.60% lower, CI 1.89 | 20.08% lower, CI 1.28 | +| Proposed vs random feasible | 22.31% lower, CI 2.18 | 29.50% lower, CI 2.81 | 20.78% lower, CI 1.87 | +| Proposed vs carbon greedy | 46.88% higher, CI 2.11 | 52.75% lower, CI 1.47 | 39.17% lower, CI 1.17 | ---- +The carbon-greedy comparison is important. A joint policy does not dominate a +single-objective carbon policy. It accepts more carbon in this synthetic case to +reduce water impact. That is a tradeoff, not a universal improvement. -## Running Tests +![Benchmark results](docs/figures/benchmark_overview.png) -```bash -python3 -m pytest tests/ -v -# Expected: 32 passed, 1 skipped -``` +At a forecast noise standard deviation equal to 20% of each city's mean carbon +intensity, the carbon-only policy used 3.65% more operational CO2e than the +perfect-forecast reference, with an approximate 95% confidence interval of 0.54 +percentage points. At 50% noise, the delta was 21.42%, with an interval of 1.71 +percentage points. -Tests cover three areas: -- **Unit tests** — the scheduler's scoring formula and job builder -- **API tests** — every endpoint with valid and invalid inputs -- **SHAP tests** — the explainer's output shape and sorting (skipped if shap not installed) - ---- - -## API Endpoints - -| Endpoint | What it does | -|---|---| -| `GET /health` | Check the server is running | -| `GET /context` | Get available cities, time range, and default settings | -| `POST /simulate` | Get a routing recommendation for one job | -| `POST /simulate-batch` | Schedule 50–200 jobs across data centers | -| `GET /explain` | Explain why a city got a particular CO₂ or WUE prediction | +![Forecast robustness](docs/figures/forecast_robustness.png) -### Example: get a routing recommendation +These results are simulation evidence only. They are not estimates of savings +for a real company or facility. Full interpretation is in +[docs/RESULTS.md](docs/RESULTS.md). -```bash -curl -X POST http://localhost:8000/simulate \ - -H "Content-Type: application/json" \ - -d '{ - "priority": "medium", - "time": "2024-06-01T14:00", - "alpha": 1.0, - "beta": 1.0, - "gamma": 1.0 - }' -``` +## Reproduce the artifact -### Example: explain a prediction +Python 3.11 or later is required. ```bash -curl "http://localhost:8000/explain?city=Dallas&target=co2&time=2024-06-01T14:00" -``` - -Returns which features (temperature, time of day, fuel mix) pushed the CO₂ prediction up or down, and by how much. - ---- - -## The Scheduling Formula - -The scheduler picks the city that minimises this score for each job: - -``` -score = α × (CO₂ intensity × server efficiency) - + β × (water usage × local water scarcity) - + γ × (distance penalty — 0 if same city, 1 if different) +python -m venv .venv +source .venv/bin/activate +python -m pip install -r requirements.txt -r requirements-dev.txt +python -m pytest +python -m src.experiments ``` -You control α, β, and γ through sliders in the dashboard: -- **α = 2, β = 0, γ = 0** → pure carbon optimisation, ignore water and distance -- **α = 0, β = 0, γ = 2** → always send to nearest data center -- **α = 1, β = 1, γ = 1** → balanced trade-off (default) - ---- - -## Project Files - -``` -Capstone_Research/ -│ -├── src/ Python source code -│ ├── fetch_eia.py Downloads electricity grid data from EIA -│ ├── fetch_weather.py Downloads weather data from Open-Meteo -│ ├── build_dataset.py Merges EIA + weather into one dataset -│ ├── preprocessing.py Cleans data and builds ML features -│ ├── train.py Trains XGBoost + Prophet + Hybrid models -│ ├── tune.py Finds best XGBoost settings using Optuna -│ ├── explain.py Explains predictions using SHAP -│ ├── scheduling.py Routes jobs to optimal data centers -│ ├── api.py FastAPI web server -│ ├── app_backend.py Backend logic used by the API -│ ├── weather_loader.py Loads weather data (CSV or live pipeline) -│ └── workload_loader.py Converts Azure VM trace into job records -│ -├── frontend/ React web dashboard -│ ├── src/components/ Map, control panel, insight panel -│ ├── nginx.conf Routes API calls to backend in Docker -│ └── Dockerfile Builds frontend for Docker -│ -├── tests/ Automated tests -│ ├── test_scheduling.py Tests the scoring formula -│ ├── test_api.py Tests every API endpoint -│ └── test_explain.py Tests the SHAP explainer output -│ -├── models/ Saved ML models (not in git — too large) -│ ├── co2_model.pkl Carbon intensity model bundle -│ ├── wue_model.pkl Water usage model bundle -│ └── best_params.json Optuna tuning results (created by tune.py) -│ -├── data/ Datasets (not in git — too large) -│ ├── processed/ Model forecasts and schedule outputs -│ ├── reference/ City coordinates -│ └── templates/ Data center configuration template -│ -├── Dockerfile Builds the Python backend for Docker -├── docker-compose.yml Starts backend + frontend together -├── requirements.txt Python package versions -└── pytest.ini Test configuration -``` - ---- - -## Data Sources (all free) - -| Source | What it provides | -|---|---| -| [EIA Open Data](https://www.eia.gov/opendata/) | Hourly electricity generation mix for 7 US grid regions | -| [Open-Meteo](https://open-meteo.com/) | Historical hourly weather — no API key needed | -| [EPA eGRID](https://www.epa.gov/egrid) | How much CO₂ each fuel type produces | -| [Azure Public Dataset](https://github.com/Azure/AzurePublicDataset) | 2.2 million real Microsoft VM job records | +The final command regenerates: ---- +- `results/benchmark_summary.csv` +- `results/paired_comparisons_by_seed.csv` +- `results/policy_summary_by_seed.csv` +- `results/forecast_robustness.csv` +- `results/example_schedule.csv` +- `results/run_metadata.json` +- `results/SHA256SUMS` +- the three PNG figures in `docs/figures/` -## Tech Stack +## Repository map -| What | How | +| Path | Purpose | |---|---| -| ML models | XGBoost, Prophet, scikit-learn | -| Hyperparameter search | Optuna (Bayesian, not random) | -| Model explanations | SHAP (TreeExplainer) | -| API | FastAPI + Uvicorn | -| Dashboard | React 19, TypeScript, MapLibre GL | -| Containers | Docker Compose | -| Tests | pytest | - ---- - -*For data source details and gaps, see [docs/data_requirements.md](docs/data_requirements.md)* -*For the Azure trace setup, see [docs/workload_trace_guide.md](docs/workload_trace_guide.md)* +| `src/scheduling.py` | Feasibility checks, candidate accounting, normalized objective, paired metrics | +| `src/baselines.py` | Immediate-local, carbon-greedy, and seeded-random feasible baselines | +| `src/synthetic_benchmark.py` | Deterministic offline environment, jobs, and data center configuration | +| `src/forecast_robustness.py` | Full-window realized carbon evaluation under forecast error | +| `src/experiments.py` | Multi-seed runner, CSV export, metadata, and figures | +| `src/fetch_eia.py` | Optional U.S. EIA generation-mix adapter | +| `src/fetch_entso_e.py` | Optional ENTSO-E generation-mix adapter with sub-hourly aggregation | +| `src/workload_loader.py` | Optional converter for the public Azure VM packing trace | +| `tests/` | Unit and offline integration tests | +| `docs/` | Problem statement, methodology, related work, provenance, limitations, and results | + +## Read next + +- [Research problem and use cases](docs/RESEARCH_PROBLEM.md) +- [Methodology and equations](docs/METHODOLOGY.md) +- [Related work and project difference](docs/RELATED_WORK.md) +- [Data provenance and canonical schemas](docs/DATA_PROVENANCE.md) +- [Results and output guide](docs/RESULTS.md) +- [Threats to validity](docs/THREATS_TO_VALIDITY.md) +- [Reproducibility checklist](docs/REPRODUCIBILITY.md) +- [Verified references](docs/REFERENCES.md) + +## Research boundaries + +This artifact models operational electricity emissions and onsite cooling water. +It does not model embodied hardware emissions, electricity-generation water, +network transfer energy, data sovereignty, hardware compatibility, electricity +price, marginal emissions, job preemption, or production failure modes. Those +boundaries are deliberate and are documented as future research directions. + +## Citation and reuse + +Use `CITATION.cff` for the software citation and `references.bib` for the +literature and data sources. Replace the placeholder contributor entry in +`CITATION.cff` with the author's preferred name and ORCID before using the +repository in an application or publication. + +No software license has been selected. Copyright therefore remains with the +repository owner, and reuse permission is not granted by default. Choosing a +license is an owner decision that should be completed before public reuse or +external contribution. diff --git a/data/reference/city_coordinates.csv b/data/reference/city_coordinates.csv deleted file mode 100644 index 14c331d..0000000 --- a/data/reference/city_coordinates.csv +++ /dev/null @@ -1,29 +0,0 @@ -city,state,latitude,longitude,capacity_mw,category -Abilene,TX,32.4487,-99.7331,1200,stargate -Shackelford County,TX,32.72,-99.35,1400,stargate -Dona Ana County,NM,32.35,-106.75,1000,stargate -Lordstown,OH,41.17,-80.87,750,stargate -Milam County,TX,30.75,-96.95,750,stargate -Port Washington,WI,43.39,-87.88,902,stargate -Mount Pleasant,WI,42.72,-87.88,500,hyperscaler -Atlanta,GA,33.749,-84.388,500,hyperscaler -Wilbarger County,TX,34.08,-99.22,600,hyperscaler -Armstrong County,TX,34.96,-101.35,400,hyperscaler -Haskell County,TX,33.16,-99.73,400,hyperscaler -Pine Island,MN,44.2,-92.65,300,hyperscaler -Richland Parish,LA,32.41,-91.72,2000,hyperscaler -Beaver Dam,WI,43.46,-88.84,200,hyperscaler -El Paso,TX,31.76,-106.45,200,hyperscaler -Bowling Green,OH,41.375,-83.65,150,hyperscaler -Carlisle,IN,40.78,-86.17,2000,hyperscaler -Northern Indiana,IN,41.6,-86.25,2400,hyperscaler -Memphis,TN,35.15,-90.05,2000,neocloud -Lancaster,PA,40.04,-76.31,300,neocloud -Kenilworth,NJ,40.68,-74.29,250,neocloud -West Texas,TX,31.9,-102.1,2000,neocloud -Dallas,TX,32.78,-96.8,500,neocloud -Albany,NY,42.65,-73.75,500,neocloud -Little Rock,AR,34.75,-92.29,1000,independent -Meridian,MS,32.36,-88.7,320,independent -Rocky Mount,NC,35.94,-77.79,900,independent -Powhatan County,VA,37.55,-77.92,365,independent diff --git a/data/reference/eu_city_coordinates.csv b/data/reference/eu_city_coordinates.csv deleted file mode 100644 index 5e6c527..0000000 --- a/data/reference/eu_city_coordinates.csv +++ /dev/null @@ -1,11 +0,0 @@ -city,country,latitude,longitude,capacity_mw,category,entso_zone -Dublin,IE,53.3498,-6.2603,600,hyperscaler,IE -Amsterdam,NL,52.3676,4.9041,800,hyperscaler,NL -Frankfurt,DE,50.1109,8.6821,900,hyperscaler,DE-LU -London,GB,51.5074,-0.1278,700,hyperscaler,GB -Paris,FR,48.8566,2.3522,750,hyperscaler,FR -Stockholm,SE,59.3293,18.0686,500,hyperscaler,SE3 -Helsinki,FI,60.1699,24.9384,400,hyperscaler,FI -Madrid,ES,40.4168,-3.7038,550,hyperscaler,ES -Warsaw,PL,52.2297,21.0122,450,hyperscaler,PL -Copenhagen,DK,55.6761,12.5683,400,hyperscaler,DK2 diff --git a/data/results/cross_region_comparison.csv b/data/results/cross_region_comparison.csv deleted file mode 100644 index 7e78c05..0000000 --- a/data/results/cross_region_comparison.csv +++ /dev/null @@ -1,3 +0,0 @@ -region,n_cities,co2_min_kg_kwh,co2_max_kg_kwh,co2_spread_x,cleanest_city,dirtiest_city,jobs_total,jobs_scheduled,coverage_pct,naive_co2_kg,optimal_co2_kg,co2_reduction_pct,co2_saved_kg,co2_saved_per_job_kg,naive_water_l,optimal_water_l,water_reduction_pct,frac_to_top3_clean,top3_clean_cities -US,28,0.2284,0.5954,2.6,Albany,Dona Ana County,100,100,100.0,10709.45,5051.1,52.8,5658.36,56.58,15657.7,13235.2,15.5,53.0,"Albany, Abilene, Armstrong County" -EU,10,0.0146,0.5931,40.6,Stockholm,Warsaw,100,100,100.0,4388.73,767.64,82.5,3621.09,36.21,7695.3,3301.5,57.1,92.0,"Stockholm, Paris, Helsinki" diff --git a/data/results/forecast_robustness_eu.csv b/data/results/forecast_robustness_eu.csv deleted file mode 100644 index 25d91f9..0000000 --- a/data/results/forecast_robustness_eu.csv +++ /dev/null @@ -1,7 +0,0 @@ -noise_level,noise_pct,forecast_rmse,oracle_co2_kg,actual_co2_kg,regret_kg,regret_pct,jobs_scheduled,coverage -0.0,0%,0.0,222.9,222.9,0.0,0.0,50,1.0 -0.05,5%,0.014553,222.9,229.93,7.03,3.15,50,1.0 -0.1,10%,0.02908,222.9,229.61,6.71,3.01,50,1.0 -0.2,20%,0.058822,222.9,233.5,10.6,4.75,50,1.0 -0.35,35%,0.102484,222.9,302.56,79.66,35.74,50,1.0 -0.5,50%,0.141497,222.9,392.03,169.13,75.88,50,1.0 diff --git a/data/results/forecast_robustness_us.csv b/data/results/forecast_robustness_us.csv deleted file mode 100644 index e0fa6a5..0000000 --- a/data/results/forecast_robustness_us.csv +++ /dev/null @@ -1,7 +0,0 @@ -noise_level,noise_pct,forecast_rmse,oracle_co2_kg,actual_co2_kg,regret_kg,regret_pct,jobs_scheduled,coverage -0.0,0%,0.0,26800.0,26800.0,0.0,0.0,24,0.24 -0.05,5%,0.017953,26800.0,27126.21,326.2,1.22,24,0.24 -0.1,10%,0.035773,26800.0,27228.46,428.46,1.6,24,0.24 -0.2,20%,0.071591,26800.0,27431.53,631.53,2.36,24,0.24 -0.35,35%,0.124972,26800.0,26650.66,-149.34,-0.56,24,0.24 -0.5,50%,0.174874,26800.0,26572.8,-227.2,-0.85,24,0.24 diff --git a/data/results/sensitivity_analysis_eu.csv b/data/results/sensitivity_analysis_eu.csv deleted file mode 100644 index 175ccf2..0000000 --- a/data/results/sensitivity_analysis_eu.csv +++ /dev/null @@ -1,16 +0,0 @@ -alpha,beta,gamma,weight_label,total_co2_kg,total_water_l,jobs_scheduled,coverage,pareto_optimal -0.0,0.0,1.0,α=0.0 β=0.0 γ=1.0,3034.45,4271.98,50,1.0,False -0.0,0.25,0.75,α=0.0 β=0.2 γ=0.8,3329.55,4271.98,50,1.0,False -0.0,0.5,0.5,α=0.0 β=0.5 γ=0.5,3329.55,4271.98,50,1.0,False -0.0,0.75,0.25,α=0.0 β=0.8 γ=0.2,2170.34,2133.26,50,1.0,False -0.0,1.0,0.0,α=0.0 β=1.0 γ=0.0,1465.85,758.78,50,1.0,True -0.25,0.0,0.75,α=0.2 β=0.0 γ=0.8,2776.23,4271.98,50,1.0,False -0.25,0.25,0.5,α=0.2 β=0.2 γ=0.5,2776.23,4271.98,50,1.0,False -0.25,0.5,0.25,α=0.2 β=0.5 γ=0.2,1424.33,2210.59,50,1.0,False -0.25,0.75,0.0,α=0.2 β=0.8 γ=0.0,190.79,1021.44,50,1.0,True -0.5,0.0,0.5,α=0.5 β=0.0 γ=0.5,2776.23,4271.98,50,1.0,False -0.5,0.25,0.25,α=0.5 β=0.2 γ=0.2,1424.33,2210.59,50,1.0,False -0.5,0.5,0.0,α=0.5 β=0.5 γ=0.0,190.79,1021.44,50,1.0,True -0.75,0.0,0.25,α=0.8 β=0.0 γ=0.2,1730.58,3687.94,50,1.0,False -0.75,0.25,0.0,α=0.8 β=0.2 γ=0.0,190.79,1021.44,50,1.0,True -1.0,0.0,0.0,α=1.0 β=0.0 γ=0.0,190.79,1021.44,50,1.0,True diff --git a/data/results/sensitivity_analysis_us.csv b/data/results/sensitivity_analysis_us.csv deleted file mode 100644 index 81c2701..0000000 --- a/data/results/sensitivity_analysis_us.csv +++ /dev/null @@ -1,16 +0,0 @@ -alpha,beta,gamma,weight_label,total_co2_kg,total_water_l,jobs_scheduled,coverage,pareto_optimal -0.0,0.0,1.0,α=0.0 β=0.0 γ=1.0,4852.73,8290.79,50,1.0,False -0.0,0.25,0.75,α=0.0 β=0.2 γ=0.8,5112.51,7810.17,50,1.0,False -0.0,0.5,0.5,α=0.0 β=0.5 γ=0.5,5112.51,7810.17,50,1.0,False -0.0,0.75,0.25,α=0.0 β=0.8 γ=0.2,5453.25,5142.45,50,1.0,False -0.0,1.0,0.0,α=0.0 β=1.0 γ=0.0,6057.94,4408.46,50,1.0,True -0.25,0.0,0.75,α=0.2 β=0.0 γ=0.8,3131.91,8760.59,50,1.0,False -0.25,0.25,0.5,α=0.2 β=0.2 γ=0.5,3382.19,7992.65,50,1.0,False -0.25,0.5,0.25,α=0.2 β=0.5 γ=0.2,3616.63,6963.19,50,1.0,False -0.25,0.75,0.0,α=0.2 β=0.8 γ=0.0,4268.85,4510.51,50,1.0,True -0.5,0.0,0.5,α=0.5 β=0.0 γ=0.5,3131.91,8760.59,50,1.0,False -0.5,0.25,0.25,α=0.5 β=0.2 γ=0.2,3244.38,8074.03,50,1.0,False -0.5,0.5,0.0,α=0.5 β=0.5 γ=0.0,2341.01,5474.65,50,1.0,True -0.75,0.0,0.25,α=0.8 β=0.0 γ=0.2,3101.05,8752.37,50,1.0,False -0.75,0.25,0.0,α=0.8 β=0.2 γ=0.0,2263.72,5599.62,50,1.0,True -1.0,0.0,0.0,α=1.0 β=0.0 γ=0.0,1772.4,12614.61,50,1.0,True diff --git a/data/sample_dataset.xlsx b/data/sample_dataset.xlsx deleted file mode 100644 index e69de29..0000000 diff --git a/data/templates/dc_config_template.csv b/data/templates/dc_config_template.csv deleted file mode 100644 index 9491c04..0000000 --- a/data/templates/dc_config_template.csv +++ /dev/null @@ -1,29 +0,0 @@ -dc_id,city,max_power_per_hour,pue,water_usage_factor,region_water_scarcity,is_new,category -dc_abilene_1,Abilene,1200,1.15,1.10,0.78,1,stargate -dc_shackelford_1,Shackelford County,1400,1.14,1.08,0.80,1,stargate -dc_donaana_1,Dona Ana County,1000,1.16,1.12,0.85,1,stargate -dc_lordstown_1,Lordstown,750,1.13,0.95,0.45,1,stargate -dc_milam_1,Milam County,750,1.15,1.06,0.75,1,stargate -dc_portwashington_1,Port Washington,902,1.12,0.92,0.42,1,stargate -dc_mountpleasant_1,Mount Pleasant,500,1.13,0.93,0.43,1,hyperscaler -dc_atlanta_1,Atlanta,500,1.18,1.00,0.61,1,hyperscaler -dc_wilbarger_1,Wilbarger County,600,1.16,1.09,0.79,1,hyperscaler -dc_armstrong_1,Armstrong County,400,1.17,1.11,0.82,1,hyperscaler -dc_haskell_1,Haskell County,400,1.16,1.09,0.80,1,hyperscaler -dc_pineisland_1,Pine Island,300,1.12,0.90,0.38,1,hyperscaler -dc_richland_1,Richland Parish,2000,1.20,1.05,0.65,1,hyperscaler -dc_beaverdam_1,Beaver Dam,200,1.13,0.91,0.41,1,hyperscaler -dc_elpaso_1,El Paso,200,1.19,1.14,0.88,1,hyperscaler -dc_bowlinggreen_1,Bowling Green,150,1.14,0.94,0.47,1,hyperscaler -dc_carlisle_1,Carlisle,2000,1.13,0.95,0.46,1,hyperscaler -dc_northernindiana_1,Northern Indiana,2400,1.12,0.94,0.44,1,hyperscaler -dc_memphis_1,Memphis,2000,1.18,1.03,0.62,1,neocloud -dc_lancaster_1,Lancaster,300,1.14,0.93,0.50,1,neocloud -dc_kenilworth_1,Kenilworth,250,1.15,0.96,0.52,1,neocloud -dc_westtexas_1,West Texas,2000,1.17,1.10,0.83,1,neocloud -dc_dallas_1,Dallas,500,1.22,1.08,0.74,0,neocloud -dc_albany_1,Albany,500,1.13,0.91,0.46,1,neocloud -dc_littlerock_1,Little Rock,1000,1.17,1.02,0.63,1,independent -dc_meridian_1,Meridian,320,1.18,1.04,0.66,1,independent -dc_rockymount_1,Rocky Mount,900,1.15,0.97,0.55,1,independent -dc_powhatan_1,Powhatan County,365,1.14,0.96,0.53,1,independent diff --git a/data/templates/eu_dc_config.csv b/data/templates/eu_dc_config.csv deleted file mode 100644 index e55f079..0000000 --- a/data/templates/eu_dc_config.csv +++ /dev/null @@ -1,11 +0,0 @@ -dc_id,city,max_power_per_hour,pue,water_usage_factor,region_water_scarcity,is_new,category -dc_dublin_1,Dublin,600,1.20,0.85,0.20,0,hyperscaler -dc_amsterdam_1,Amsterdam,800,1.15,0.90,0.30,0,hyperscaler -dc_frankfurt_1,Frankfurt,900,1.25,1.00,0.25,0,hyperscaler -dc_london_1,London,700,1.22,0.88,0.22,0,hyperscaler -dc_paris_1,Paris,750,1.18,0.92,0.28,0,hyperscaler -dc_stockholm_1,Stockholm,500,1.12,0.70,0.10,0,hyperscaler -dc_helsinki_1,Helsinki,400,1.10,0.65,0.08,0,hyperscaler -dc_madrid_1,Madrid,550,1.30,1.05,0.65,0,hyperscaler -dc_warsaw_1,Warsaw,450,1.28,0.95,0.40,0,hyperscaler -dc_copenhagen_1,Copenhagen,400,1.15,0.80,0.18,0,hyperscaler diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index c6e4646..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,23 +0,0 @@ -services: - - # ── Backend: FastAPI ───────────────────────────────────────────────────────── - backend: - build: - context: . # use root Dockerfile - dockerfile: Dockerfile - container_name: capstone_backend - ports: - - "8000:8000" # localhost:8000 → container:8000 - restart: unless-stopped - - # ── Frontend: React + nginx ────────────────────────────────────────────────── - frontend: - build: - context: ./frontend # use frontend/Dockerfile - dockerfile: Dockerfile - container_name: capstone_frontend - ports: - - "3000:80" # localhost:3000 → nginx:80 - depends_on: - - backend - restart: unless-stopped diff --git a/docs/BASELINE_COMPARISON.md b/docs/BASELINE_COMPARISON.md deleted file mode 100644 index 4e8edf9..0000000 --- a/docs/BASELINE_COMPARISON.md +++ /dev/null @@ -1,165 +0,0 @@ -# Baseline Comparison — Research Validation - -This document explains the baseline comparison feature that validates the optimized scheduler against industry-standard strategies. - -## Motivation - -To claim a method is "better," research papers must compare it against baselines. This project compares the optimal (carbon-aware) scheduler against three baselines: - -1. **Random Routing** — picks any data center at random -2. **Nearest Neighbor** — sends jobs to the nearest data center (current industry practice) -3. **Time-of-Day Heuristic** — sends to the city with lowest CO₂ at that hour - -## The Endpoint - -**POST /compare-baselines** - -Schedule the same batch of jobs using all four strategies and compare results. - -### Request - -```json -{ - "time": "2024-06-01T14:00", - "batch_size": 50, - "priority": "medium" -} -``` - -### Response - -```json -{ - "timestamp": "2024-06-01T14:00:00", - "batch_size": 50, - "results": [ - { - "strategy": "Optimal (Your Method)", - "jobs_scheduled": 50, - "total_co2_kg": 245.3, - "total_water_liters": 1203.5, - "avg_co2_per_job": 4.91, - "avg_water_per_job": 24.07 - }, - { - "strategy": "Random Routing", - "jobs_scheduled": 50, - "total_co2_kg": 318.7, - "total_water_liters": 1456.2, - "avg_co2_per_job": 6.37, - "avg_water_per_job": 29.12, - "co2_reduction_vs_optimal_pct": 23.0, - "water_reduction_vs_optimal_pct": 17.3 - }, - { - "strategy": "Nearest Data Center", - "jobs_scheduled": 50, - "total_co2_kg": 301.2, - "total_water_liters": 1389.6, - "avg_co2_per_job": 6.02, - "avg_water_per_job": 27.79, - "co2_reduction_vs_optimal_pct": 18.6, - "water_reduction_vs_optimal_pct": 13.4 - }, - { - "strategy": "Time-of-Day Heuristic", - "jobs_scheduled": 50, - "total_co2_kg": 267.4, - "total_water_liters": 1298.3, - "avg_co2_per_job": 5.35, - "avg_water_per_job": 25.97, - "co2_reduction_vs_optimal_pct": 8.2, - "water_reduction_vs_optimal_pct": 7.1 - } - ], - "summary": { - "total_jobs_in_batch": 50, - "co2_improvement_vs_random_pct": 23.0, - "winner": "Optimal" - } -} -``` - -## Key Metrics - -- **co2_reduction_vs_optimal_pct** — How much worse (%) each baseline is at carbon than optimal -- **water_reduction_vs_optimal_pct** — How much worse (%) each baseline is at water than optimal -- **avg_co2_per_job** — Per-job carbon footprint (useful for normalizing across batch sizes) - -## Using This for Research - -### Expected Results - -On real data, you should see: - -1. **Optimal > Time-of-Day > Nearest > Random** - - Time-of-day respects carbon patterns → better than nearest/random - - Nearest is better than random (same city = lower latency = less overall cost) - - Random should be worst - -2. **Improvement magnitudes** - - vs Random: expect 15–30% CO₂ reduction - - vs Nearest: expect 5–15% CO₂ reduction - - vs Time-of-Day: expect 5–10% CO₂ reduction - -### Writing the Paper - -Include in your results section: - -``` -Table 4: Scheduler Comparison on 50-Job Batch (June 1, 14:00) - -Strategy Total CO₂ (kg) Reduction vs Optimal Jobs Scheduled -────────────────────────────────────────────────────────────────────── -Optimal (Ours) 245.3 — 50 -Time-of-Day 267.4 8.2% 50 -Nearest Neighbor 301.2 18.6% 50 -Random 318.7 23.0% 50 -``` - -Then add prose: - -> The carbon-aware scheduler outperforms all baselines. Against random routing (the worst case), it saves 23% CO₂; against nearest-neighbor (industry standard), it saves 18.6%. Even the time-of-day heuristic, which uses real-time grid data to pick the lowest-carbon city, is beaten by 8.2% because our method also optimizes routing over time and incorporates water efficiency constraints. - -## Running Bulk Comparisons - -To generate results for multiple time windows: - -```python -from src.api import app -from src.app_backend import SustainabilitySchedulingBackend -import pandas as pd -from fastapi.testclient import TestClient - -client = TestClient(app) - -# Run comparison for 7 days, every 6 hours -start = pd.Timestamp("2024-06-01") -results_df_rows = [] - -for i in range(0, 7 * 24, 6): # Every 6 hours - time = start + pd.Timedelta(hours=i) - response = client.post( - "/compare-baselines", - json={"time": time.isoformat(), "batch_size": 50, "priority": "medium"} - ) - - data = response.json() - for strategy_result in data["results"]: - results_df_rows.append({ - "timestamp": data["timestamp"], - "strategy": strategy_result["strategy"], - "total_co2_kg": strategy_result["total_co2_kg"], - "jobs_scheduled": strategy_result["jobs_scheduled"], - }) - -results_df = pd.DataFrame(results_df_rows) -print(results_df.groupby("strategy")["total_co2_kg"].mean()) -``` - -## Files - -- `src/baselines.py` — Three baseline strategies -- `tests/test_baselines.py` — 14 unit tests covering each baseline -- `src/api.py` — `/compare-baselines` endpoint -- `src/app_backend.py` — `run_baseline_comparison()` method diff --git a/docs/DATA_PROVENANCE.md b/docs/DATA_PROVENANCE.md new file mode 100644 index 0000000..9dbbe5f --- /dev/null +++ b/docs/DATA_PROVENANCE.md @@ -0,0 +1,81 @@ +# Data Provenance and Schemas + +## Default experiment + +The default experiment uses no downloaded data. `src/synthetic_benchmark.py` +generates every input deterministically from a seed. The four zone names are +fictional. Signal levels, site characteristics, and jobs are plausible test +values created for method development. They are not measurements and do not +represent named companies, facilities, grids, or communities. + +| Item | Provenance | Version control status | +|---|---|---| +| Hourly carbon and water signals | Deterministic synthetic generator | Source code and generated aggregate results are tracked | +| Job arrivals, duration, power, slack, origin, priority | Deterministic synthetic generator | Source code is tracked | +| PUE, capacity, water factor, scarcity | Deterministic synthetic configuration | Source code is tracked | +| Result tables and figures | `python -m src.experiments` | Tracked | + +## Canonical environment schema + +| Column | Unit | Meaning | +|---|---|---| +| `timestamp` | UTC hour | Start of the environmental observation | +| `city` | text | Modeled execution zone | +| `carbon_intensity_kg_per_kwh` | kg CO2e per facility kWh | Average operational electricity intensity used by the experiment | +| `water_intensity_l_per_kwh` | liters per IT kWh | Onsite WUE for cooling and humidification | + +## Canonical job schema + +| Column | Unit | Meaning | +|---|---|---| +| `job_id` | text | Unique functional unit identifier | +| `origin_city` | text | Preferred or originating execution zone | +| `power_demand_kw` | kW | Constant IT power during execution | +| `duration_hours` | hours | Positive integer, non-preemptive duration | +| `earliest_start` | UTC timestamp | Release time | +| `deadline` | UTC timestamp | Required completion time | +| `priority` | ordinal integer | Lower values are considered first | + +## Canonical data center schema + +| Column | Unit | Meaning | +|---|---|---| +| `dc_id` | text | Unique site identifier | +| `city` | text | Join key for environmental signals | +| `max_power_kw` | kW | Hourly IT power capacity | +| `pue` | ratio | Facility energy divided by IT energy | +| `water_usage_factor` | multiplier | Site-specific adjustment to the input WUE | +| `water_scarcity_index` | 0 to 1 | Relative local scarcity used in the impact score | + +## Optional external adapters + +The adapters are not used by the committed benchmark. They are starting points +for future trace-based research and require an explicit provenance record for +every download. + +| Adapter | Primary source | Output | Important boundary | +|---|---|---|---| +| `src/fetch_eia.py` | U.S. Energy Information Administration Open Data API [[R12](REFERENCES.md#r12)] | Hourly generation mix by balancing authority | Generation mix is not itself a validated marginal carbon signal | +| `src/fetch_entso_e.py` | ENTSO-E Transparency Platform [[R13](REFERENCES.md#r13)] | Hourly energy by production type after interval aggregation | Access requires registration and remains subject to platform terms | +| `src/workload_loader.py` | Microsoft Azure public VM traces [[R14](REFERENCES.md#r14)] | Scheduler job table | CPU allocation is converted to a power proxy and tenant is mapped to a synthetic origin | + +The Azure repository identifies its released trace data under CC BY 4.0 and its +code under MIT. Check the source repository at download time. EIA and ENTSO-E +users must also review the current source terms and citation requirements. + +## Required provenance record for a real-data study + +For each file, record: + +- source organization and direct URL; +- dataset or API version; +- retrieval timestamp in UTC; +- query parameters and geographic identifiers; +- license or terms of use; +- checksum of the raw file; +- timezone and unit conversions; +- missing-data and aggregation rules; +- whether carbon is average or marginal; +- whether water includes onsite consumption, offsite electricity water, or both. + +The source pages in this document were checked on 2026-08-30. diff --git a/docs/METHODOLOGY.md b/docs/METHODOLOGY.md new file mode 100644 index 0000000..8e699c2 --- /dev/null +++ b/docs/METHODOLOGY.md @@ -0,0 +1,138 @@ +# Methodology + +## System boundary + +The modeled unit is one non-preemptive compute job. Each job has an origin, +IT power demand, integer duration, release time, deadline, and priority. Each data +center has an hourly IT power limit, PUE, water-use factor, and water-scarcity +index. Environmental inputs provide hourly carbon intensity and onsite Water +Usage Effectiveness for each modeled zone. + +All timestamps are converted to UTC and stored without a timezone after +conversion. Hourly windows use a closed start and open end. A two-hour job that +starts at 01:00 occupies the 01:00 and 02:00 slots and ends at 03:00. + +## Physical accounting + +For job j at data center d over execution window H: + +```text +IT energy (kWh) = power demand (kW) * duration (h) + +Operational CO2e (kg) = power demand * PUE_d + * sum over H of carbon intensity (kg CO2e/kWh) + +Onsite water (L) = power demand + * sum over H of WUE (L/IT kWh) + * site water-use factor + +Scarcity-weighted water (impact liters) = onsite water + * (1 + scarcity index) +``` + +PUE is applied to electricity and carbon. It is not applied to WUE because WUE is +defined relative to IT energy [[R15](REFERENCES.md#r15)]. Scarcity-weighted water +is an impact score, not a claim about additional physical liters consumed. + +## Feasibility constraints + +A candidate placement is feasible only when: + +```text +start >= earliest_start +start + duration <= deadline +IT power already reserved + job power <= site max power in every occupied hour +carbon and water signals exist for every occupied hour +``` + +No fallback mean fills missing execution hours. Missing realized data is treated +as an evaluation error because silent imputation would weaken traceability. + +## Multi-objective score + +Raw carbon and water values use different units and scales. Before combining +them, candidate impacts are normalized within the feasible candidate set for the +current job: + +```text +normalized(x) = (x - minimum feasible x) / (maximum feasible x - minimum feasible x) +``` + +If every candidate has the same value, the normalized component is zero. The +default score is: + +```text +score = 0.55 * normalized operational CO2e + + 0.35 * normalized scarcity-weighted water + + 0.10 * relocation indicator +``` + +The weights are non-negative and must sum to one. The relocation indicator is 0 +for the origin zone and 1 for another zone. It is a transparent proxy, not a +network-latency model. + +Jobs are considered in priority, release-time, and job-ID order. The proposed +policy greedily reserves the lowest-scoring feasible candidate. This is a +heuristic, not a proof of global optimality. + +## Baselines + +All baselines share the same feasibility checks and job order. + +| Baseline | Rule | +|---|---| +| Immediate local | Prefer the origin zone, then choose the earliest feasible slot | +| Carbon greedy | Choose the feasible candidate with the least predicted operational CO2e | +| Random feasible | Choose one feasible candidate using a seeded random generator | + +The carbon-greedy baseline is intentionally strong for its single objective. A +balanced policy is expected to trade some carbon performance for water impact in +cases where those objectives conflict. + +## Paired evaluation + +Environmental reductions are calculated only for job IDs completed by both the +baseline and proposed policy. Coverage is reported separately for each policy. +The common job count and common IT energy are saved with every comparison. This +prevents a policy from appearing cleaner only because it completed less work. +The approach is consistent with the requirement that a sustainability comparison +keep the functional unit and methodology unchanged [[R16](REFERENCES.md#r16)]. + +## Synthetic benchmark + +The default benchmark has: + +- four fictional zones; +- 168 hourly observations per zone; +- 180 jobs per seed; +- heterogeneous power, duration, slack, origin, and priority; +- explicit carbon-water tradeoffs; +- 10 workload and signal seeds beginning at seed 42. + +The signals are plausible test values, not reconstructions of real facilities. +Their purpose is repeatability, regression testing, and transparent method +development. + +## Uncertainty reporting + +Tables report the arithmetic mean across seeds and an approximate 95% confidence +interval half-width: + +```text +CI half-width = 1.96 * sample standard deviation / square root of seed count +``` + +Ten seeds support a useful reproducibility check but not a population-level +claim. A publication study should use more seeds and trace-derived workloads. + +## Forecast stress test + +The robustness experiment isolates carbon scheduling by setting the carbon +weight to 1 and the other weights to 0. Gaussian error with zero mean is added to +each hourly carbon intensity. Its standard deviation is the configured fraction +of that city's mean intensity. Negative forecasts are clipped to zero. + +The noisy policy and perfect-forecast reference are evaluated on realized carbon +for every occupied hour, including PUE. Only jobs completed by both are compared. +The reported quantity is a delta against the perfect-forecast policy, not global +optimization regret. diff --git a/docs/REFERENCES.md b/docs/REFERENCES.md new file mode 100644 index 0000000..4983e1f --- /dev/null +++ b/docs/REFERENCES.md @@ -0,0 +1,123 @@ +# Verified References + +The publication links below point to publisher, conference, government, +institutional, or official project pages. Web sources were checked on +2026-08-30. A machine-readable version is available in `references.bib`. + +## R1 + +Arman Shehabi, Sarah Josephine Smith, Alex Hubbard, Alexander Newkirk, Nuoa Lei, +Md AbuBakar Siddik, Billie Holecek, Jonathan G. Koomey, Eric R. Masanet, and +Dale A. Sartor. 2024. *2024 United States Data Center Energy Usage Report*. +Lawrence Berkeley National Laboratory. DOI: +[10.71468/P1WC7Q](https://doi.org/10.71468/P1WC7Q). + +## R2 + +Ana Radovanović et al. 2023. "Carbon-Aware Computing for Datacenters." +*IEEE Transactions on Power Systems* 38(2): 1270-1280. DOI: +[10.1109/TPWRS.2022.3173250](https://doi.org/10.1109/TPWRS.2022.3173250). + +## R3 + +Philipp Wiesner, Ilja Behnke, Dominik Scheinert, Kordian Gontarska, and Lauritz +Thamsen. 2021. "Let's Wait Awhile: How Temporal Workload Shifting Can Reduce +Carbon Emissions in the Cloud." In *Proceedings of the 22nd International +Middleware Conference*. DOI: +[10.1145/3464298.3493399](https://doi.org/10.1145/3464298.3493399). + +## R4 + +Mohammad A. Islam, Kishwar Ahmed, Shaolei Ren, and Gang Quan. 2014. "Exploiting +Temporal Diversity of Water Efficiency to Make Data Center Less 'Thirsty'." +In *11th International Conference on Autonomic Computing*, 145-152. +[USENIX publication page](https://www.usenix.org/conference/icac14/technical-sessions/presentation/islam). + +## R5 + +Bilge Acun, Benjamin Lee, Fiodar Kazhamiaka, Kiwan Maeng, Udit Gupta, Manoj +Chakkaravarthy, David Brooks, and Carole-Jean Wu. 2023. "Carbon Explorer: A +Holistic Framework for Designing Carbon Aware Datacenters." In *ASPLOS 2023*, +118-132. DOI: +[10.1145/3575693.3575754](https://doi.org/10.1145/3575693.3575754). + +## R6 + +Abel Souza, Noman Bashir, Jorge Murillo, Walid A. Hanafy, Qianlin Liang, David +Irwin, and Prashant Shenoy. 2023. "Ecovisor: A Virtual Energy System for +Carbon-Efficient Applications." In *ASPLOS 2023*, 252-265. DOI: +[10.1145/3575693.3575709](https://doi.org/10.1145/3575693.3575709). + +## R7 + +Walid A. Hanafy, Qianlin Liang, Noman Bashir, David Irwin, and Prashant Shenoy. +2023. "CarbonScaler: Leveraging Cloud Workload Elasticity for Optimizing +Carbon-Efficiency." *Proceedings of the ACM on Measurement and Analysis of +Computing Systems* 7(3), Article 57. DOI: +[10.1145/3626788](https://doi.org/10.1145/3626788). + +## R8 + +Sirui Qi, Dejan Milojicic, Cullen Bash, and Sudeep Pasricha. 2023. "SHIELD: +Sustainable Hybrid Evolutionary Learning Framework for Carbon, Wastewater, and +Energy-Aware Data Center Management." In *14th International Green and +Sustainable Computing Conference*. DOI: +[10.1145/3634769.3634810](https://doi.org/10.1145/3634769.3634810). + +## R9 + +Pengfei Li, Jianyi Yang, Adam Wierman, and Shaolei Ren. 2024. "Towards +Environmentally Equitable AI via Geographical Load Balancing." In *15th ACM +International Conference on Future and Sustainable Energy Systems*. DOI: +[10.1145/3632775.3661938](https://doi.org/10.1145/3632775.3661938). + +## R10 + +Kaiqiang Xu, Decang Sun, Han Tian, Junxue Zhang, and Kai Chen. 2025. "GREEN: +Carbon-efficient Resource Scheduling for Machine Learning Clusters." In *22nd +USENIX Symposium on Networked Systems Design and Implementation*, 999-1014. +[USENIX publication page](https://www.usenix.org/conference/nsdi25/presentation/xu-kaiqiang). + +## R11 + +Pengfei Li, Jianyi Yang, Mohammad A. Islam, and Shaolei Ren. 2025. "Making AI +Less 'Thirsty'." *Communications of the ACM* 68(7): 54-61. DOI: +[10.1145/3724499](https://doi.org/10.1145/3724499). + +## R12 + +U.S. Energy Information Administration. *Open Data API and Electricity Data*. +[Official API page](https://www.eia.gov/opendata/index.php/api) and +[technical documentation](https://www.eia.gov/opendata/documentation.php). + +## R13 + +European Network of Transmission System Operators for Electricity. *ENTSO-E +Transparency Platform*. +[Official platform information](https://www.entsoe.eu/data/transparency-platform/) +and [actual generation by production type](https://transparency.entsoe.eu/generation/r2/actualGenerationPerProductionType/show). + +## R14 + +Microsoft Azure. *Azure Public Dataset: Microsoft Azure Traces*. +[Official GitHub repository](https://github.com/Azure/AzurePublicDataset). + +## R15 + +Microsoft. *Measuring Energy and Water Efficiency for Microsoft Datacenters*. +[Official definition and reporting page](https://datacenters.microsoft.com/sustainability/efficiency/). + +## R16 + +Green Software Foundation. *Software Carbon Intensity Specification*. +[Official specification source](https://github.com/Green-Software-Foundation/iso_update_review/blob/main/Software_Carbon_Intensity_Specification.md). + +## R17 + +Google Cloud. *Optimize Resource Usage for Sustainability*. +[Official architecture guidance](https://docs.cloud.google.com/architecture/framework/sustainability/optimize-resource-usage). + +## R18 + +Amazon Web Services. *Sustainability Pillar, AWS Well-Architected Framework*. +[Official framework document](https://docs.aws.amazon.com/wellarchitected/latest/sustainability-pillar/sustainability-pillar.html). diff --git a/docs/RELATED_WORK.md b/docs/RELATED_WORK.md new file mode 100644 index 0000000..83c2239 --- /dev/null +++ b/docs/RELATED_WORK.md @@ -0,0 +1,42 @@ +# Related Work and Project Positioning + +## Summary + +Carbon-aware computing and water-aware workload management are established +research areas. The defensible value of this project is its reproducible +evaluation scaffold, not a first-of-kind scheduling claim. + +| Work | Main contribution | Difference from this project | +|---|---|---| +| Radovanovic et al., Carbon-Aware Computing for Datacenters [[R2](REFERENCES.md#r2)] | Risk-aware control of temporally flexible workloads across Google's fleet using forecast-driven capacity curves | This artifact is a small offline simulator. It adds explicit onsite water and paired-job metrics, but it has no fleet deployment or production validation. | +| Wiesner et al., Let's Wait Awhile [[R3](REFERENCES.md#r3)] | Quantifies temporal shifting potential and studies carbon forecast accuracy across four grids | This project combines temporal and geographic placement with site capacity and water impact. Its default data are synthetic rather than a year of historical grid data. | +| Islam et al., WACE [[R4](REFERENCES.md#r4)] | Online delay-tolerant scheduling that jointly considers water, carbon, and electricity cost | WACE predates and overlaps the core environmental objective. This project omits price and online control, while emphasizing paired evaluation and a reproducible offline artifact. | +| Acun et al., Carbon Explorer [[R5](REFERENCES.md#r5)] | Design-space analysis across renewable capacity, batteries, scheduling, operational carbon, and embodied carbon | This project studies job placement only. It does not model renewable procurement, storage, or embodied emissions. | +| Souza et al., Ecovisor [[R6](REFERENCES.md#r6)] | Application-facing abstraction for renewable energy, grid carbon, server power, and batteries | This project does not expose an application runtime or energy virtualization layer. It evaluates a scheduling policy offline. | +| Hanafy et al., CarbonScaler [[R7](REFERENCES.md#r7)] | Dynamically changes resource allocation for elastic batch workloads to improve carbon efficiency | This project assumes fixed job power and non-preemptive execution. It shifts start time and location instead of scaling resource allocation. | +| Qi et al., SHIELD [[R8](REFERENCES.md#r8)] | Multi-objective geo-distributed optimization of carbon, wastewater, and energy cost using evolutionary search and learned guidance | SHIELD is the closest objective-level comparison. This project is simpler and more interpretable, adds deadline and hourly capacity checks, and avoids claiming Pareto superiority. | +| Li et al., environmentally equitable AI [[R9](REFERENCES.md#r9)] | Carbon and water aware geographic load balancing with an environmental equity objective | This project uses a scarcity-weighted water metric but does not model distributional equity across communities. Equity is a future extension, not a current result. | +| Xu et al., GREEN [[R10](REFERENCES.md#r10)] | Carbon-efficient ML cluster scheduler with job-completion-time evaluation and production workload traces | This project adds geographic and water dimensions, but lacks a cluster prototype, detailed JCT model, and production trace validation. | +| Li et al., Making AI Less Thirsty [[R11](REFERENCES.md#r11)] | Method and policy discussion for the spatial and temporal water footprint of AI | This project operationalizes onsite WUE in a scheduler, but it excludes electricity-generation water and should not be read as a full AI water-footprint model. | + +## Claim boundary + +The project can reasonably claim that it implements and tests a transparent +combination of known ideas under a consistent evaluation protocol. It should not +claim any of the following: + +- the first carbon-aware scheduler; +- the first water-aware scheduler; +- the first joint carbon-water scheduler; +- production readiness; +- superiority to evolutionary, optimization, or deployed systems; +- real-world savings based on the synthetic benchmark. + +## Research gap this artifact can support + +A stronger future paper could investigate whether paired functional-unit +evaluation, forecast uncertainty, deadline slack, and regional water stress +change the conclusions drawn from conventional carbon-only baselines. That study +would need trace-derived jobs, measured or operator-reported WUE, authoritative +hourly carbon data, network and policy constraints, and a larger experimental +design. diff --git a/docs/REPRODUCIBILITY.md b/docs/REPRODUCIBILITY.md new file mode 100644 index 0000000..fecb3a3 --- /dev/null +++ b/docs/REPRODUCIBILITY.md @@ -0,0 +1,89 @@ +# Reproducibility Checklist + +## Environment + +- Python 3.11 or later +- Dependency versions pinned in `requirements.txt` and `requirements-dev.txt` +- No API key or network access required for the default benchmark +- Fixed seed start, seed count, jobs, horizon, and weights in run metadata +- Headless figure generation for CI and remote systems + +## Clean run + +```bash +git clone https://github.com/drona23/Personal_Project.git +cd Personal_Project +python -m venv .venv +source .venv/bin/activate +python -m pip install -r requirements.txt -r requirements-dev.txt +python -m ruff check src tests +python -m pytest +python -m src.experiments +``` + +## Expected checks + +The test suite covers: + +- UTC conversion; +- exact deadline completion and impossible deadlines; +- hourly capacity enforcement; +- score-weight validation; +- paired-job comparison when coverage differs; +- immediate-local and seeded-random baseline behavior; +- full-window realized carbon accounting with PUE; +- zero-noise forecast identity; +- EIA fuel-code parsing; +- ENTSO-E 15-minute energy aggregation; +- end-to-end artifact creation. + +## Output verification + +After the full run, confirm: + +```bash +python -m pytest +python -m ruff check src tests +git diff --exit-code -- results docs/figures +cd results && shasum -a 256 -c SHA256SUMS +``` + +The final command succeeds when the regenerated tracked artifacts match the +committed files. PNG bytes can vary if operating-system font rendering differs, +even with the same Matplotlib version. In that case, compare the CSV files first +and inspect the figures visually. + +## Parameterized run + +```bash +python -m src.experiments \ + --seeds 30 \ + --jobs 500 \ + --hours 336 \ + --results-dir scratch/results \ + --figures-dir scratch/figures +``` + +Use a scratch path for exploratory runs so the reviewed benchmark remains +unchanged. + +## Continuous integration + +`.github/workflows/ci.yml` installs the pinned environment, runs Ruff and pytest, +and executes a smaller offline smoke benchmark. No private dataset or developer +home-directory fallback is used. + +## Reporting checklist for a paper extension + +- Commit the experiment configuration. +- Save raw and derived data checksums. +- Record source versions and retrieval times. +- Separate average from marginal carbon intensity. +- Define onsite and offsite water boundaries. +- Report every policy's coverage. +- Use a paired functional unit for impact reductions. +- Publish seed-level outputs, not only aggregate charts. +- Include negative and null results. +- Document all exclusions and failed runs. +- Replace the placeholder author metadata in `CITATION.cff`. +- Select and add a software license before inviting external reuse. diff --git a/docs/RESEARCH_PROBLEM.md b/docs/RESEARCH_PROBLEM.md new file mode 100644 index 0000000..8aedaa4 --- /dev/null +++ b/docs/RESEARCH_PROBLEM.md @@ -0,0 +1,93 @@ +# Research Problem and Use Cases + +## Business problem + +Compute operators usually optimize placement for service quality, utilization, +and cost. Environmental signals introduce another operational question: if a job +can wait and more than one site can execute it, which feasible site and start time +produce the preferred carbon-water tradeoff? + +This is not a recommendation to delay interactive requests or move regulated data +across borders. The scoped workload is non-preemptive, deadline-flexible compute, +such as model training, batch inference, scientific workflows, data engineering, +CI jobs, report generation, backups, and software maintenance. Temporal and +geographic shifting of flexible workloads is also recognized in published systems +research and cloud architecture guidance [[R2](REFERENCES.md#r2), +[R3](REFERENCES.md#r3), [R17](REFERENCES.md#r17), +[R18](REFERENCES.md#r18)]. + +## Decision statement + +Given hourly forecasts, a set of jobs, and a portfolio of data centers, select one +site and one contiguous execution window for each admitted job so that: + +1. execution does not start before release time; +2. execution ends no later than the deadline; +3. IT power does not exceed site capacity in any hour; +4. environmental impact and relocation preference are evaluated consistently; +5. unscheduled jobs remain visible in the reported coverage metric. + +## Primary research question + +How much do spatiotemporal placement decisions change operational carbon, +physical onsite water use, and scarcity-weighted water impact relative to +feasible operational baselines, at equal completed work? + +## Secondary questions + +- How does a balanced carbon-water policy compare with a carbon-only policy? +- How sensitive is a carbon-only policy to forecast error? +- Do claimed reductions persist across workload and signal seeds? +- What fraction of submitted jobs remains feasible under each policy? + +## Organizations that could benefit + +### Cloud and AI infrastructure operators + +AWS, Google Cloud, Microsoft Azure, and other multi-region operators are +representative of organizations with placement flexibility. Cloud guidance +already treats region choice and carbon-aware timing as sustainability decisions +[[R17](REFERENCES.md#r17), [R18](REFERENCES.md#r18)]. A production version of this +research could inform a batch admission controller or planning service. + +### Universities, national laboratories, and HPC centers + +Research institutions operate queued workloads with explicit resource requests +and completion expectations. They can use this artifact to study how much +deadline slack is needed before environmental signals affect scheduling, without +changing a production scheduler. + +### Colocation and enterprise data center portfolios + +Operators with several sites can use the experiment design for scenario analysis. +The most relevant decision is whether carbon and water objectives align or +conflict across facilities. + +### Utilities, grid operators, and public agencies + +Flexible compute can be studied as a form of load flexibility. This repository +can support controlled simulations that connect hourly grid signals to admitted +IT load, but it does not yet model grid dispatch or demand response payments. + +## Success criteria + +A useful research result must report all of the following: + +- deadline and capacity feasibility; +- coverage for every policy; +- the number and IT energy of jobs completed by both policies; +- operational carbon in kg CO2e; +- physical onsite water in liters; +- scarcity-weighted water as a separate decision metric; +- uncertainty across seeds; +- sensitivity to imperfect forecasts; +- limitations that prevent direct operational claims. + +## Non-goals + +- A production orchestrator or cloud broker +- An interactive dashboard +- A real-time API +- A claim that carbon-aware or water-aware scheduling is novel +- A claim that synthetic results predict savings for a named company +- A complete life-cycle assessment of computing diff --git a/docs/RESULTS.md b/docs/RESULTS.md new file mode 100644 index 0000000..26cfaad --- /dev/null +++ b/docs/RESULTS.md @@ -0,0 +1,76 @@ +# Results and Output Guide + +## Run configuration + +The committed artifacts were regenerated with: + +```bash +python -m src.experiments --seeds 10 --jobs 180 --hours 168 +``` + +The seed range is 42 through 51. The proposed weights are 0.55 for carbon, +0.35 for scarcity-weighted water, and 0.10 for relocation. The configuration is +also recorded in `results/run_metadata.json`. + +## Main result + +All four policies completed 100% of submitted jobs for all 10 seeds. Paired +comparisons therefore contain 180 jobs per seed and an average of 19,073.6 kWh +of common IT work. + +| Baseline | CO2e reduction | Physical water reduction | Scarcity-weighted water reduction | Proposed coverage | Baseline coverage | +|---|---:|---:|---:|---:|---:| +| Immediate local | 23.94% (CI 1.92) | 28.60% (CI 1.89) | 20.08% (CI 1.28) | 100% | 100% | +| Random feasible | 22.31% (CI 2.18) | 29.50% (CI 2.81) | 20.78% (CI 1.87) | 100% | 100% | +| Carbon greedy | -46.88% (CI 2.11) | 52.75% (CI 1.47) | 39.17% (CI 1.17) | 100% | 100% | + +A negative reduction means the proposed policy used more operational CO2e. In +this benchmark, the balanced policy used 46.88% more CO2e than carbon greedy. +This is expected because carbon greedy ignores water and relocation. The result +shows that the objectives conflict and that the chosen weights matter. + +![Benchmark overview](figures/benchmark_overview.png) + +## Forecast stress test + +The robustness study uses a carbon-only policy so the reference and noisy policy +optimize the same objective. It evaluates every hour of every scheduled job with +realized intensity and PUE. + +| Noise standard deviation | Forecast RMSE | CO2e delta vs perfect forecast | Paired jobs | Coverage | +|---:|---:|---:|---:|---:| +| 0% | 0.0000 | 0.00% (CI 0.00) | 180 | 100% | +| 5% | 0.0169 | 0.64% (CI 0.21) | 180 | 100% | +| 10% | 0.0342 | 1.23% (CI 0.17) | 180 | 100% | +| 20% | 0.0684 | 3.65% (CI 0.54) | 180 | 100% | +| 35% | 0.1196 | 11.63% (CI 1.56) | 180 | 100% | +| 50% | 0.1681 | 21.42% (CI 1.71) | 180 | 100% | + +![Forecast robustness](figures/forecast_robustness.png) + +## Example output + +The schedule CSV keeps the submitted job fields, assignment, timestamps, +normalized decision score, physical impacts, relocation flag, completion flag, +and failure reason. Unscheduled jobs are never dropped. + +![Example schedule](figures/example_schedule.png) + +## Artifact files + +| File | Level | Interpretation | +|---|---|---| +| `results/policy_summary_by_seed.csv` | Policy and seed | Coverage, total impacts, and relocation rate | +| `results/paired_comparisons_by_seed.csv` | Baseline and seed | Equal-work environmental comparison | +| `results/benchmark_summary.csv` | Baseline aggregate | Mean and approximate 95% CI across seeds | +| `results/forecast_robustness.csv` | Noise level aggregate | Forecast RMSE, realized carbon delta, coverage, and paired jobs | +| `results/example_schedule.csv` | Job | Auditable assignment output for seed 42 | +| `results/run_metadata.json` | Run | Benchmark version, parameters, weights, and claim scope | +| `results/SHA256SUMS` | Run | SHA-256 digests for generated tables, metadata, and figures | + +## Interpretation boundary + +The benchmark is designed to test code and methodology. It is not calibrated to +a real data center fleet. The percentages must not be used in a sustainability +report, investment case, or operational forecast without replacing synthetic +inputs and validating the full system boundary. diff --git a/docs/THREATS_TO_VALIDITY.md b/docs/THREATS_TO_VALIDITY.md new file mode 100644 index 0000000..5170f64 --- /dev/null +++ b/docs/THREATS_TO_VALIDITY.md @@ -0,0 +1,80 @@ +# Threats to Validity + +## External validity + +The default inputs are synthetic. They demonstrate behavior under controlled +tradeoffs but cannot estimate savings for an actual fleet. Real job arrivals, +hardware constraints, carbon signals, WUE, and capacity congestion may produce +materially different results. + +Only four zones and one simulated week are modeled per seed. Seasonal effects, +grid outages, heat waves, drought restrictions, and long-running workloads are +outside the benchmark. + +## Construct validity + +Operational carbon uses average grid intensity. Marginal emissions may be more +appropriate when the research question concerns the consequence of an +incremental load decision. + +The water model includes onsite cooling and humidification represented by WUE. +It excludes electricity-generation water, supply-chain water, and withdrawal +versus consumption distinctions. Research on AI water footprints shows why those +boundaries matter [[R11](REFERENCES.md#r11)]. + +Scarcity-weighted water is a simple multiplier. It is not a watershed impact +assessment or environmental equity metric. The equity dimension studied by Li +et al. requires a broader distributional model [[R9](REFERENCES.md#r9)]. + +The relocation indicator is not measured latency, data-transfer energy, network +cost, or legal feasibility. + +## Internal validity + +The scheduler is greedy. Earlier assignments consume capacity that can change +later decisions. The chosen job order and tie-breaking rules therefore affect +the result. No claim of global optimality is made. + +Carbon and water components are normalized within each job's current feasible +candidate set. This makes weights interpretable within a decision, but scores +are not comparable across jobs. + +The random baseline is seeded and feasible, but it is not an estimate of a real +operator's policy. Immediate local is also a simplified operational reference. + +## Statistical conclusion validity + +Approximate confidence intervals use 10 seeds and a normal critical value. The +seeds are independent generator states but do not represent a sampled population +of real fleets. More seeds and trace-derived replications are needed for a paper. + +The forecast-error process is Gaussian and independent by hour. Real forecasting +errors can be biased, autocorrelated, heavy-tailed, and synchronized across +regions. The reported stress test is diagnostic, not a calibrated forecast study. + +## Measurement validity + +PUE and WUE can vary with load, temperature, cooling technology, and reporting +boundary. The benchmark treats PUE as static per site and WUE as hourly input. +Microsoft's public definitions illustrate that both metrics depend on facility +and location conditions [[R15](REFERENCES.md#r15)]. + +Power demand is constant throughout a job. Accelerator utilization, idle power, +checkpointing, and variable resource profiles are not modeled. + +## Missing production constraints + +The artifact does not enforce: + +- data residency and privacy; +- hardware or accelerator compatibility; +- network bandwidth and transfer time; +- storage locality; +- electricity price or demand charges; +- job dependencies and preemption; +- queue fairness across users; +- failures, maintenance, or reserve margins; +- embodied emissions and hardware lifetime. + +These limitations prevent production use but create a clear agenda for follow-on +research. diff --git a/docs/data_requirements.md b/docs/data_requirements.md deleted file mode 100644 index ef78036..0000000 --- a/docs/data_requirements.md +++ /dev/null @@ -1,148 +0,0 @@ -# Data Requirements and Online Sources - -## Best Dataset Available Right Now - -The strongest dataset currently available for this project is the master table -inside `Archive 3.zip`: - -- `base_data_with_metrics.parquet` -- `base_data_with_metrics.csv` - -This dataset is already strong enough for the prediction pipeline because it -contains: - -- hourly timestamps from 2019 through 2023 -- city and ZIP identifiers -- eGRID regions -- weather variables -- generation mix variables -- `WUE_total` -- `liters` -- `co2_kg` -- water scarcity signal `dsci_0_500` - -## What We Still Need Exactly - -### 1. Workload or Job Trace Data - -This is the main missing input for the scheduling stage. - -Required fields: - -- `job_id` -- `origin_city` -- `power_demand` -- `duration_hours` -- `earliest_start` -- `deadline` - -Optional but helpful: - -- `max_distance_km` -- `job_type` -- `priority` -- `latency_sla_ms` - -Can we get it online: - -- Yes, public workload traces exist. -- They usually do not come with city labels, so we may still need to map or - synthesize `origin_city`. - -Good sources: - -- [Google cluster-data](https://github.com/google/cluster-data) -- [Alibaba clusterdata](https://github.com/alibaba/clusterdata) -- [Azure Trace for Packing 2020](https://github.com/Azure/AzurePublicDataset/blob/master/AzureTracesForPacking2020.md) - -Recommended first choice: - -- Azure Trace for Packing 2020 - - Direct download: - [AzurePackingTraceV1.zip](https://azurepublicdatasettraces.blob.core.windows.net/azurepublicdatasetv2/azurevmallocation_dataset2020/AzurePackingTraceV1.zip) - - Why: - it includes VM start/end times, priorities, and normalized resource sizes, - and the compressed download is manageable. - -### 2. Data Center Inventory and Constraints - -This is the second major missing input for scheduling. - -Required fields: - -- `dc_id` -- `city` -- `max_power_per_hour` -- `pue` -- `water_usage_factor` -- `region_water_scarcity` -- `is_new` - -Can we get it online: - -- Partially. -- City and region presence can be sourced online. -- Exact per-facility capacity, hourly power ceilings, and water-usage factors - are usually not public at the level we need. -- In practice, this part often needs assumptions, proxy values, or a synthetic - scenario table. - -Useful sources: - -- [AWS Global Infrastructure](https://aws.amazon.com/about-aws/global-infrastructure/) -- [Google Cloud Locations](https://cloud.google.com/about/locations) -- [Azure Global Infrastructure](https://azure.microsoft.com/en-us/explore/global-infrastructure/geographies/) -- [Google Data Center Efficiency](https://datacenters.google/efficiency/) - -### 3. Water Scarcity Weights - -We already have `dsci_0_500` in the master dataset, but if we want a fresh or -more explicit scarcity layer, we can source it online. - -Can we get it online: - -- Yes. - -Good source: - -- [WRI Aqueduct Water Risk Atlas](https://www.wri.org/data/aqueduct-water-risk-atlas) - -### 4. Forecast Weather for Future Inference - -Historical weather is already available in the master dataset, but future -forecasting requires forecast weather inputs. - -Can we get it online: - -- Yes. - -Good source: - -- [NOAA / NWS API](https://www.weather.gov/documentation/services-web-api) - -### 5. City Coordinates for Forecast APIs - -If we use NOAA or other live forecast services, we need city coordinates. - -Can we get it online: - -- Yes. - -Good source: - -- [U.S. Census Geocoder](https://geocoding.geo.census.gov/) - -## Recommendation - -The best next step is: - -1. Use `base_data_with_metrics.parquet` as the primary training dataset. -2. Use the Azure packing trace as the first workload trace source for scheduling. -3. Use the starter template in `data/templates/dc_config_template.csv` and then refine it with better capacity and efficiency assumptions. -4. Only after that, operationalize the scheduling notebook into the main codebase. - -## What We Do Not Strictly Need Right Now - -We do not need more historical weather or historical carbon data immediately if -we use the master dataset from `Archive 3.zip`, because those signals are -already present there. diff --git a/docs/figures/benchmark_overview.png b/docs/figures/benchmark_overview.png new file mode 100644 index 0000000..607455e Binary files /dev/null and b/docs/figures/benchmark_overview.png differ diff --git a/docs/figures/example_schedule.png b/docs/figures/example_schedule.png new file mode 100644 index 0000000..8289258 Binary files /dev/null and b/docs/figures/example_schedule.png differ diff --git a/docs/figures/forecast_robustness.png b/docs/figures/forecast_robustness.png new file mode 100644 index 0000000..1674a78 Binary files /dev/null and b/docs/figures/forecast_robustness.png differ diff --git a/docs/workload_trace_guide.md b/docs/workload_trace_guide.md deleted file mode 100644 index 1e7840d..0000000 --- a/docs/workload_trace_guide.md +++ /dev/null @@ -1,94 +0,0 @@ -# Workload Trace Guide - -## Recommended Public Trace - -Use the official Microsoft Azure packing trace: - -- Dataset doc: - [Azure Trace for Packing 2020](https://github.com/Azure/AzurePublicDataset/blob/master/AzureTracesForPacking2020.md) -- Direct download: - [AzurePackingTraceV1.zip](https://azurepublicdatasettraces.blob.core.windows.net/azurepublicdatasetv2/azurevmallocation_dataset2020/AzurePackingTraceV1.zip) - -## Why This One - -This trace is a good fit for the scheduling stage because it contains: - -- VM request ids -- tenant ids -- VM types -- VM priority -- start times -- end times -- normalized resource requirements through the `vmType` table - -The download is practical at roughly 51 MB compressed, and it expands into a -single SQLite file with millions of VM requests. - -## What We Downloaded Locally - -Local path used during setup: - -- `data/external/AzurePackingTraceV1.zip` -- `data/external/azure_packing/packing_trace_zone_a_v1.sqlite` - -These files are intentionally ignored by Git because they are raw external -artifacts. - -## Schema We Found - -The SQLite database contains two tables: - -- `vm` -- `vmType` - -The `vm` table provides: - -- `vmId` -- `tenantId` -- `vmTypeId` -- `priority` -- `starttime` -- `endtime` - -The `vmType` table provides: - -- `vmTypeId` -- `machineId` -- `core` -- `memory` -- `hdd` -- `ssd` -- `nic` - -## How We Use It - -The project now includes [src/workload_loader.py](../src/workload_loader.py), -which converts the Azure trace into a scheduler-oriented jobs table. - -Example: - -```bash -python3 -m src.workload_loader \ - --input-path data/external/azure_packing/packing_trace_zone_a_v1.sqlite \ - --output-path data/processed/azure_jobs_sample.csv \ - --limit 5000 \ - --start-datetime "2019-01-01 00:00:00" -``` - -## Important Limitations - -This trace is excellent for workload behavior, but it does not directly include: - -- real `origin_city` -- explicit `power_demand` in kW -- user-facing latency SLAs -- true geographic placement - -Because of that, the current converter uses proxies: - -- `origin_city` is deterministically assigned from tenant id across a city pool -- `power_demand` is approximated from normalized CPU allocation -- `deadline` is derived from observed duration plus configurable slack - -These assumptions are acceptable for a research scheduler prototype, but they -should be stated clearly in any report or presentation. diff --git a/frontend/.env.example b/frontend/.env.example deleted file mode 100644 index 6c9bbfd..0000000 --- a/frontend/.env.example +++ /dev/null @@ -1 +0,0 @@ -VITE_API_BASE_URL=http://127.0.0.1:8000 diff --git a/frontend/.gitignore b/frontend/.gitignore deleted file mode 100644 index a547bf3..0000000 --- a/frontend/.gitignore +++ /dev/null @@ -1,24 +0,0 @@ -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -pnpm-debug.log* -lerna-debug.log* - -node_modules -dist -dist-ssr -*.local - -# Editor directories and files -.vscode/* -!.vscode/extensions.json -.idea -.DS_Store -*.suo -*.ntvs* -*.njsproj -*.sln -*.sw? diff --git a/frontend/Dockerfile b/frontend/Dockerfile deleted file mode 100644 index a3ece76..0000000 --- a/frontend/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -# ─── React Frontend Dockerfile (Multi-Stage) ────────────────────────────────── - -# ── Stage 1: Build ──────────────────────────────────────────────────────────── -# Use Node to compile TypeScript and bundle with Vite -FROM node:20-slim AS builder - -WORKDIR /app - -# Install dependencies first (layer-cached unless package-lock.json changes) -COPY package.json package-lock.json ./ -RUN npm ci - -# Copy source and build -COPY . . -RUN npm run build -# Output is in /app/dist/ - -# ── Stage 2: Serve ──────────────────────────────────────────────────────────── -# Use tiny nginx image to serve the static files -FROM nginx:alpine - -# Copy only the compiled output from Stage 1 -COPY --from=builder /app/dist /usr/share/nginx/html - -# Replace default nginx config with our proxy config -COPY nginx.conf /etc/nginx/conf.d/default.conf - -# Nginx serves on port 80 by default -EXPOSE 80 - -CMD ["nginx", "-g", "daemon off;"] diff --git a/frontend/README.md b/frontend/README.md deleted file mode 100644 index eaa5ac7..0000000 --- a/frontend/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# React Visualization - -This frontend is a Vite + React + TypeScript client for the sustainability-aware -workload scheduling simulator. - -## Local development - -Start the FastAPI backend from the repository root: - -```bash -python3 -m uvicorn src.api:app --reload -``` - -Then start the frontend: - -```bash -cd frontend -npm install -npm run dev -``` - -The Vite dev server proxies `/api/*` requests to `http://127.0.0.1:8000`. - -## Environment override - -If you want to point the frontend at a different backend, create a local -`.env` file in this directory and set: - -```bash -VITE_API_BASE_URL=http://127.0.0.1:8000 -``` diff --git a/frontend/eslint.config.js b/frontend/eslint.config.js deleted file mode 100644 index 5e6b472..0000000 --- a/frontend/eslint.config.js +++ /dev/null @@ -1,23 +0,0 @@ -import js from '@eslint/js' -import globals from 'globals' -import reactHooks from 'eslint-plugin-react-hooks' -import reactRefresh from 'eslint-plugin-react-refresh' -import tseslint from 'typescript-eslint' -import { defineConfig, globalIgnores } from 'eslint/config' - -export default defineConfig([ - globalIgnores(['dist']), - { - files: ['**/*.{ts,tsx}'], - extends: [ - js.configs.recommended, - tseslint.configs.recommended, - reactHooks.configs.flat.recommended, - reactRefresh.configs.vite, - ], - languageOptions: { - ecmaVersion: 2020, - globals: globals.browser, - }, - }, -]) diff --git a/frontend/index.html b/frontend/index.html deleted file mode 100644 index 4fc2353..0000000 --- a/frontend/index.html +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - Sustainability Routing Simulator - - -
- - - diff --git a/frontend/nginx.conf b/frontend/nginx.conf deleted file mode 100644 index d71c1b8..0000000 --- a/frontend/nginx.conf +++ /dev/null @@ -1,19 +0,0 @@ -server { - listen 80; - root /usr/share/nginx/html; - index index.html; - - # Proxy all /api/* requests to the FastAPI backend container - # "backend" resolves to the backend service via Docker's internal DNS - location /api/ { - proxy_pass http://backend:8000/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } - - # SPA fallback: serve index.html for any unknown route - # (React Router handles client-side navigation) - location / { - try_files $uri $uri/ /index.html; - } -} diff --git a/frontend/package-lock.json b/frontend/package-lock.json deleted file mode 100644 index 7fffdd9..0000000 --- a/frontend/package-lock.json +++ /dev/null @@ -1,3225 +0,0 @@ -{ - "name": "frontend", - "version": "0.0.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "frontend", - "version": "0.0.0", - "dependencies": { - "@types/geojson": "^7946.0.16", - "maplibre-gl": "^5.21.1", - "react": "^19.2.4", - "react-dom": "^19.2.4" - }, - "devDependencies": { - "@eslint/js": "^9.39.4", - "@types/node": "^24.12.0", - "@types/react": "^19.2.14", - "@types/react-dom": "^19.2.3", - "@vitejs/plugin-react": "^6.0.1", - "eslint": "^9.39.4", - "eslint-plugin-react-hooks": "^7.0.1", - "eslint-plugin-react-refresh": "^0.5.2", - "globals": "^17.4.0", - "typescript": "~5.9.3", - "typescript-eslint": "^8.57.0", - "vite": "^8.0.1" - } - }, - "node_modules/@babel/code-frame": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", - "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-validator-identifier": "^7.28.5", - "js-tokens": "^4.0.0", - "picocolors": "^1.1.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/compat-data": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.0.tgz", - "integrity": "sha512-T1NCJqT/j9+cn8fvkt7jtwbLBfLC/1y1c7NtCeXFRgzGTsafi68MRv8yzkYSapBnFA6L3U2VSc02ciDzoAJhJg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/core": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.0.tgz", - "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.29.0", - "@babel/generator": "^7.29.0", - "@babel/helper-compilation-targets": "^7.28.6", - "@babel/helper-module-transforms": "^7.28.6", - "@babel/helpers": "^7.28.6", - "@babel/parser": "^7.29.0", - "@babel/template": "^7.28.6", - "@babel/traverse": "^7.29.0", - "@babel/types": "^7.29.0", - "@jridgewell/remapping": "^2.3.5", - "convert-source-map": "^2.0.0", - "debug": "^4.1.0", - "gensync": "^1.0.0-beta.2", - "json5": "^2.2.3", - "semver": "^6.3.1" - }, - "engines": { - "node": ">=6.9.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/babel" - } - }, - "node_modules/@babel/generator": { - "version": "7.29.1", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.1.tgz", - "integrity": "sha512-qsaF+9Qcm2Qv8SRIMMscAvG4O3lJ0F1GuMo5HR/Bp02LopNgnZBC/EkbevHFeGs4ls/oPz9v+Bsmzbkbe+0dUw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/parser": "^7.29.0", - "@babel/types": "^7.29.0", - "@jridgewell/gen-mapping": "^0.3.12", - "@jridgewell/trace-mapping": "^0.3.28", - "jsesc": "^3.0.2" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-compilation-targets": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.28.6.tgz", - "integrity": "sha512-JYtls3hqi15fcx5GaSNL7SCTJ2MNmjrkHXg4FSpOA/grxK8KwyZ5bubHsCq8FXCkua6xhuaaBit+3b7+VZRfcA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/compat-data": "^7.28.6", - "@babel/helper-validator-option": "^7.27.1", - "browserslist": "^4.24.0", - "lru-cache": "^5.1.1", - "semver": "^6.3.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-globals": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", - "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-imports": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.28.6.tgz", - "integrity": "sha512-l5XkZK7r7wa9LucGw9LwZyyCUscb4x37JWTPz7swwFE/0FMQAGpiWUZn8u9DzkSBWEcK25jmvubfpw2dnAMdbw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/traverse": "^7.28.6", - "@babel/types": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-transforms": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.6.tgz", - "integrity": "sha512-67oXFAYr2cDLDVGLXTEABjdBJZ6drElUSI7WKp70NrpyISso3plG9SAGEF6y7zbha/wOzUByWWTJvEDVNIUGcA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-module-imports": "^7.28.6", - "@babel/helper-validator-identifier": "^7.28.5", - "@babel/traverse": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0" - } - }, - "node_modules/@babel/helper-string-parser": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", - "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-identifier": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", - "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-option": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", - "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helpers": { - "version": "7.29.2", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.2.tgz", - "integrity": "sha512-HoGuUs4sCZNezVEKdVcwqmZN8GoHirLUcLaYVNBK2J0DadGtdcqgr3BCbvH8+XUo4NGjNl3VOtSjEKNzqfFgKw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/template": "^7.28.6", - "@babel/types": "^7.29.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/parser": { - "version": "7.29.2", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.2.tgz", - "integrity": "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.29.0" - }, - "bin": { - "parser": "bin/babel-parser.js" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@babel/template": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz", - "integrity": "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.28.6", - "@babel/parser": "^7.28.6", - "@babel/types": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/traverse": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.0.tgz", - "integrity": "sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.29.0", - "@babel/generator": "^7.29.0", - "@babel/helper-globals": "^7.28.0", - "@babel/parser": "^7.29.0", - "@babel/template": "^7.28.6", - "@babel/types": "^7.29.0", - "debug": "^4.3.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/types": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", - "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-string-parser": "^7.27.1", - "@babel/helper-validator-identifier": "^7.28.5" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@emnapi/core": { - "version": "1.9.2", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.2.tgz", - "integrity": "sha512-UC+ZhH3XtczQYfOlu3lNEkdW/p4dsJ1r/bP7H8+rhao3TTTMO1ATq/4DdIi23XuGoFY+Cz0JmCbdVl0hz9jZcA==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.1", - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.9.2", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.2.tgz", - "integrity": "sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/wasi-threads": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", - "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@eslint-community/eslint-utils": { - "version": "4.9.1", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", - "integrity": "sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "eslint-visitor-keys": "^3.4.3" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - }, - "peerDependencies": { - "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" - } - }, - "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", - "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/@eslint-community/regexpp": { - "version": "4.12.2", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", - "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^12.0.0 || ^14.0.0 || >=16.0.0" - } - }, - "node_modules/@eslint/config-array": { - "version": "0.21.2", - "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.2.tgz", - "integrity": "sha512-nJl2KGTlrf9GjLimgIru+V/mzgSK0ABCDQRvxw5BjURL7WfH5uoWmizbH7QB6MmnMBd8cIC9uceWnezL1VZWWw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@eslint/object-schema": "^2.1.7", - "debug": "^4.3.1", - "minimatch": "^3.1.5" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - } - }, - "node_modules/@eslint/config-helpers": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", - "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@eslint/core": "^0.17.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - } - }, - "node_modules/@eslint/core": { - "version": "0.17.0", - "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", - "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@types/json-schema": "^7.0.15" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - } - }, - "node_modules/@eslint/eslintrc": { - "version": "3.3.5", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.5.tgz", - "integrity": "sha512-4IlJx0X0qftVsN5E+/vGujTRIFtwuLbNsVUe7TO6zYPDR1O6nFwvwhIKEKSrl6dZchmYBITazxKoUYOjdtjlRg==", - "dev": true, - "license": "MIT", - "dependencies": { - "ajv": "^6.14.0", - "debug": "^4.3.2", - "espree": "^10.0.1", - "globals": "^14.0.0", - "ignore": "^5.2.0", - "import-fresh": "^3.2.1", - "js-yaml": "^4.1.1", - "minimatch": "^3.1.5", - "strip-json-comments": "^3.1.1" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/@eslint/eslintrc/node_modules/globals": { - "version": "14.0.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", - "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@eslint/js": { - "version": "9.39.4", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.4.tgz", - "integrity": "sha512-nE7DEIchvtiFTwBw4Lfbu59PG+kCofhjsKaCWzxTpt4lfRjRMqG6uMBzKXuEcyXhOHoUp9riAm7/aWYGhXZ9cw==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://eslint.org/donate" - } - }, - "node_modules/@eslint/object-schema": { - "version": "2.1.7", - "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz", - "integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - } - }, - "node_modules/@eslint/plugin-kit": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz", - "integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@eslint/core": "^0.17.0", - "levn": "^0.4.1" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - } - }, - "node_modules/@humanfs/core": { - "version": "0.19.1", - "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", - "integrity": "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=18.18.0" - } - }, - "node_modules/@humanfs/node": { - "version": "0.16.7", - "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.7.tgz", - "integrity": "sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@humanfs/core": "^0.19.1", - "@humanwhocodes/retry": "^0.4.0" - }, - "engines": { - "node": ">=18.18.0" - } - }, - "node_modules/@humanwhocodes/module-importer": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", - "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=12.22" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/nzakas" - } - }, - "node_modules/@humanwhocodes/retry": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", - "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=18.18" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/nzakas" - } - }, - "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.13", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", - "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.0", - "@jridgewell/trace-mapping": "^0.3.24" - } - }, - "node_modules/@jridgewell/remapping": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", - "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.24" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", - "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.31", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", - "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.1.0", - "@jridgewell/sourcemap-codec": "^1.4.14" - } - }, - "node_modules/@mapbox/jsonlint-lines-primitives": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/@mapbox/jsonlint-lines-primitives/-/jsonlint-lines-primitives-2.0.2.tgz", - "integrity": "sha512-rY0o9A5ECsTQRVhv7tL/OyDpGAoUB4tTvLiW1DSzQGq4bvTPhNw1VpSNjDJc5GFZ2XuyOtSWSVN05qOtcD71qQ==", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/@mapbox/point-geometry": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@mapbox/point-geometry/-/point-geometry-1.1.0.tgz", - "integrity": "sha512-YGcBz1cg4ATXDCM/71L9xveh4dynfGmcLDqufR+nQQy3fKwsAZsWd/x4621/6uJaeB9mwOHE6hPeDgXz9uViUQ==", - "license": "ISC" - }, - "node_modules/@mapbox/tiny-sdf": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/@mapbox/tiny-sdf/-/tiny-sdf-2.0.7.tgz", - "integrity": "sha512-25gQLQMcpivjOSA40g3gO6qgiFPDpWRoMfd+G/GoppPIeP6JDaMMkMrEJnMZhKyyS6iKwVt5YKu02vCUyJM3Ug==", - "license": "BSD-2-Clause" - }, - "node_modules/@mapbox/unitbezier": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/@mapbox/unitbezier/-/unitbezier-0.0.1.tgz", - "integrity": "sha512-nMkuDXFv60aBr9soUG5q+GvZYL+2KZHVvsqFCzqnkGEf46U2fvmytHaEVc1/YZbiLn8X+eR3QzX1+dwDO1lxlw==", - "license": "BSD-2-Clause" - }, - "node_modules/@mapbox/vector-tile": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@mapbox/vector-tile/-/vector-tile-2.0.4.tgz", - "integrity": "sha512-AkOLcbgGTdXScosBWwmmD7cDlvOjkg/DetGva26pIRiZPdeJYjYKarIlb4uxVzi6bwHO6EWH82eZ5Nuv4T5DUg==", - "license": "BSD-3-Clause", - "dependencies": { - "@mapbox/point-geometry": "~1.1.0", - "@types/geojson": "^7946.0.16", - "pbf": "^4.0.1" - } - }, - "node_modules/@mapbox/whoots-js": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@mapbox/whoots-js/-/whoots-js-3.1.0.tgz", - "integrity": "sha512-Es6WcD0nO5l+2BOQS4uLfNPYQaNDfbot3X1XUoloz+x0mPDS3eeORZJl06HXjwBG1fOGwCRnzK88LMdxKRrd6Q==", - "license": "ISC", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@maplibre/geojson-vt": { - "version": "6.0.4", - "resolved": "https://registry.npmjs.org/@maplibre/geojson-vt/-/geojson-vt-6.0.4.tgz", - "integrity": "sha512-HYv3POhMRCdhP3UPPATM/hfcy6/WuVIf5FKboH8u/ZuFMTnAIcSVlq5nfOqroLokd925w2QtE7YwquFOIacwVQ==", - "license": "ISC", - "dependencies": { - "kdbush": "^4.0.2" - } - }, - "node_modules/@maplibre/maplibre-gl-style-spec": { - "version": "24.8.1", - "resolved": "https://registry.npmjs.org/@maplibre/maplibre-gl-style-spec/-/maplibre-gl-style-spec-24.8.1.tgz", - "integrity": "sha512-zxa92qF96ZNojLxeAjnaRpjVCy+swoUNJvDhtpC90k7u5F0TMr4GmvNqMKvYrMoPB8d7gRSXbMG1hBbmgESIsw==", - "license": "ISC", - "dependencies": { - "@mapbox/jsonlint-lines-primitives": "~2.0.2", - "@mapbox/unitbezier": "^0.0.1", - "json-stringify-pretty-compact": "^4.0.0", - "minimist": "^1.2.8", - "quickselect": "^3.0.0", - "rw": "^1.3.3", - "tinyqueue": "^3.0.0" - }, - "bin": { - "gl-style-format": "dist/gl-style-format.mjs", - "gl-style-migrate": "dist/gl-style-migrate.mjs", - "gl-style-validate": "dist/gl-style-validate.mjs" - } - }, - "node_modules/@maplibre/mlt": { - "version": "1.1.8", - "resolved": "https://registry.npmjs.org/@maplibre/mlt/-/mlt-1.1.8.tgz", - "integrity": "sha512-8vtfYGidr1rNkv5IwIoU2lfe3Oy+Wa8HluzQYcQi9cveU9K3pweAal/poQj4GJ0K/EW4bTQp2wVAs09g2yDRZg==", - "license": "(MIT OR Apache-2.0)", - "dependencies": { - "@mapbox/point-geometry": "^1.1.0" - } - }, - "node_modules/@maplibre/vt-pbf": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/@maplibre/vt-pbf/-/vt-pbf-4.3.0.tgz", - "integrity": "sha512-jIvp8F5hQCcreqOOpEt42TJMUlsrEcpf/kI1T2v85YrQRV6PPXUcEXUg5karKtH6oh47XJZ4kHu56pUkOuqA7w==", - "license": "MIT", - "dependencies": { - "@mapbox/point-geometry": "^1.1.0", - "@mapbox/vector-tile": "^2.0.4", - "@maplibre/geojson-vt": "^5.0.4", - "@types/geojson": "^7946.0.16", - "@types/supercluster": "^7.1.3", - "pbf": "^4.0.1", - "supercluster": "^8.0.1" - } - }, - "node_modules/@maplibre/vt-pbf/node_modules/@maplibre/geojson-vt": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/@maplibre/geojson-vt/-/geojson-vt-5.0.4.tgz", - "integrity": "sha512-KGg9sma45S+stfH9vPCJk1J0lSDLWZgCT9Y8u8qWZJyjFlP8MNP1WGTxIMYJZjDvVT3PDn05kN1C95Sut1HpgQ==", - "license": "ISC" - }, - "node_modules/@napi-rs/wasm-runtime": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.2.tgz", - "integrity": "sha512-sNXv5oLJ7ob93xkZ1XnxisYhGYXfaG9f65/ZgYuAu3qt7b3NadcOEhLvx28hv31PgX8SZJRYrAIPQilQmFpLVw==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@tybys/wasm-util": "^0.10.1" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Brooooooklyn" - }, - "peerDependencies": { - "@emnapi/core": "^1.7.1", - "@emnapi/runtime": "^1.7.1" - } - }, - "node_modules/@oxc-project/types": { - "version": "0.122.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.122.0.tgz", - "integrity": "sha512-oLAl5kBpV4w69UtFZ9xqcmTi+GENWOcPF7FCrczTiBbmC0ibXxCwyvZGbO39rCVEuLGAZM84DH0pUIyyv/YJzA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/Boshen" - } - }, - "node_modules/@rolldown/binding-android-arm64": { - "version": "1.0.0-rc.12", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.12.tgz", - "integrity": "sha512-pv1y2Fv0JybcykuiiD3qBOBdz6RteYojRFY1d+b95WVuzx211CRh+ytI/+9iVyWQ6koTh5dawe4S/yRfOFjgaA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.0.0-rc.12", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.0-rc.12.tgz", - "integrity": "sha512-cFYr6zTG/3PXXF3pUO+umXxt1wkRK/0AYT8lDwuqvRC+LuKYWSAQAQZjCWDQpAH172ZV6ieYrNnFzVVcnSflAg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-darwin-x64": { - "version": "1.0.0-rc.12", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.0-rc.12.tgz", - "integrity": "sha512-ZCsYknnHzeXYps0lGBz8JrF37GpE9bFVefrlmDrAQhOEi4IOIlcoU1+FwHEtyXGx2VkYAvhu7dyBf75EJQffBw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.0.0-rc.12", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.0-rc.12.tgz", - "integrity": "sha512-dMLeprcVsyJsKolRXyoTH3NL6qtsT0Y2xeuEA8WQJquWFXkEC4bcu1rLZZSnZRMtAqwtrF/Ib9Ddtpa/Gkge9Q==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.0.0-rc.12", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.0-rc.12.tgz", - "integrity": "sha512-YqWjAgGC/9M1lz3GR1r1rP79nMgo3mQiiA+Hfo+pvKFK1fAJ1bCi0ZQVh8noOqNacuY1qIcfyVfP6HoyBRZ85Q==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.0.0-rc.12", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.0-rc.12.tgz", - "integrity": "sha512-/I5AS4cIroLpslsmzXfwbe5OmWvSsrFuEw3mwvbQ1kDxJ822hFHIx+vsN/TAzNVyepI/j/GSzrtCIwQPeKCLIg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.0.0-rc.12", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.0-rc.12.tgz", - "integrity": "sha512-V6/wZztnBqlx5hJQqNWwFdxIKN0m38p8Jas+VoSfgH54HSj9tKTt1dZvG6JRHcjh6D7TvrJPWFGaY9UBVOaWPw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-ppc64-gnu": { - "version": "1.0.0-rc.12", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.0-rc.12.tgz", - "integrity": "sha512-AP3E9BpcUYliZCxa3w5Kwj9OtEVDYK6sVoUzy4vTOJsjPOgdaJZKFmN4oOlX0Wp0RPV2ETfmIra9x1xuayFB7g==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-s390x-gnu": { - "version": "1.0.0-rc.12", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.0-rc.12.tgz", - "integrity": "sha512-nWwpvUSPkoFmZo0kQazZYOrT7J5DGOJ/+QHHzjvNlooDZED8oH82Yg67HvehPPLAg5fUff7TfWFHQS8IV1n3og==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.0.0-rc.12", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.0-rc.12.tgz", - "integrity": "sha512-RNrafz5bcwRy+O9e6P8Z/OCAJW/A+qtBczIqVYwTs14pf4iV1/+eKEjdOUta93q2TsT/FI0XYDP3TCky38LMAg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.0.0-rc.12", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.0-rc.12.tgz", - "integrity": "sha512-Jpw/0iwoKWx3LJ2rc1yjFrj+T7iHZn2JDg1Yny1ma0luviFS4mhAIcd1LFNxK3EYu3DHWCps0ydXQ5i/rrJ2ig==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.0.0-rc.12", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.0-rc.12.tgz", - "integrity": "sha512-vRugONE4yMfVn0+7lUKdKvN4D5YusEiPilaoO2sgUWpCvrncvWgPMzK00ZFFJuiPgLwgFNP5eSiUlv2tfc+lpA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-wasm32-wasi": { - "version": "1.0.0-rc.12", - "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.0-rc.12.tgz", - "integrity": "sha512-ykGiLr/6kkiHc0XnBfmFJuCjr5ZYKKofkx+chJWDjitX+KsJuAmrzWhwyOMSHzPhzOHOy7u9HlFoa5MoAOJ/Zg==", - "cpu": [ - "wasm32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@napi-rs/wasm-runtime": "^1.1.1" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.0.0-rc.12", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.0-rc.12.tgz", - "integrity": "sha512-5eOND4duWkwx1AzCxadcOrNeighiLwMInEADT0YM7xeEOOFcovWZCq8dadXgcRHSf3Ulh1kFo/qvzoFiCLOL1Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.0.0-rc.12", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.0-rc.12.tgz", - "integrity": "sha512-PyqoipaswDLAZtot351MLhrlrh6lcZPo2LSYE+VDxbVk24LVKAGOuE4hb8xZQmrPAuEtTZW8E6D2zc5EUZX4Lw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/pluginutils": { - "version": "1.0.0-rc.7", - "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.7.tgz", - "integrity": "sha512-qujRfC8sFVInYSPPMLQByRh7zhwkGFS4+tyMQ83srV1qrxL4g8E2tyxVVyxd0+8QeBM1mIk9KbWxkegRr76XzA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@tybys/wasm-util": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", - "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@types/estree": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", - "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/geojson": { - "version": "7946.0.16", - "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz", - "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", - "license": "MIT" - }, - "node_modules/@types/json-schema": { - "version": "7.0.15", - "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", - "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/node": { - "version": "24.12.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.1.tgz", - "integrity": "sha512-v6Ct1W1Fdz7xg5jYCg4FTrbNcIqzds2jv/HL6+5Rs/Cyjf0oljAgW59zvDZXyYG7nt9MLrAFJv9erP/fLjwt+g==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~7.16.0" - } - }, - "node_modules/@types/react": { - "version": "19.2.14", - "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", - "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", - "dev": true, - "license": "MIT", - "dependencies": { - "csstype": "^3.2.2" - } - }, - "node_modules/@types/react-dom": { - "version": "19.2.3", - "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.3.tgz", - "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", - "dev": true, - "license": "MIT", - "peerDependencies": { - "@types/react": "^19.2.0" - } - }, - "node_modules/@types/supercluster": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/@types/supercluster/-/supercluster-7.1.3.tgz", - "integrity": "sha512-Z0pOY34GDFl3Q6hUFYf3HkTwKEE02e7QgtJppBt+beEAxnyOpJua+voGFvxINBHa06GwLFFym7gRPY2SiKIfIA==", - "license": "MIT", - "dependencies": { - "@types/geojson": "*" - } - }, - "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.58.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.58.0.tgz", - "integrity": "sha512-RLkVSiNuUP1C2ROIWfqX+YcUfLaSnxGE/8M+Y57lopVwg9VTYYfhuz15Yf1IzCKgZj6/rIbYTmJCUSqr76r0Wg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@eslint-community/regexpp": "^4.12.2", - "@typescript-eslint/scope-manager": "8.58.0", - "@typescript-eslint/type-utils": "8.58.0", - "@typescript-eslint/utils": "8.58.0", - "@typescript-eslint/visitor-keys": "8.58.0", - "ignore": "^7.0.5", - "natural-compare": "^1.4.0", - "ts-api-utils": "^2.5.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "@typescript-eslint/parser": "^8.58.0", - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { - "version": "7.0.5", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.5.tgz", - "integrity": "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, - "node_modules/@typescript-eslint/parser": { - "version": "8.58.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.58.0.tgz", - "integrity": "sha512-rLoGZIf9afaRBYsPUMtvkDWykwXwUPL60HebR4JgTI8mxfFe2cQTu3AGitANp4b9B2QlVru6WzjgB2IzJKiCSA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/scope-manager": "8.58.0", - "@typescript-eslint/types": "8.58.0", - "@typescript-eslint/typescript-estree": "8.58.0", - "@typescript-eslint/visitor-keys": "8.58.0", - "debug": "^4.4.3" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/project-service": { - "version": "8.58.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.58.0.tgz", - "integrity": "sha512-8Q/wBPWLQP1j16NxoPNIKpDZFMaxl7yWIoqXWYeWO+Bbd2mjgvoF0dxP2jKZg5+x49rgKdf7Ck473M8PC3V9lg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/tsconfig-utils": "^8.58.0", - "@typescript-eslint/types": "^8.58.0", - "debug": "^4.4.3" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/scope-manager": { - "version": "8.58.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.58.0.tgz", - "integrity": "sha512-W1Lur1oF50FxSnNdGp3Vs6P+yBRSmZiw4IIjEeYxd8UQJwhUF0gDgDD/W/Tgmh73mxgEU3qX0Bzdl/NGuSPEpQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/types": "8.58.0", - "@typescript-eslint/visitor-keys": "8.58.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/tsconfig-utils": { - "version": "8.58.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.58.0.tgz", - "integrity": "sha512-doNSZEVJsWEu4htiVC+PR6NpM+pa+a4ClH9INRWOWCUzMst/VA9c4gXq92F8GUD1rwhNvRLkgjfYtFXegXQF7A==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/type-utils": { - "version": "8.58.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.58.0.tgz", - "integrity": "sha512-aGsCQImkDIqMyx1u4PrVlbi/krmDsQUs4zAcCV6M7yPcPev+RqVlndsJy9kJ8TLihW9TZ0kbDAzctpLn5o+lOg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/types": "8.58.0", - "@typescript-eslint/typescript-estree": "8.58.0", - "@typescript-eslint/utils": "8.58.0", - "debug": "^4.4.3", - "ts-api-utils": "^2.5.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/types": { - "version": "8.58.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.58.0.tgz", - "integrity": "sha512-O9CjxypDT89fbHxRfETNoAnHj/i6IpRK0CvbVN3qibxlLdo5p5hcLmUuCCrHMpxiWSwKyI8mCP7qRNYuOJ0Uww==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.58.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.58.0.tgz", - "integrity": "sha512-7vv5UWbHqew/dvs+D3e1RvLv1v2eeZ9txRHPnEEBUgSNLx5ghdzjHa0sgLWYVKssH+lYmV0JaWdoubo0ncGYLA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/project-service": "8.58.0", - "@typescript-eslint/tsconfig-utils": "8.58.0", - "@typescript-eslint/types": "8.58.0", - "@typescript-eslint/visitor-keys": "8.58.0", - "debug": "^4.4.3", - "minimatch": "^10.2.2", - "semver": "^7.7.3", - "tinyglobby": "^0.2.15", - "ts-api-utils": "^2.5.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/balanced-match": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", - "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", - "dev": true, - "license": "MIT", - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", - "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^4.0.2" - }, - "engines": { - "node": "18 || 20 || >=22" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", - "dev": true, - "license": "BlueOak-1.0.0", - "dependencies": { - "brace-expansion": "^5.0.5" - }, - "engines": { - "node": "18 || 20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/@typescript-eslint/utils": { - "version": "8.58.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.58.0.tgz", - "integrity": "sha512-RfeSqcFeHMHlAWzt4TBjWOAtoW9lnsAGiP3GbaX9uVgTYYrMbVnGONEfUCiSss+xMHFl+eHZiipmA8WkQ7FuNA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@eslint-community/eslint-utils": "^4.9.1", - "@typescript-eslint/scope-manager": "8.58.0", - "@typescript-eslint/types": "8.58.0", - "@typescript-eslint/typescript-estree": "8.58.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.58.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.58.0.tgz", - "integrity": "sha512-XJ9UD9+bbDo4a4epraTwG3TsNPeiB9aShrUneAVXy8q4LuwowN+qu89/6ByLMINqvIMeI9H9hOHQtg/ijrYXzQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/types": "8.58.0", - "eslint-visitor-keys": "^5.0.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", - "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=24" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/@vitejs/plugin-react": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.0.1.tgz", - "integrity": "sha512-l9X/E3cDb+xY3SWzlG1MOGt2usfEHGMNIaegaUGFsLkb3RCn/k8/TOXBcab+OndDI4TBtktT8/9BwwW8Vi9KUQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@rolldown/pluginutils": "1.0.0-rc.7" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - }, - "peerDependencies": { - "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", - "babel-plugin-react-compiler": "^1.0.0", - "vite": "^8.0.0" - }, - "peerDependenciesMeta": { - "@rolldown/plugin-babel": { - "optional": true - }, - "babel-plugin-react-compiler": { - "optional": true - } - } - }, - "node_modules/acorn": { - "version": "8.16.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", - "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", - "dev": true, - "license": "MIT", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-jsx": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", - "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", - "dev": true, - "license": "MIT", - "peerDependencies": { - "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" - } - }, - "node_modules/ajv": { - "version": "6.14.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.14.0.tgz", - "integrity": "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, - "license": "Python-2.0" - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true, - "license": "MIT" - }, - "node_modules/baseline-browser-mapping": { - "version": "2.10.13", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.13.tgz", - "integrity": "sha512-BL2sTuHOdy0YT1lYieUxTw/QMtPBC3pmlJC6xk8BBYVv6vcw3SGdKemQ+Xsx9ik2F/lYDO9tqsFQH1r9PFuHKw==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "baseline-browser-mapping": "dist/cli.cjs" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/brace-expansion": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.13.tgz", - "integrity": "sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/browserslist": { - "version": "4.28.2", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.2.tgz", - "integrity": "sha512-48xSriZYYg+8qXna9kwqjIVzuQxi+KYWp2+5nCYnYKPTr0LvD89Jqk2Or5ogxz0NUMfIjhh2lIUX/LyX9B4oIg==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "baseline-browser-mapping": "^2.10.12", - "caniuse-lite": "^1.0.30001782", - "electron-to-chromium": "^1.5.328", - "node-releases": "^2.0.36", - "update-browserslist-db": "^1.2.3" - }, - "bin": { - "browserslist": "cli.js" - }, - "engines": { - "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" - } - }, - "node_modules/callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/caniuse-lite": { - "version": "1.0.30001784", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001784.tgz", - "integrity": "sha512-WU346nBTklUV9YfUl60fqRbU5ZqyXlqvo1SgigE1OAXK5bFL8LL9q1K7aap3N739l4BvNqnkm3YrGHiY9sfUQw==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/caniuse-lite" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "CC-BY-4.0" - }, - "node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, - "license": "MIT" - }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, - "license": "MIT" - }, - "node_modules/convert-source-map": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", - "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", - "dev": true, - "license": "MIT" - }, - "node_modules/cross-spawn": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", - "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "dev": true, - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/csstype": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", - "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/deep-is": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", - "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/detect-libc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", - "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=8" - } - }, - "node_modules/earcut": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/earcut/-/earcut-3.0.2.tgz", - "integrity": "sha512-X7hshQbLyMJ/3RPhyObLARM2sNxxmRALLKx1+NVFFnQ9gKzmCrxm9+uLIAdBcvc8FNLpctqlQ2V6AE92Ol9UDQ==", - "license": "ISC" - }, - "node_modules/electron-to-chromium": { - "version": "1.5.331", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.331.tgz", - "integrity": "sha512-IbxXrsTlD3hRodkLnbxAPP4OuJYdWCeM3IOdT+CpcMoIwIoDfCmRpEtSPfwBXxVkg9xmBeY7Lz2Eo2TDn/HC3Q==", - "dev": true, - "license": "ISC" - }, - "node_modules/escalade": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", - "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/escape-string-regexp": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/eslint": { - "version": "9.39.4", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.4.tgz", - "integrity": "sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@eslint-community/eslint-utils": "^4.8.0", - "@eslint-community/regexpp": "^4.12.1", - "@eslint/config-array": "^0.21.2", - "@eslint/config-helpers": "^0.4.2", - "@eslint/core": "^0.17.0", - "@eslint/eslintrc": "^3.3.5", - "@eslint/js": "9.39.4", - "@eslint/plugin-kit": "^0.4.1", - "@humanfs/node": "^0.16.6", - "@humanwhocodes/module-importer": "^1.0.1", - "@humanwhocodes/retry": "^0.4.2", - "@types/estree": "^1.0.6", - "ajv": "^6.14.0", - "chalk": "^4.0.0", - "cross-spawn": "^7.0.6", - "debug": "^4.3.2", - "escape-string-regexp": "^4.0.0", - "eslint-scope": "^8.4.0", - "eslint-visitor-keys": "^4.2.1", - "espree": "^10.4.0", - "esquery": "^1.5.0", - "esutils": "^2.0.2", - "fast-deep-equal": "^3.1.3", - "file-entry-cache": "^8.0.0", - "find-up": "^5.0.0", - "glob-parent": "^6.0.2", - "ignore": "^5.2.0", - "imurmurhash": "^0.1.4", - "is-glob": "^4.0.0", - "json-stable-stringify-without-jsonify": "^1.0.1", - "lodash.merge": "^4.6.2", - "minimatch": "^3.1.5", - "natural-compare": "^1.4.0", - "optionator": "^0.9.3" - }, - "bin": { - "eslint": "bin/eslint.js" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://eslint.org/donate" - }, - "peerDependencies": { - "jiti": "*" - }, - "peerDependenciesMeta": { - "jiti": { - "optional": true - } - } - }, - "node_modules/eslint-plugin-react-hooks": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-7.0.1.tgz", - "integrity": "sha512-O0d0m04evaNzEPoSW+59Mezf8Qt0InfgGIBJnpC0h3NH/WjUAR7BIKUfysC6todmtiZ/A0oUVS8Gce0WhBrHsA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.24.4", - "@babel/parser": "^7.24.4", - "hermes-parser": "^0.25.1", - "zod": "^3.25.0 || ^4.0.0", - "zod-validation-error": "^3.5.0 || ^4.0.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0" - } - }, - "node_modules/eslint-plugin-react-refresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/eslint-plugin-react-refresh/-/eslint-plugin-react-refresh-0.5.2.tgz", - "integrity": "sha512-hmgTH57GfzoTFjVN0yBwTggnsVUF2tcqi7RJZHqi9lIezSs4eFyAMktA68YD4r5kNw1mxyY4dmkyoFDb3FIqrA==", - "dev": true, - "license": "MIT", - "peerDependencies": { - "eslint": "^9 || ^10" - } - }, - "node_modules/eslint-scope": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", - "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "esrecurse": "^4.3.0", - "estraverse": "^5.2.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/eslint-visitor-keys": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", - "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/espree": { - "version": "10.4.0", - "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", - "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "acorn": "^8.15.0", - "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^4.2.1" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/esquery": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", - "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "estraverse": "^5.1.0" - }, - "engines": { - "node": ">=0.10" - } - }, - "node_modules/esrecurse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", - "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "estraverse": "^5.2.0" - }, - "engines": { - "node": ">=4.0" - } - }, - "node_modules/estraverse": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/esutils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", - "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true, - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/fast-json-stable-stringify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true, - "license": "MIT" - }, - "node_modules/fast-levenshtein": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", - "dev": true, - "license": "MIT" - }, - "node_modules/fdir": { - "version": "6.5.0", - "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", - "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12.0.0" - }, - "peerDependencies": { - "picomatch": "^3 || ^4" - }, - "peerDependenciesMeta": { - "picomatch": { - "optional": true - } - } - }, - "node_modules/file-entry-cache": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", - "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "flat-cache": "^4.0.0" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/find-up": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", - "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^6.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/flat-cache": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", - "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", - "dev": true, - "license": "MIT", - "dependencies": { - "flatted": "^3.2.9", - "keyv": "^4.5.4" - }, - "engines": { - "node": ">=16" - } - }, - "node_modules/flatted": { - "version": "3.4.2", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.2.tgz", - "integrity": "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==", - "dev": true, - "license": "ISC" - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/gensync": { - "version": "1.0.0-beta.2", - "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", - "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/gl-matrix": { - "version": "3.4.4", - "resolved": "https://registry.npmjs.org/gl-matrix/-/gl-matrix-3.4.4.tgz", - "integrity": "sha512-latSnyDNt/8zYUB6VIJ6PCh2jBjJX6gnDsoCZ7LyW7GkqrD51EWwa9qCoGixj8YqBtETQK/xY7OmpTF8xz1DdQ==", - "license": "MIT" - }, - "node_modules/glob-parent": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", - "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", - "dev": true, - "license": "ISC", - "dependencies": { - "is-glob": "^4.0.3" - }, - "engines": { - "node": ">=10.13.0" - } - }, - "node_modules/globals": { - "version": "17.4.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-17.4.0.tgz", - "integrity": "sha512-hjrNztw/VajQwOLsMNT1cbJiH2muO3OROCHnbehc8eY5JyD2gqz4AcMHPqgaOR59DjgUjYAYLeH699g/eWi2jw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/hermes-estree": { - "version": "0.25.1", - "resolved": "https://registry.npmjs.org/hermes-estree/-/hermes-estree-0.25.1.tgz", - "integrity": "sha512-0wUoCcLp+5Ev5pDW2OriHC2MJCbwLwuRx+gAqMTOkGKJJiBCLjtrvy4PWUGn6MIVefecRpzoOZ/UV6iGdOr+Cw==", - "dev": true, - "license": "MIT" - }, - "node_modules/hermes-parser": { - "version": "0.25.1", - "resolved": "https://registry.npmjs.org/hermes-parser/-/hermes-parser-0.25.1.tgz", - "integrity": "sha512-6pEjquH3rqaI6cYAXYPcz9MS4rY6R4ngRgrgfDshRptUZIc3lw0MCIJIGDj9++mfySOuPTHB4nrSW99BCvOPIA==", - "dev": true, - "license": "MIT", - "dependencies": { - "hermes-estree": "0.25.1" - } - }, - "node_modules/ignore": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", - "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, - "node_modules/import-fresh": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", - "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "parent-module": "^1.0.0", - "resolve-from": "^4.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/imurmurhash": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", - "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.8.19" - } - }, - "node_modules/is-extglob": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", - "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-glob": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", - "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-extglob": "^2.1.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true, - "license": "ISC" - }, - "node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/js-yaml": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", - "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/jsesc": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", - "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", - "dev": true, - "license": "MIT", - "bin": { - "jsesc": "bin/jsesc" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/json-buffer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", - "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true, - "license": "MIT" - }, - "node_modules/json-stable-stringify-without-jsonify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", - "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", - "dev": true, - "license": "MIT" - }, - "node_modules/json-stringify-pretty-compact": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/json-stringify-pretty-compact/-/json-stringify-pretty-compact-4.0.0.tgz", - "integrity": "sha512-3CNZ2DnrpByG9Nqj6Xo8vqbjT4F6N+tb4Gb28ESAZjYZ5yqvmc56J+/kuIwkaAMOyblTQhUW7PxMkUb8Q36N3Q==", - "license": "MIT" - }, - "node_modules/json5": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", - "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", - "dev": true, - "license": "MIT", - "bin": { - "json5": "lib/cli.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/kdbush": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/kdbush/-/kdbush-4.0.2.tgz", - "integrity": "sha512-WbCVYJ27Sz8zi9Q7Q0xHC+05iwkm3Znipc2XTlrnJbsHMYktW4hPhXUE8Ys1engBrvffoSCqbil1JQAa7clRpA==", - "license": "ISC" - }, - "node_modules/keyv": { - "version": "4.5.4", - "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", - "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", - "dev": true, - "license": "MIT", - "dependencies": { - "json-buffer": "3.0.1" - } - }, - "node_modules/levn": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", - "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "prelude-ls": "^1.2.1", - "type-check": "~0.4.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/lightningcss": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", - "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", - "dev": true, - "license": "MPL-2.0", - "dependencies": { - "detect-libc": "^2.0.3" - }, - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - }, - "optionalDependencies": { - "lightningcss-android-arm64": "1.32.0", - "lightningcss-darwin-arm64": "1.32.0", - "lightningcss-darwin-x64": "1.32.0", - "lightningcss-freebsd-x64": "1.32.0", - "lightningcss-linux-arm-gnueabihf": "1.32.0", - "lightningcss-linux-arm64-gnu": "1.32.0", - "lightningcss-linux-arm64-musl": "1.32.0", - "lightningcss-linux-x64-gnu": "1.32.0", - "lightningcss-linux-x64-musl": "1.32.0", - "lightningcss-win32-arm64-msvc": "1.32.0", - "lightningcss-win32-x64-msvc": "1.32.0" - } - }, - "node_modules/lightningcss-android-arm64": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", - "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-darwin-arm64": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", - "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-darwin-x64": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", - "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-freebsd-x64": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", - "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-arm-gnueabihf": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", - "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-arm64-gnu": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", - "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-arm64-musl": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", - "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-x64-gnu": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", - "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-x64-musl": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", - "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-win32-arm64-msvc": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", - "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-win32-x64-msvc": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", - "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MPL-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/locate-path": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", - "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^5.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/lodash.merge": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", - "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/lru-cache": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", - "dev": true, - "license": "ISC", - "dependencies": { - "yallist": "^3.0.2" - } - }, - "node_modules/maplibre-gl": { - "version": "5.21.1", - "resolved": "https://registry.npmjs.org/maplibre-gl/-/maplibre-gl-5.21.1.tgz", - "integrity": "sha512-zto1RTnFkOpOO1bm93ElCXF1huey2N4LvXaGLMFcYAu9txh0OhGIdX1q3LZLkrMKgMxMeYduaQo+DVNzg098fg==", - "license": "BSD-3-Clause", - "dependencies": { - "@mapbox/jsonlint-lines-primitives": "^2.0.2", - "@mapbox/point-geometry": "^1.1.0", - "@mapbox/tiny-sdf": "^2.0.7", - "@mapbox/unitbezier": "^0.0.1", - "@mapbox/vector-tile": "^2.0.4", - "@mapbox/whoots-js": "^3.1.0", - "@maplibre/geojson-vt": "^6.0.4", - "@maplibre/maplibre-gl-style-spec": "^24.7.0", - "@maplibre/mlt": "^1.1.8", - "@maplibre/vt-pbf": "^4.3.0", - "@types/geojson": "^7946.0.16", - "earcut": "^3.0.2", - "gl-matrix": "^3.4.4", - "kdbush": "^4.0.2", - "murmurhash-js": "^1.0.0", - "pbf": "^4.0.1", - "potpack": "^2.1.0", - "quickselect": "^3.0.0", - "tinyqueue": "^3.0.0" - }, - "engines": { - "node": ">=16.14.0", - "npm": ">=8.1.0" - }, - "funding": { - "url": "https://github.com/maplibre/maplibre-gl-js?sponsor=1" - } - }, - "node_modules/minimatch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", - "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/minimist": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, - "node_modules/murmurhash-js": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/murmurhash-js/-/murmurhash-js-1.0.0.tgz", - "integrity": "sha512-TvmkNhkv8yct0SVBSy+o8wYzXjE4Zz3PCesbfs8HiCXXdcTuocApFv11UWlNFWKYsP2okqrhb7JNlSm9InBhIw==", - "license": "MIT" - }, - "node_modules/nanoid": { - "version": "3.3.11", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", - "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, - "node_modules/natural-compare": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", - "dev": true, - "license": "MIT" - }, - "node_modules/node-releases": { - "version": "2.0.37", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.37.tgz", - "integrity": "sha512-1h5gKZCF+pO/o3Iqt5Jp7wc9rH3eJJ0+nh/CIoiRwjRxde/hAHyLPXYN4V3CqKAbiZPSeJFSWHmJsbkicta0Eg==", - "dev": true, - "license": "MIT" - }, - "node_modules/optionator": { - "version": "0.9.4", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", - "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", - "dev": true, - "license": "MIT", - "dependencies": { - "deep-is": "^0.1.3", - "fast-levenshtein": "^2.0.6", - "levn": "^0.4.1", - "prelude-ls": "^1.2.1", - "type-check": "^0.4.0", - "word-wrap": "^1.2.5" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "yocto-queue": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-locate": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", - "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^3.0.2" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/parent-module": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", - "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", - "dev": true, - "license": "MIT", - "dependencies": { - "callsites": "^3.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/pbf": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/pbf/-/pbf-4.0.1.tgz", - "integrity": "sha512-SuLdBvS42z33m8ejRbInMapQe8n0D3vN/Xd5fmWM3tufNgRQFBpaW2YVJxQZV4iPNqb0vEFvssMEo5w9c6BTIA==", - "license": "BSD-3-Clause", - "dependencies": { - "resolve-protobuf-schema": "^2.1.0" - }, - "bin": { - "pbf": "bin/pbf" - } - }, - "node_modules/picocolors": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true, - "license": "ISC" - }, - "node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/postcss": { - "version": "8.5.8", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz", - "integrity": "sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/postcss" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "nanoid": "^3.3.11", - "picocolors": "^1.1.1", - "source-map-js": "^1.2.1" - }, - "engines": { - "node": "^10 || ^12 || >=14" - } - }, - "node_modules/potpack": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/potpack/-/potpack-2.1.0.tgz", - "integrity": "sha512-pcaShQc1Shq0y+E7GqJqvZj8DTthWV1KeHGdi0Z6IAin2Oi3JnLCOfwnCo84qc+HAp52wT9nK9H7FAJp5a44GQ==", - "license": "ISC" - }, - "node_modules/prelude-ls": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", - "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/protocol-buffers-schema": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/protocol-buffers-schema/-/protocol-buffers-schema-3.6.0.tgz", - "integrity": "sha512-TdDRD+/QNdrCGCE7v8340QyuXd4kIWIgapsE2+n/SaGiSSbomYl4TjHlvIoCWRpE7wFt02EpB35VVA2ImcBVqw==", - "license": "MIT" - }, - "node_modules/punycode": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", - "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/quickselect": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/quickselect/-/quickselect-3.0.0.tgz", - "integrity": "sha512-XdjUArbK4Bm5fLLvlm5KpTFOiOThgfWWI4axAZDWg4E/0mKdZyI9tNEfds27qCi1ze/vwTR16kvmmGhRra3c2g==", - "license": "ISC" - }, - "node_modules/react": { - "version": "19.2.4", - "resolved": "https://registry.npmjs.org/react/-/react-19.2.4.tgz", - "integrity": "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/react-dom": { - "version": "19.2.4", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.4.tgz", - "integrity": "sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ==", - "license": "MIT", - "dependencies": { - "scheduler": "^0.27.0" - }, - "peerDependencies": { - "react": "^19.2.4" - } - }, - "node_modules/resolve-from": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", - "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/resolve-protobuf-schema": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/resolve-protobuf-schema/-/resolve-protobuf-schema-2.1.0.tgz", - "integrity": "sha512-kI5ffTiZWmJaS/huM8wZfEMer1eRd7oJQhDuxeCLe3t7N7mX3z94CN0xPxBQxFYQTSNz9T0i+v6inKqSdK8xrQ==", - "license": "MIT", - "dependencies": { - "protocol-buffers-schema": "^3.3.1" - } - }, - "node_modules/rolldown": { - "version": "1.0.0-rc.12", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.0-rc.12.tgz", - "integrity": "sha512-yP4USLIMYrwpPHEFB5JGH1uxhcslv6/hL0OyvTuY+3qlOSJvZ7ntYnoWpehBxufkgN0cvXxppuTu5hHa/zPh+A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@oxc-project/types": "=0.122.0", - "@rolldown/pluginutils": "1.0.0-rc.12" - }, - "bin": { - "rolldown": "bin/cli.mjs" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - }, - "optionalDependencies": { - "@rolldown/binding-android-arm64": "1.0.0-rc.12", - "@rolldown/binding-darwin-arm64": "1.0.0-rc.12", - "@rolldown/binding-darwin-x64": "1.0.0-rc.12", - "@rolldown/binding-freebsd-x64": "1.0.0-rc.12", - "@rolldown/binding-linux-arm-gnueabihf": "1.0.0-rc.12", - "@rolldown/binding-linux-arm64-gnu": "1.0.0-rc.12", - "@rolldown/binding-linux-arm64-musl": "1.0.0-rc.12", - "@rolldown/binding-linux-ppc64-gnu": "1.0.0-rc.12", - "@rolldown/binding-linux-s390x-gnu": "1.0.0-rc.12", - "@rolldown/binding-linux-x64-gnu": "1.0.0-rc.12", - "@rolldown/binding-linux-x64-musl": "1.0.0-rc.12", - "@rolldown/binding-openharmony-arm64": "1.0.0-rc.12", - "@rolldown/binding-wasm32-wasi": "1.0.0-rc.12", - "@rolldown/binding-win32-arm64-msvc": "1.0.0-rc.12", - "@rolldown/binding-win32-x64-msvc": "1.0.0-rc.12" - } - }, - "node_modules/rolldown/node_modules/@rolldown/pluginutils": { - "version": "1.0.0-rc.12", - "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.12.tgz", - "integrity": "sha512-HHMwmarRKvoFsJorqYlFeFRzXZqCt2ETQlEDOb9aqssrnVBB1/+xgTGtuTrIk5vzLNX1MjMtTf7W9z3tsSbrxw==", - "dev": true, - "license": "MIT" - }, - "node_modules/rw": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/rw/-/rw-1.3.3.tgz", - "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==", - "license": "BSD-3-Clause" - }, - "node_modules/scheduler": { - "version": "0.27.0", - "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", - "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", - "license": "MIT" - }, - "node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - } - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, - "license": "MIT", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/source-map-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/strip-json-comments": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", - "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/supercluster": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/supercluster/-/supercluster-8.0.1.tgz", - "integrity": "sha512-IiOea5kJ9iqzD2t7QJq/cREyLHTtSmUT6gQsweojg9WH2sYJqZK9SswTu6jrscO6D1G5v5vYZ9ru/eq85lXeZQ==", - "license": "ISC", - "dependencies": { - "kdbush": "^4.0.2" - } - }, - "node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/tinyglobby": { - "version": "0.2.15", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", - "integrity": "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "fdir": "^6.5.0", - "picomatch": "^4.0.3" - }, - "engines": { - "node": ">=12.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/SuperchupuDev" - } - }, - "node_modules/tinyqueue": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/tinyqueue/-/tinyqueue-3.0.0.tgz", - "integrity": "sha512-gRa9gwYU3ECmQYv3lslts5hxuIa90veaEcxDYuu3QGOIAEM2mOZkVHp48ANJuu1CURtRdHKUBY5Lm1tHV+sD4g==", - "license": "ISC" - }, - "node_modules/ts-api-utils": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz", - "integrity": "sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18.12" - }, - "peerDependencies": { - "typescript": ">=4.8.4" - } - }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, - "license": "0BSD", - "optional": true - }, - "node_modules/type-check": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", - "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", - "dev": true, - "license": "MIT", - "dependencies": { - "prelude-ls": "^1.2.1" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/typescript-eslint": { - "version": "8.58.0", - "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.58.0.tgz", - "integrity": "sha512-e2TQzKfaI85fO+F3QywtX+tCTsu/D3WW5LVU6nz8hTFKFZ8yBJ6mSYRpXqdR3mFjPWmO0eWsTa5f+UpAOe/FMA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@typescript-eslint/eslint-plugin": "8.58.0", - "@typescript-eslint/parser": "8.58.0", - "@typescript-eslint/typescript-estree": "8.58.0", - "@typescript-eslint/utils": "8.58.0" - }, - "engines": { - "node": "^18.18.0 || ^20.9.0 || >=21.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", - "typescript": ">=4.8.4 <6.1.0" - } - }, - "node_modules/undici-types": { - "version": "7.16.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", - "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", - "dev": true, - "license": "MIT" - }, - "node_modules/update-browserslist-db": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", - "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "escalade": "^3.2.0", - "picocolors": "^1.1.1" - }, - "bin": { - "update-browserslist-db": "cli.js" - }, - "peerDependencies": { - "browserslist": ">= 4.21.0" - } - }, - "node_modules/uri-js": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", - "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "punycode": "^2.1.0" - } - }, - "node_modules/vite": { - "version": "8.0.3", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.3.tgz", - "integrity": "sha512-B9ifbFudT1TFhfltfaIPgjo9Z3mDynBTJSUYxTjOQruf/zHH+ezCQKcoqO+h7a9Pw9Nm/OtlXAiGT1axBgwqrQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "lightningcss": "^1.32.0", - "picomatch": "^4.0.4", - "postcss": "^8.5.8", - "rolldown": "1.0.0-rc.12", - "tinyglobby": "^0.2.15" - }, - "bin": { - "vite": "bin/vite.js" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - }, - "funding": { - "url": "https://github.com/vitejs/vite?sponsor=1" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - }, - "peerDependencies": { - "@types/node": "^20.19.0 || >=22.12.0", - "@vitejs/devtools": "^0.1.0", - "esbuild": "^0.27.0", - "jiti": ">=1.21.0", - "less": "^4.0.0", - "sass": "^1.70.0", - "sass-embedded": "^1.70.0", - "stylus": ">=0.54.8", - "sugarss": "^5.0.0", - "terser": "^5.16.0", - "tsx": "^4.8.1", - "yaml": "^2.4.2" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - }, - "@vitejs/devtools": { - "optional": true - }, - "esbuild": { - "optional": true - }, - "jiti": { - "optional": true - }, - "less": { - "optional": true - }, - "sass": { - "optional": true - }, - "sass-embedded": { - "optional": true - }, - "stylus": { - "optional": true - }, - "sugarss": { - "optional": true - }, - "terser": { - "optional": true - }, - "tsx": { - "optional": true - }, - "yaml": { - "optional": true - } - } - }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, - "license": "ISC", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/word-wrap": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", - "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/yallist": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", - "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", - "dev": true, - "license": "ISC" - }, - "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/zod": { - "version": "4.3.6", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", - "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/zod-validation-error": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/zod-validation-error/-/zod-validation-error-4.0.2.tgz", - "integrity": "sha512-Q6/nZLe6jxuU80qb/4uJ4t5v2VEZ44lzQjPDhYJNztRQ4wyWc6VF3D3Kb/fAuPetZQnhS3hnajCf9CsWesghLQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18.0.0" - }, - "peerDependencies": { - "zod": "^3.25.0 || ^4.0.0" - } - } - } -} diff --git a/frontend/package.json b/frontend/package.json deleted file mode 100644 index b3dfcad..0000000 --- a/frontend/package.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "frontend", - "private": true, - "version": "0.0.0", - "type": "module", - "scripts": { - "dev": "vite", - "build": "tsc -b && vite build", - "lint": "eslint .", - "preview": "vite preview" - }, - "dependencies": { - "@types/geojson": "^7946.0.16", - "maplibre-gl": "^5.21.1", - "react": "^19.2.4", - "react-dom": "^19.2.4" - }, - "devDependencies": { - "@eslint/js": "^9.39.4", - "@types/node": "^24.12.0", - "@types/react": "^19.2.14", - "@types/react-dom": "^19.2.3", - "@vitejs/plugin-react": "^6.0.1", - "eslint": "^9.39.4", - "eslint-plugin-react-hooks": "^7.0.1", - "eslint-plugin-react-refresh": "^0.5.2", - "globals": "^17.4.0", - "typescript": "~5.9.3", - "typescript-eslint": "^8.57.0", - "vite": "^8.0.1" - } -} diff --git a/frontend/public/favicon.svg b/frontend/public/favicon.svg deleted file mode 100644 index 6893eb1..0000000 --- a/frontend/public/favicon.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/frontend/public/icons.svg b/frontend/public/icons.svg deleted file mode 100644 index e952219..0000000 --- a/frontend/public/icons.svg +++ /dev/null @@ -1,24 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/frontend/src/App.css b/frontend/src/App.css deleted file mode 100644 index f9c6a0e..0000000 --- a/frontend/src/App.css +++ /dev/null @@ -1,666 +0,0 @@ -.app-shell { - min-height: 100vh; - padding: 32px 32px 40px; - display: flex; - flex-direction: column; - gap: 24px; -} - -.app-shell--centered { - justify-content: center; - align-items: center; -} - -.hero-header { - display: flex; - justify-content: space-between; - align-items: flex-end; - gap: 24px; -} - -.hero-header h1 { - margin: 0; - max-width: 860px; -} - -.hero-header__eyebrow, -.loading-state__eyebrow, -.panel__eyebrow, -.map-stage__eyebrow { - margin: 0 0 10px; - text-transform: uppercase; - letter-spacing: 0.1em; - font-size: 0.78rem; - font-weight: 700; - color: #446257; -} - -.hero-header__copy { - margin: 12px 0 0; - max-width: 780px; - color: #53695f; - font-size: 1rem; -} - -.hero-header__status { - align-self: flex-start; - display: inline-flex; - align-items: center; - gap: 10px; - padding: 10px 14px; - border-radius: 999px; - background: rgba(255, 255, 255, 0.88); - border: 1px solid rgba(16, 38, 31, 0.08); - color: #18412e; - box-shadow: 0 8px 26px rgba(18, 44, 32, 0.08); - font-size: 0.92rem; -} - -.status-dot { - width: 10px; - height: 10px; - border-radius: 50%; - background: #34a853; - box-shadow: 0 0 0 6px rgba(52, 168, 83, 0.15); -} - -.map-stage, -.panel { - background: rgba(255, 255, 255, 0.84); - border: 1px solid rgba(16, 38, 31, 0.08); - border-radius: 28px; - box-shadow: 0 20px 60px rgba(18, 44, 32, 0.08); -} - -.map-stage { - padding: 18px 18px 14px; -} - -.map-stage__header { - display: flex; - justify-content: space-between; - align-items: flex-end; - gap: 20px; - margin-bottom: 16px; -} - -.map-stage__header h2, -.panel__title { - margin: 0; - font-size: 1.35rem; - color: #11281f; -} - -.map-stage__legend { - margin: 0; - max-width: 480px; - text-align: right; - color: #556b61; - font-size: 0.95rem; -} - -.simulation-map { - width: 100%; - height: min(76vh, 760px); - border-radius: 22px; - overflow: hidden; -} - -.bottom-layout { - display: grid; - grid-template-columns: minmax(320px, 0.9fr) minmax(420px, 1.1fr); - gap: 20px; -} - -.panel { - padding: 22px; -} - -.panel__header { - display: flex; - justify-content: space-between; - align-items: flex-start; - gap: 16px; - margin-bottom: 18px; -} - -.panel__meta, -.panel__status { - padding: 10px 14px; - border-radius: 16px; - background: #eef5f1; - color: #214336; - display: inline-flex; - flex-direction: column; - gap: 2px; - min-width: 110px; - text-align: right; - font-size: 0.9rem; -} - -.panel__status { - align-self: center; - min-width: auto; - font-weight: 700; -} - -.panel__copy { - margin: 0; - color: #5a6f65; -} - -.control-grid { - display: grid; - grid-template-columns: 1fr 1.2fr; - gap: 14px; - margin-bottom: 16px; -} - -.field { - display: flex; - flex-direction: column; - gap: 8px; -} - -.field__label { - font-size: 0.88rem; - font-weight: 600; - color: #375448; -} - -.field--summary { - opacity: 0.85; -} - -.field select, -.field input, -.run-button, -.path-row { - font: inherit; -} - -.field select, -.field input { - width: 100%; - box-sizing: border-box; - padding: 12px 14px; - border-radius: 16px; - border: 1px solid rgba(16, 38, 31, 0.12); - background: #f8fbf9; - color: #142d22; -} - -.field__value { - padding: 12px 14px; - border-radius: 16px; - border: 1px solid rgba(16, 38, 31, 0.08); - background: #f5faf7; - color: #173124; - font-weight: 600; -} - -.segmented-control { - display: grid; - grid-template-columns: 1fr 1fr; - gap: 8px; - grid-column: 1 / -1; -} - -.segmented-control--modes { - margin-bottom: 16px; -} - -.segmented-control button { - border: none; - border-radius: 999px; - padding: 10px 14px; - background: #edf2ef; - color: #395347; - font-weight: 600; - cursor: pointer; - transition: background 0.2s ease, color 0.2s ease, transform 0.2s ease; -} - -.segmented-control button.is-active { - background: #143828; - color: white; -} - -.slider-stack { - display: flex; - flex-direction: column; - gap: 14px; - margin-bottom: 18px; -} - -.slider-field { - display: flex; - flex-direction: column; - gap: 10px; -} - -.slider-field__header { - display: flex; - justify-content: space-between; - align-items: center; - color: #385549; - font-size: 0.92rem; -} - -.slider-field input[type='range'] { - width: 100%; - accent-color: #1d6a48; -} - -.control-footer { - display: flex; - justify-content: space-between; - align-items: center; - gap: 14px; -} - -.control-footer__meta { - display: flex; - flex-direction: column; - gap: 2px; - color: #577065; - font-size: 0.9rem; -} - -.run-button { - border: none; - border-radius: 999px; - padding: 14px 20px; - background: linear-gradient(135deg, #143828 0%, #2b7a55 100%); - color: white; - font-weight: 700; - cursor: pointer; - box-shadow: 0 12px 30px rgba(26, 93, 64, 0.25); -} - -.run-button:disabled { - opacity: 0.7; - cursor: progress; -} - -.route-summary { - display: grid; - grid-template-columns: 1fr 1fr; - gap: 12px; - margin-bottom: 16px; -} - -.route-summary__card { - padding: 14px 16px; - border-radius: 18px; - background: linear-gradient(135deg, rgba(18, 56, 40, 0.96), rgba(36, 99, 70, 0.92)); - color: white; -} - -.route-summary__card--muted { - background: linear-gradient(135deg, rgba(88, 101, 116, 0.94), rgba(112, 124, 138, 0.9)); -} - -.route-summary__label { - display: block; - margin-bottom: 8px; - font-size: 0.82rem; - opacity: 0.8; - text-transform: uppercase; - letter-spacing: 0.08em; -} - -.metric-grid { - display: grid; - grid-template-columns: repeat(3, minmax(0, 1fr)); - gap: 12px; - margin-bottom: 16px; -} - -.metric-grid--batch { - grid-template-columns: repeat(2, minmax(0, 1fr)); -} - -.metric-card { - padding: 14px 16px; - border-radius: 18px; - background: #f6faf7; - border: 1px solid rgba(16, 38, 31, 0.08); -} - -.metric-card__label { - display: block; - margin-bottom: 8px; - color: #5d7268; - font-size: 0.82rem; - text-transform: uppercase; - letter-spacing: 0.08em; -} - -.metric-card strong { - display: block; - font-size: 1.35rem; - color: #133022; - margin-bottom: 6px; -} - -.metric-card p { - margin: 0; - color: #5b6f66; - font-size: 0.88rem; -} - -.insight-callout { - padding: 16px 18px; - border-radius: 22px; - background: linear-gradient(135deg, #143828 0%, #1b523a 100%); - color: white; - margin-bottom: 18px; -} - -.insight-callout__eyebrow { - display: block; - margin-bottom: 8px; - font-size: 0.8rem; - text-transform: uppercase; - letter-spacing: 0.08em; - opacity: 0.82; -} - -.insight-callout p { - margin: 0; - line-height: 1.6; -} - -.panel__subsection { - display: flex; - flex-direction: column; - gap: 10px; -} - -.panel__subheader { - display: flex; - justify-content: space-between; - align-items: center; - gap: 12px; -} - -.panel__subheader h3 { - margin: 0; - font-size: 1rem; - color: #153124; -} - -.panel__subheader span { - color: #5a6f65; - font-size: 0.88rem; -} - -.path-table { - display: flex; - flex-direction: column; - gap: 8px; -} - -.path-table__head, -.path-row { - display: grid; - grid-template-columns: 120px 1.6fr 90px 90px 100px; - align-items: center; - gap: 12px; -} - -.path-table__head { - color: #61756b; - font-size: 0.82rem; - padding: 0 12px; -} - -.path-row { - border: 1px solid rgba(16, 38, 31, 0.08); - background: #f7faf8; - border-radius: 18px; - padding: 12px; - text-align: left; - color: #163124; - cursor: pointer; - transition: transform 0.2s ease, box-shadow 0.2s ease, border-color 0.2s ease; -} - -.path-row:hover, -.path-row--active { - transform: translateY(-1px); - border-color: rgba(20, 56, 40, 0.25); - box-shadow: 0 10px 24px rgba(19, 49, 35, 0.08); -} - -.path-row__route { - font-weight: 600; -} - -.path-tag { - display: inline-flex; - justify-content: center; - padding: 7px 10px; - border-radius: 999px; - font-size: 0.8rem; - font-weight: 700; -} - -.path-tag--balanced { - background: rgba(211, 157, 26, 0.14); - color: #9d6d00; -} - -.path-tag--low_carbon { - background: rgba(47, 158, 99, 0.14); - color: #23784b; -} - -.path-tag--low_water { - background: rgba(47, 111, 228, 0.12); - color: #2659b7; -} - -.path-tag--low_latency { - background: rgba(207, 47, 47, 0.12); - color: #ab2a2a; -} - -.path-tag--optimized { - background: rgba(31, 143, 87, 0.14); - color: #1f6f46; -} - -.path-tag--baseline { - background: rgba(123, 135, 150, 0.16); - color: #4a5563; -} - -.inline-error, -.error-state, -.loading-state { - border-radius: 24px; - padding: 20px 22px; - background: rgba(255, 255, 255, 0.9); - border: 1px solid rgba(16, 38, 31, 0.08); - box-shadow: 0 20px 50px rgba(18, 44, 32, 0.08); - max-width: 760px; -} - -.inline-error { - color: #7d2525; - background: rgba(255, 245, 245, 0.94); - border-color: rgba(190, 66, 66, 0.16); -} - -.loading-state h1, -.error-state h1 { - margin: 0 0 12px; -} - -.loading-state p, -.error-state p { - margin: 0; - color: #556b61; -} - -.map-popup { - color: #0f231a; - font-size: 0.9rem; - line-height: 1.45; -} - -.map-popup__title { - font-weight: 700; - margin-bottom: 6px; -} - -.maplibregl-popup-content { - border-radius: 14px; - padding: 10px 12px; - border: 1px solid rgba(16, 38, 31, 0.08); - box-shadow: 0 12px 26px rgba(18, 44, 32, 0.18); -} - -.maplibregl-popup-tip { - display: none; -} - -@media (max-width: 1180px) { - .bottom-layout { - grid-template-columns: 1fr; - } - - .metric-grid { - grid-template-columns: 1fr; - } -} - -@media (max-width: 900px) { - .app-shell { - padding: 18px 14px 24px; - } - - .hero-header, - .map-stage__header, - .panel__header, - .panel__subheader, - .control-footer { - flex-direction: column; - align-items: flex-start; - } - - .hero-header__status, - .panel__meta, - .panel__status { - align-self: flex-start; - text-align: left; - } - - .control-grid { - grid-template-columns: 1fr; - } - - .route-summary, - .path-table__head, - .path-row { - grid-template-columns: 1fr; - } - - .path-table__head { - display: none; - } -} - -/* ── SHAP Feature Importance Chart ─────────────────────────────────────────── */ - -.shap-loading { - margin: 0; - color: #7a9186; - font-size: 0.88rem; -} - -.shap-legend { - margin: 0 0 10px; - font-size: 0.86rem; - color: #5a6e64; -} - -.shap-toggle { - display: flex; - gap: 6px; -} - -.shap-toggle button { - border: 1px solid rgba(16, 38, 31, 0.12); - border-radius: 999px; - padding: 5px 12px; - background: #edf2ef; - color: #395347; - font-size: 0.82rem; - font-weight: 600; - cursor: pointer; -} - -.shap-toggle button.is-active { - background: #143828; - color: white; - border-color: transparent; -} - -.shap-bars { - display: flex; - flex-direction: column; - gap: 7px; -} - -.shap-row { - display: grid; - grid-template-columns: 148px 1fr 72px; - align-items: center; - gap: 10px; -} - -.shap-row__label { - font-size: 0.82rem; - color: #3d5549; - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; -} - -.shap-row__track { - height: 8px; - background: #edf2ef; - border-radius: 999px; - overflow: hidden; -} - -.shap-row__bar { - height: 100%; - border-radius: 999px; - transition: width 0.4s ease; -} - -.shap-row__bar--up { - background: #d44833; -} - -.shap-row__bar--down { - background: #2ca86b; -} - -.shap-row__value { - font-size: 0.8rem; - font-weight: 700; - text-align: right; -} - -.shap-value--up { - color: #a83422; -} - -.shap-value--down { - color: #1a7a4a; -} diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx deleted file mode 100644 index 0214cae..0000000 --- a/frontend/src/App.tsx +++ /dev/null @@ -1,284 +0,0 @@ -import { useEffect, useMemo, useState } from 'react' - -import { loadContext, runBatchSimulation, runSimulation } from './api/schedulerApi' -import { ControlPanel } from './components/ControlPanel' -import { InsightPanel } from './components/InsightPanel' -import { SimulationMap } from './components/SimulationMap' -import { toDatetimeLocalValue } from './lib/format' -import type { - BatchSimulationRequest, - BatchSimulationResponse, - CompareMode, - PathType, - SchedulerContext, - SimulationControls, - SimulationMode, - SimulationRequest, - SimulationResponse, -} from './lib/types' -import './App.css' - -function buildInitialControls(context: SchedulerContext): SimulationControls { - return { - mode: 'batch', - priority: 'medium', - alpha: 1, - beta: 1, - gamma: 1, - time: toDatetimeLocalValue(context.default_time), - batchSize: context.default_batch_size, - } -} - -function createSingleRequestPayload( - controls: SimulationControls, - context: SchedulerContext, -): SimulationRequest { - return { - priority: controls.priority, - alpha: controls.alpha, - beta: controls.beta, - gamma: controls.gamma, - time: controls.time, - latency_sensitivity: 0.55, - workload_size: context.default_workload_size, - origin_city: context.origin_city, - top_k: 4, - } -} - -function createBatchRequestPayload( - controls: SimulationControls, -): BatchSimulationRequest { - return { - priority: controls.priority, - alpha: controls.alpha, - beta: controls.beta, - gamma: controls.gamma, - time: controls.time, - latency_sensitivity: 0.55, - batch_size: controls.batchSize, - } -} - -function App() { - const [context, setContext] = useState(null) - const [controls, setControls] = useState(null) - const [submittedControls, setSubmittedControls] = useState(null) - const [singleSimulation, setSingleSimulation] = useState(null) - const [batchSimulation, setBatchSimulation] = useState(null) - const [compareMode, setCompareMode] = useState('candidates') - const [highlightedPathType, setHighlightedPathType] = useState(null) - const [highlightedFlowId, setHighlightedFlowId] = useState(null) - const [loadingContext, setLoadingContext] = useState(true) - const [loadingSimulation, setLoadingSimulation] = useState(false) - const [contextError, setContextError] = useState(null) - const [simulationError, setSimulationError] = useState(null) - - useEffect(() => { - let cancelled = false - - async function initialize() { - try { - setLoadingContext(true) - const nextContext = await loadContext() - if (cancelled) { - return - } - setContext(nextContext) - const nextControls = buildInitialControls(nextContext) - setControls(nextControls) - setSubmittedControls(nextControls) - } catch (error) { - if (!cancelled) { - setContextError(error instanceof Error ? error.message : 'Unable to load scheduler context.') - } - } finally { - if (!cancelled) { - setLoadingContext(false) - } - } - } - - void initialize() - - return () => { - cancelled = true - } - }, []) - - useEffect(() => { - if (!context || !submittedControls) { - return - } - - const activeControls = submittedControls - const activeContext = context - let cancelled = false - - async function executeSimulation() { - try { - setLoadingSimulation(true) - setSimulationError(null) - - if (activeControls.mode === 'batch') { - const result = await runBatchSimulation(createBatchRequestPayload(activeControls)) - if (cancelled) { - return - } - setBatchSimulation(result) - setHighlightedFlowId(result.selected_flow?.flow_id ?? result.optimized_flows[0]?.flow_id ?? null) - return - } - - const result = await runSimulation(createSingleRequestPayload(activeControls, activeContext)) - if (cancelled) { - return - } - setSingleSimulation(result) - setHighlightedPathType(result.selected_path?.type ?? result.paths[0]?.type ?? null) - } catch (error) { - if (!cancelled) { - setSimulationError(error instanceof Error ? error.message : 'Simulation request failed.') - } - } finally { - if (!cancelled) { - setLoadingSimulation(false) - } - } - } - - void executeSimulation() - - return () => { - cancelled = true - } - }, [context, submittedControls]) - - const activeMode = useMemo(() => { - if (submittedControls) { - return submittedControls.mode - } - return controls?.mode ?? 'batch' - }, [controls?.mode, submittedControls]) - - const selectedPathType = useMemo(() => { - if (activeMode !== 'single') { - return null - } - if (highlightedPathType) { - return highlightedPathType - } - return singleSimulation?.selected_path?.type ?? null - }, [activeMode, highlightedPathType, singleSimulation?.selected_path?.type]) - - const selectedFlowId = useMemo(() => { - if (activeMode !== 'batch') { - return null - } - if (highlightedFlowId) { - return highlightedFlowId - } - return batchSimulation?.selected_flow?.flow_id ?? batchSimulation?.optimized_flows[0]?.flow_id ?? null - }, [activeMode, batchSimulation?.optimized_flows, batchSimulation?.selected_flow?.flow_id, highlightedFlowId]) - - if (loadingContext) { - return ( -
-
-

Connecting

-

Loading sustainability routing context…

-
-
- ) - } - - if (contextError || !context || !controls) { - return ( -
-
-

Backend unavailable

-

React frontend could not reach the scheduling API.

-

{contextError ?? 'Start `python3 -m uvicorn src.api:app --reload` and try again.'}

-
-
- ) - } - - return ( -
-
-
-

Interactive sustainability-aware workload scheduler

-

Simulate one decision or watch 50–200 jobs move as a coordinated batch.

-

- Switch between a single routing recommendation and a multi-job stress test to see how - carbon, water, and latency trade-offs change when the system schedules many workloads - at once. -

-
-
- - FastAPI simulation backend connected -
-
- - {simulationError ? ( -
- Simulation error: {simulationError} -
- ) : null} - -
-
-
-

Main map

-

{activeMode === 'batch' ? 'Batch routing flows' : 'Routing decision space'}

-
-

- {activeMode === 'batch' - ? 'Green bands show the optimized batch schedule. Toggle the baseline overlay to compare how traffic shifts across data centers.' - : 'Green routes optimize carbon, blue routes optimize water, gold routes balance objectives, and red routes favor latency.'} -

-
- - -
- -
- setSubmittedControls({ ...controls })} - /> - - -
-
- ) -} - -export default App diff --git a/frontend/src/api/schedulerApi.ts b/frontend/src/api/schedulerApi.ts deleted file mode 100644 index ccdf5f3..0000000 --- a/frontend/src/api/schedulerApi.ts +++ /dev/null @@ -1,57 +0,0 @@ -import type { - BatchSimulationRequest, - BatchSimulationResponse, - ExplanationResponse, - SchedulerContext, - SimulationRequest, - SimulationResponse, -} from '../lib/types' - -const rawBaseUrl = import.meta.env.VITE_API_BASE_URL?.trim() -const API_BASE_URL = rawBaseUrl ? rawBaseUrl.replace(/\/$/, '') : '/api' - -async function fetchJson(path: string, init?: RequestInit): Promise { - const response = await fetch(`${API_BASE_URL}${path}`, { - headers: { - 'Content-Type': 'application/json', - ...(init?.headers ?? {}), - }, - ...init, - }) - - if (!response.ok) { - const body = await response.text() - throw new Error(body || `Request failed with status ${response.status}`) - } - - return response.json() as Promise -} - -export function loadContext(): Promise { - return fetchJson('/context') -} - -export function runSimulation(payload: SimulationRequest): Promise { - return fetchJson('/simulate', { - method: 'POST', - body: JSON.stringify(payload), - }) -} - -export function runBatchSimulation( - payload: BatchSimulationRequest, -): Promise { - return fetchJson('/simulate-batch', { - method: 'POST', - body: JSON.stringify(payload), - }) -} - -export function fetchExplanation( - city: string, - target: 'co2' | 'wue', - time: string, -): Promise { - const params = new URLSearchParams({ city, target, time }) - return fetchJson(`/explain?${params.toString()}`) -} diff --git a/frontend/src/components/ControlPanel.tsx b/frontend/src/components/ControlPanel.tsx deleted file mode 100644 index 1fe3b22..0000000 --- a/frontend/src/components/ControlPanel.tsx +++ /dev/null @@ -1,197 +0,0 @@ -import type { CompareMode, SchedulerContext, SimulationControls } from '../lib/types' -import { toDatetimeLocalValue } from '../lib/format' - -interface ControlPanelProps { - controls: SimulationControls - context: SchedulerContext - compareMode: CompareMode - isLoading: boolean - onControlsChange: (next: SimulationControls) => void - onCompareModeChange: (mode: CompareMode) => void - onRun: () => void -} - -function updateControl( - controls: SimulationControls, - key: K, - value: SimulationControls[K], -): SimulationControls { - return { ...controls, [key]: value } -} - -export function ControlPanel({ - controls, - context, - compareMode, - isLoading, - onControlsChange, - onCompareModeChange, - onRun, -}: ControlPanelProps) { - const isBatch = controls.mode === 'batch' - - return ( -
-
-
-

Control panel

-

Shape the scheduling scenario

-
-
- Mode - {isBatch ? 'Batch' : 'Single job'} -
-
- -
- - -
- -
- - - - - {isBatch ? ( - - ) : ( -
- Origin city -
{context.origin_city}
-
- )} - -
- - -
-
- -
- - - - - -
- -
-
- {isBatch ? 'Simulation window' : 'Workload size'} - - {isBatch ? `${controls.batchSize} jobs across a 24-hour window` : `${context.default_workload_size.toFixed(1)} units`} - -
- -
-
- ) -} diff --git a/frontend/src/components/InsightPanel.tsx b/frontend/src/components/InsightPanel.tsx deleted file mode 100644 index 1126741..0000000 --- a/frontend/src/components/InsightPanel.tsx +++ /dev/null @@ -1,376 +0,0 @@ -import { useEffect, useState } from 'react' -import type { - BatchFlow, - BatchSimulationResponse, - ExplanationResponse, - PathCandidate, - PathType, - SimulationMode, - SimulationResponse, -} from '../lib/types' -import { formatLatency, formatPercent, formatDatetimeLabel } from '../lib/format' -import { fetchExplanation } from '../api/schedulerApi' - -const PATH_LABELS: Record = { - balanced: 'Balanced', - low_carbon: 'Low carbon', - low_water: 'Low water', - low_latency: 'Low latency', - baseline: 'Baseline', -} - -interface InsightPanelProps { - mode: SimulationMode - simulation: SimulationResponse | null - batchSimulation: BatchSimulationResponse | null - highlightedPathType: PathType | null - highlightedFlowId: string | null - onHighlightPath: (pathType: PathType) => void - onHighlightFlow: (flowId: string) => void -} - -const TARGET_UNIT: Record<'co2' | 'wue', string> = { - co2: 'kg CO₂/kWh', - wue: 'L/kWh (WUE)', -} - -function ShapChart({ city, time }: { city: string; time: string }) { - const [co2Data, setCo2Data] = useState(null) - const [wueData, setWueData] = useState(null) - const [activeTarget, setActiveTarget] = useState<'co2' | 'wue'>('co2') - const [loading, setLoading] = useState(true) - - useEffect(() => { - let cancelled = false - setLoading(true) - setCo2Data(null) - setWueData(null) - - Promise.all([ - fetchExplanation(city, 'co2', time), - fetchExplanation(city, 'wue', time), - ]) - .then(([co2, wue]) => { - if (cancelled) return - setCo2Data(co2) - setWueData(wue) - }) - .catch(() => { /* silently ignore — SHAP is optional enrichment */ }) - .finally(() => { if (!cancelled) setLoading(false) }) - - return () => { cancelled = true } - }, [city, time]) - - const data = activeTarget === 'co2' ? co2Data : wueData - if (loading) { - return

Computing feature importance…

- } - if (!data) return null - - const maxAbs = Math.max(...data.features.map(f => Math.abs(f.shap_value)), 0.0001) - - return ( -
-
-

Why this prediction?

-
- - -
-
-

- Predicted {TARGET_UNIT[activeTarget]}: {data.prediction.toFixed(4)} - {' '}(baseline: {data.base_value.toFixed(4)}) -

-
- {data.features.map(f => { - const pct = (Math.abs(f.shap_value) / maxAbs) * 100 - const positive = f.shap_value > 0 - return ( -
- {f.name} -
-
-
- - {positive ? '+' : ''}{f.shap_value.toFixed(4)} - -
- ) - })} -
-
- ) -} - -function pathLabel(path: PathCandidate | null): string { - if (!path) { - return 'No route available' - } - return path.route -} - -function renderSingleView( - simulation: SimulationResponse, - highlightedPathType: PathType | null, - onHighlightPath: (pathType: PathType) => void, -) { - const { selected_path: selectedPath, baseline_path: baselinePath, metrics, insight, paths } = simulation - const comparisonPaths = paths.slice(0, 3) - - return ( - <> -
-
- Best path - {pathLabel(selectedPath)} -
-
- Baseline - {pathLabel(baselinePath)} -
-
- -
-
- CO2 reduction - {formatPercent(metrics.co2_reduction_pct)} -

Compared with the naive route.

-
-
- Water reduction - {formatPercent(metrics.water_reduction_pct)} -

Lower water stress and WUE impact.

-
-
- Latency impact - {formatLatency(metrics.latency_delta_ms)} -

Positive means slower than baseline.

-
-
- -
- {insight.title} -

{insight.summary}

-
- -
-
-

Top route options

- {formatDatetimeLabel(simulation.time)} -
-
-
- Path - Route - CO2 - WUE - Latency -
- {comparisonPaths.map((path) => ( - - ))} -
-
- - {selectedPath && ( - - )} - - ) -} - -function renderBatchRow( - flow: BatchFlow, - highlightedFlowId: string | null, - onHighlightFlow: (flowId: string) => void, -) { - return ( - - ) -} - -function renderBatchView( - simulation: BatchSimulationResponse, - highlightedFlowId: string | null, - onHighlightFlow: (flowId: string) => void, -) { - const { selected_flow: selectedFlow, optimized_summary, comparison, insight, optimized_flows } = simulation - const topFlows = optimized_flows.slice(0, 6) - const baselineCoverage = simulation.baseline_summary.coverage * 100 - const optimizedCoverage = optimized_summary.coverage * 100 - - return ( - <> -
-
- Busiest optimized flow - {selectedFlow?.route ?? 'No active route'} -
-
- Jobs scheduled - - {Math.round(optimized_summary.scheduled_jobs)} / {Math.round(optimized_summary.total_jobs)} - -
-
- -
-
- CO2 reduction - {formatPercent(comparison.co2_reduction_pct)} -

Optimized batch versus baseline placement.

-
-
- Water reduction - {formatPercent(comparison.water_reduction_pct)} -

Lower total water impact across the batch.

-
-
- Latency impact - {formatLatency(comparison.latency_delta_ms)} -

Weighted average route latency versus baseline.

-
-
- Coverage - {optimizedCoverage.toFixed(0)}% -

Baseline was {baselineCoverage.toFixed(0)}% for the same batch.

-
-
- -
- {insight.title} -

{insight.summary}

-
- -
-
-

Top batch flows

- - {formatDatetimeLabel(simulation.time)} to {formatDatetimeLabel(simulation.window_end)} - -
-
-
- Flow - Route - CO2 - Water - Latency -
- {topFlows.map((flow) => renderBatchRow(flow, highlightedFlowId, onHighlightFlow))} -
-
- - ) -} - -export function InsightPanel({ - mode, - simulation, - batchSimulation, - highlightedPathType, - highlightedFlowId, - onHighlightPath, - onHighlightFlow, -}: InsightPanelProps) { - if (mode === 'batch') { - if (!batchSimulation) { - return ( -
-
-
-

Batch result

-

Awaiting batch simulation

-
-
-

- Run the batch simulation to see how many jobs move, which routes carry the heaviest load, - and how the optimized schedule compares with the baseline. -

-
- ) - } - - return ( -
-
-
-

Batch result and insight

-

How the system coordinates many jobs at once

-
-
Optimized batch
-
- {renderBatchView(batchSimulation, highlightedFlowId, onHighlightFlow)} -
- ) - } - - if (!simulation || !simulation.selected_path) { - return ( -
-
-
-

Result

-

Awaiting simulation

-
-
-

- Run the simulation to see the recommended route, environmental trade-offs, and the top candidate paths. -

-
- ) - } - - return ( -
-
-
-

Result and insight

-

Why the system sends the job there

-
-
Recommended
-
- {renderSingleView(simulation, highlightedPathType, onHighlightPath)} -
- ) -} diff --git a/frontend/src/components/SimulationMap.tsx b/frontend/src/components/SimulationMap.tsx deleted file mode 100644 index bc88b26..0000000 --- a/frontend/src/components/SimulationMap.tsx +++ /dev/null @@ -1,484 +0,0 @@ -import { useEffect, useMemo, useRef } from 'react' -import type { Feature, FeatureCollection, LineString, Point } from 'geojson' -import maplibregl, { LngLatBounds, type Map as MapLibreMap, type Popup } from 'maplibre-gl' -import 'maplibre-gl/dist/maplibre-gl.css' - -import type { - BatchFlow, - CompareMode, - PathCandidate, - PathType, - SimulationMode, - SimulationNode, -} from '../lib/types' - -const MAP_STYLE_URL = 'https://basemaps.cartocdn.com/gl/positron-gl-style/style.json' -const NODE_SOURCE_ID = 'simulation-nodes' -const PATH_SOURCE_ID = 'simulation-paths' -const NODE_LAYER_ID = 'simulation-node-circles' -const PATH_LAYER_ID = 'simulation-path-lines' -const UNITED_STATES_BOUNDS: [[number, number], [number, number]] = [ - [-127.5, 23.0], - [-65.0, 50.5], -] -const UNITED_STATES_CENTER: [number, number] = [-98.5, 39.8] -const UNITED_STATES_MIN_ZOOM = 3.1 - -const PATH_TYPE_COLORS: Record = { - balanced: '#d39d1a', - low_carbon: '#2f9e63', - low_water: '#2f6fe4', - low_latency: '#cf2f2f', - baseline: '#677282', -} - -const FLOW_SCENARIO_COLORS = { - optimized: '#1f8f57', - baseline: '#7b8796', -} as const - -type NodeProperties = { - city: string - state?: string - avg_co2_intensity: number - avg_wue: number - avg_scarcity: number - records: number - has_data_center: number -} - -type LineProperties = { - route: string - typeLabel: string - co2: number - water: number - latency: number - score: number - lineColor: string - lineWidth: number - arrow: string - jobs?: number - scenario?: string -} - -interface SimulationMapProps { - mode: SimulationMode - nodes: SimulationNode[] - paths: PathCandidate[] - flows: BatchFlow[] - baselineFlows: BatchFlow[] - selectedPathType: PathType | null - selectedFlowId: string | null - compareMode: CompareMode - recommendedPath: PathCandidate | null - baselinePath: PathCandidate | null -} - -function buildNodeCollection(nodes: SimulationNode[]): FeatureCollection { - return { - type: 'FeatureCollection', - features: nodes.map>((node) => ({ - type: 'Feature', - geometry: { - type: 'Point', - coordinates: [node.longitude, node.latitude], - }, - properties: { - city: node.city, - state: node.state, - avg_co2_intensity: node.avg_co2_intensity, - avg_wue: node.avg_wue, - avg_scarcity: node.avg_scarcity, - records: node.records, - has_data_center: node.has_data_center, - }, - })), - } -} - -function buildDisplayedPaths( - compareMode: CompareMode, - paths: PathCandidate[], - recommendedPath: PathCandidate | null, - baselinePath: PathCandidate | null, -): PathCandidate[] { - if (compareMode === 'baseline') { - return [baselinePath, recommendedPath].filter((path): path is PathCandidate => Boolean(path)) - } - return paths -} - -function buildDisplayedFlows( - compareMode: CompareMode, - flows: BatchFlow[], - baselineFlows: BatchFlow[], -): BatchFlow[] { - if (compareMode === 'baseline') { - return [...baselineFlows, ...flows] - } - return flows -} - -function buildPathCollection( - paths: PathCandidate[], - selectedPathType: PathType | null, -): FeatureCollection { - return { - type: 'FeatureCollection', - features: paths.map>((path) => { - const isHighlighted = selectedPathType === null || selectedPathType === path.type - return { - type: 'Feature', - geometry: { - type: 'LineString', - coordinates: [ - [path.origin_longitude, path.origin_latitude], - [path.destination_longitude, path.destination_latitude], - ], - }, - properties: { - route: path.route, - typeLabel: path.type.replace('_', ' '), - co2: path.co2, - water: path.water_liters, - latency: path.latency, - score: path.score, - lineColor: PATH_TYPE_COLORS[path.type], - lineWidth: isHighlighted ? 6.5 : 3.5, - arrow: '▶', - }, - } - }), - } -} - -function buildFlowCollection( - flows: BatchFlow[], - selectedFlowId: string | null, -): FeatureCollection { - const jobs = flows.map((flow) => flow.jobs) - const minJobs = jobs.length > 0 ? Math.min(...jobs) : 1 - const maxJobs = jobs.length > 0 ? Math.max(...jobs) : 1 - const span = Math.max(1, maxJobs - minJobs) - - return { - type: 'FeatureCollection', - features: flows.map>((flow) => { - const emphasized = selectedFlowId === null || selectedFlowId === flow.flow_id - const width = 3.5 + ((flow.jobs - minJobs) / span) * 6.5 + (emphasized ? 1.5 : 0) - return { - type: 'Feature', - geometry: { - type: 'LineString', - coordinates: [ - [flow.origin_longitude, flow.origin_latitude], - [flow.destination_longitude, flow.destination_latitude], - ], - }, - properties: { - route: flow.route, - typeLabel: flow.scenario === 'optimized' ? 'Optimized batch flow' : 'Baseline batch flow', - co2: flow.co2, - water: flow.water_liters, - latency: flow.latency, - score: flow.score, - jobs: flow.jobs, - scenario: flow.scenario, - lineColor: FLOW_SCENARIO_COLORS[flow.scenario], - lineWidth: width, - arrow: '▶', - }, - } - }), - } -} - -function extent(values: number[]): [number, number] { - if (values.length === 0) { - return [0, 1] - } - const min = Math.min(...values) - const max = Math.max(...values) - return min === max ? [min, min + 1] : [min, max] -} - -function nodePopupHtml(properties: NodeProperties): string { - return ` -
-
${properties.city}
-
CO2 intensity: ${properties.avg_co2_intensity.toFixed(3)} kg/kWh
-
WUE: ${properties.avg_wue.toFixed(3)} L/kWh
-
Scarcity: ${properties.avg_scarcity.toFixed(3)}
-
- ` -} - -function linePopupHtml(properties: LineProperties): string { - const batchLine = typeof properties.jobs === 'number' ? `
Jobs: ${properties.jobs}
` : '' - return ` -
-
${properties.route}
-
${properties.typeLabel}
- ${batchLine} -
CO2: ${properties.co2.toFixed(2)} kg
-
Water: ${properties.water.toFixed(2)} L
-
Latency: ${properties.latency.toFixed(1)} ms
-
- ` -} - -export function SimulationMap({ - mode, - nodes, - paths, - flows, - baselineFlows, - selectedPathType, - selectedFlowId, - compareMode, - recommendedPath, - baselinePath, -}: SimulationMapProps) { - const mapContainerRef = useRef(null) - const mapRef = useRef(null) - const popupRef = useRef(null) - const hasFittedRef = useRef(false) - - const displayedPaths = useMemo( - () => buildDisplayedPaths(compareMode, paths, recommendedPath, baselinePath), - [baselinePath, compareMode, paths, recommendedPath], - ) - const displayedFlows = useMemo( - () => buildDisplayedFlows(compareMode, flows, baselineFlows), - [baselineFlows, compareMode, flows], - ) - const nodeCollection = useMemo(() => buildNodeCollection(nodes), [nodes]) - const lineCollection = useMemo( - () => - mode === 'batch' - ? buildFlowCollection(displayedFlows, selectedFlowId) - : buildPathCollection(displayedPaths, selectedPathType), - [displayedFlows, displayedPaths, mode, selectedFlowId, selectedPathType], - ) - - useEffect(() => { - if (!mapContainerRef.current || mapRef.current) { - return - } - - const map = new maplibregl.Map({ - container: mapContainerRef.current, - style: MAP_STYLE_URL, - center: UNITED_STATES_CENTER, - zoom: 3.45, - maxBounds: UNITED_STATES_BOUNDS, - minZoom: UNITED_STATES_MIN_ZOOM, - attributionControl: false, - }) - - map.addControl(new maplibregl.NavigationControl({ visualizePitch: true }), 'top-right') - const popup = new maplibregl.Popup({ - closeButton: false, - closeOnClick: false, - offset: 18, - className: 'simulation-map-popup', - }) - - mapRef.current = map - popupRef.current = popup - - map.on('load', () => { - map.addSource(NODE_SOURCE_ID, { - type: 'geojson', - data: nodeCollection, - }) - map.addSource(PATH_SOURCE_ID, { - type: 'geojson', - data: lineCollection, - }) - - map.addLayer({ - id: 'simulation-scarcity-heat', - type: 'heatmap', - source: NODE_SOURCE_ID, - paint: { - 'heatmap-weight': ['coalesce', ['get', 'avg_scarcity'], 0], - 'heatmap-intensity': 0.7, - 'heatmap-radius': 38, - 'heatmap-opacity': 0.38, - 'heatmap-color': [ - 'interpolate', - ['linear'], - ['heatmap-density'], - 0, - 'rgba(227, 242, 235, 0)', - 0.4, - 'rgba(208, 230, 201, 0.45)', - 1, - 'rgba(112, 177, 116, 0.75)', - ], - }, - }) - - map.addLayer({ - id: PATH_LAYER_ID, - type: 'line', - source: PATH_SOURCE_ID, - layout: { - 'line-cap': 'round', - 'line-join': 'round', - }, - paint: { - 'line-color': ['to-color', ['get', 'lineColor']], - 'line-width': ['coalesce', ['get', 'lineWidth'], 4], - 'line-opacity': [ - 'case', - ['==', ['get', 'scenario'], 'baseline'], - 0.52, - 0.94, - ], - }, - }) - - map.addLayer({ - id: 'simulation-path-arrows', - type: 'symbol', - source: PATH_SOURCE_ID, - layout: { - 'symbol-placement': 'line', - 'symbol-spacing': 110, - 'text-field': ['get', 'arrow'], - 'text-size': 14, - 'text-keep-upright': false, - }, - paint: { - 'text-color': ['to-color', ['get', 'lineColor']], - 'text-opacity': 0.88, - }, - }) - - map.addLayer({ - id: NODE_LAYER_ID, - type: 'circle', - source: NODE_SOURCE_ID, - paint: { - 'circle-color': '#5ca86d', - 'circle-radius': 12, - 'circle-stroke-width': ['case', ['==', ['get', 'has_data_center'], 1], 2.5, 1.25], - 'circle-stroke-color': [ - 'case', - ['==', ['get', 'has_data_center'], 1], - '#123c2b', - 'rgba(255,255,255,0.85)', - ], - 'circle-opacity': 0.92, - }, - }) - - const enterLayer = () => { - map.getCanvas().style.cursor = 'pointer' - } - const leaveLayer = () => { - map.getCanvas().style.cursor = '' - popup.remove() - } - - map.on('mouseenter', NODE_LAYER_ID, enterLayer) - map.on('mouseleave', NODE_LAYER_ID, leaveLayer) - map.on('mousemove', NODE_LAYER_ID, (event) => { - const feature = event.features?.[0] - if (!feature || feature.geometry.type !== 'Point') { - return - } - popup - .setLngLat(feature.geometry.coordinates as [number, number]) - .setHTML(nodePopupHtml(feature.properties as unknown as NodeProperties)) - .addTo(map) - }) - - map.on('mouseenter', PATH_LAYER_ID, enterLayer) - map.on('mouseleave', PATH_LAYER_ID, leaveLayer) - map.on('mousemove', PATH_LAYER_ID, (event) => { - const feature = event.features?.[0] - if (!feature || !event.lngLat) { - return - } - popup - .setLngLat(event.lngLat) - .setHTML(linePopupHtml(feature.properties as unknown as LineProperties)) - .addTo(map) - }) - }) - - return () => { - popup.remove() - map.remove() - mapRef.current = null - popupRef.current = null - } - }, [lineCollection, nodeCollection]) - - useEffect(() => { - const map = mapRef.current - if (!map || !map.isStyleLoaded()) { - return - } - - const nodeSource = map.getSource(NODE_SOURCE_ID) as maplibregl.GeoJSONSource | undefined - const pathSource = map.getSource(PATH_SOURCE_ID) as maplibregl.GeoJSONSource | undefined - nodeSource?.setData(nodeCollection) - pathSource?.setData(lineCollection) - - const [co2Min, co2Max] = extent(nodes.map((node) => Number(node.avg_co2_intensity))) - const [wueMin, wueMax] = extent(nodes.map((node) => Number(node.avg_wue))) - - map.setPaintProperty(NODE_LAYER_ID, 'circle-color', [ - 'interpolate', - ['linear'], - ['coalesce', ['get', 'avg_co2_intensity'], co2Min], - co2Min, - '#2ca86b', - (co2Min + co2Max) / 2, - '#e7ad2e', - co2Max, - '#d44833', - ]) - - map.setPaintProperty(NODE_LAYER_ID, 'circle-radius', [ - 'interpolate', - ['linear'], - ['coalesce', ['get', 'avg_wue'], wueMin], - wueMin, - 8, - wueMax, - 24, - ]) - - if (nodes.length === 0) { - return - } - - const bounds = new LngLatBounds() - nodes.forEach((node) => { - bounds.extend([node.longitude, node.latitude]) - }) - if (!bounds.isEmpty() && (!hasFittedRef.current || lineCollection.features.length > 0)) { - map.fitBounds(bounds, { - padding: { - top: 70, - bottom: 70, - left: 70, - right: 70, - }, - maxZoom: 5.2, - duration: 900, - }) - if (map.getZoom() < UNITED_STATES_MIN_ZOOM) { - map.setZoom(UNITED_STATES_MIN_ZOOM) - } - hasFittedRef.current = true - } - }, [lineCollection, nodeCollection, nodes]) - - return
-} diff --git a/frontend/src/index.css b/frontend/src/index.css deleted file mode 100644 index 781a5b7..0000000 --- a/frontend/src/index.css +++ /dev/null @@ -1,43 +0,0 @@ -:root { - font-family: - 'Instrument Sans', - 'Avenir Next', - 'Segoe UI', - sans-serif; - line-height: 1.5; - font-weight: 400; - color: #1b3127; - background: - radial-gradient(circle at top left, rgba(230, 244, 236, 0.95), transparent 24%), - radial-gradient(circle at top right, rgba(226, 236, 246, 0.95), transparent 22%), - linear-gradient(180deg, #f7faf8 0%, #edf4f2 100%); - font-synthesis: none; - text-rendering: optimizeLegibility; - -webkit-font-smoothing: antialiased; - -moz-osx-font-smoothing: grayscale; -} - -* { - box-sizing: border-box; -} - -html, -body, -#root { - margin: 0; - min-height: 100%; -} - -body { - min-width: 320px; -} - -button, -input, -select { - font: inherit; -} - -button { - appearance: none; -} diff --git a/frontend/src/lib/format.ts b/frontend/src/lib/format.ts deleted file mode 100644 index 29835ba..0000000 --- a/frontend/src/lib/format.ts +++ /dev/null @@ -1,43 +0,0 @@ -export function formatPercent(value: number): string { - return `${value >= 0 ? '+' : ''}${value.toFixed(1)}%` -} - -export function formatLatency(value: number): string { - return `${value >= 0 ? '+' : ''}${value.toFixed(1)} ms` -} - -export function formatDatetimeLabel(value: string): string { - return new Intl.DateTimeFormat('en-US', { - month: 'short', - day: 'numeric', - year: 'numeric', - hour: 'numeric', - minute: '2-digit', - }).format(new Date(value)) -} - -export function toDatetimeLocalValue(value: string): string { - const date = new Date(value) - const year = date.getFullYear() - const month = `${date.getMonth() + 1}`.padStart(2, '0') - const day = `${date.getDate()}`.padStart(2, '0') - const hours = `${date.getHours()}`.padStart(2, '0') - const minutes = `${date.getMinutes()}`.padStart(2, '0') - return `${year}-${month}-${day}T${hours}:${minutes}` -} - -export function clampIsoDatetime(value: string, minValue: string, maxValue: string): string { - const current = new Date(value).getTime() - const min = new Date(minValue).getTime() - const max = new Date(maxValue).getTime() - if (Number.isNaN(current)) { - return toDatetimeLocalValue(minValue) - } - if (current < min) { - return toDatetimeLocalValue(minValue) - } - if (current > max) { - return toDatetimeLocalValue(maxValue) - } - return toDatetimeLocalValue(new Date(current).toISOString()) -} diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts deleted file mode 100644 index 4219627..0000000 --- a/frontend/src/lib/types.ts +++ /dev/null @@ -1,168 +0,0 @@ -export type Priority = 'low' | 'medium' | 'high' -export type SimulationMode = 'single' | 'batch' -export type CompareMode = 'candidates' | 'baseline' -export type PathType = 'balanced' | 'low_carbon' | 'low_water' | 'low_latency' | 'baseline' -export type FlowScenario = 'optimized' | 'baseline' - -export interface SchedulerContext { - origin_city: string - time_min: string - time_max: string - default_time: string - default_workload_size: number - default_batch_size: number - batch_size_min: number - batch_size_max: number - priority_counts: Record -} - -export interface SimulationRequest { - priority: Priority - alpha: number - beta: number - gamma: number - time: string - latency_sensitivity: number - workload_size: number - origin_city: string - top_k: number -} - -export interface BatchSimulationRequest { - priority: Priority - alpha: number - beta: number - gamma: number - time: string - latency_sensitivity: number - batch_size: number -} - -export interface PathCandidate { - route: string - path: string[] - co2: number - wue: number - latency: number - score: number - type: PathType - scheduled_start: string - scheduled_end: string - origin_city: string - assigned_city: string - origin_latitude: number - origin_longitude: number - destination_latitude: number - destination_longitude: number - water_liters: number -} - -export interface BatchFlow { - flow_id: string - route: string - origin_city: string - assigned_city: string - origin_latitude: number - origin_longitude: number - destination_latitude: number - destination_longitude: number - jobs: number - co2: number - water_liters: number - latency: number - score: number - scenario: FlowScenario -} - -export interface SimulationNode { - city: string - state?: string - zip?: string - avg_co2_intensity: number - avg_wue: number - avg_scarcity: number - records: number - latitude: number - longitude: number - has_data_center: number -} - -export interface SimulationMetrics { - co2_reduction_pct: number - water_reduction_pct: number - latency_delta_ms: number - latency_delta_pct: number -} - -export interface BatchComparison { - co2_reduction_pct: number - water_reduction_pct: number - coverage_delta_pct: number - scheduled_jobs_delta: number - latency_delta_ms: number -} - -export interface SimulationInsight { - title: string - summary: string -} - -export interface SimulationResponse { - time: string - origin_city: string - paths: PathCandidate[] - selected_path: PathCandidate | null - baseline_path: PathCandidate | null - metrics: SimulationMetrics - insight: SimulationInsight - nodes: SimulationNode[] -} - -export interface BatchSummary { - scheduled_jobs: number - total_jobs: number - coverage: number - total_expected_co2_kg: number - total_expected_water_liters: number - avg_latency_ms: number -} - -export interface BatchSimulationResponse { - time: string - window_end: string - batch_size: number - priority: Priority - nodes: SimulationNode[] - optimized_flows: BatchFlow[] - baseline_flows: BatchFlow[] - selected_flow: BatchFlow | null - optimized_summary: BatchSummary - baseline_summary: BatchSummary - comparison: BatchComparison - insight: SimulationInsight -} - -export interface ShapFeature { - name: string - raw_name: string - value: number - shap_value: number -} - -export interface ExplanationResponse { - city: string - target: 'co2' | 'wue' - prediction: number - base_value: number - features: ShapFeature[] -} - -export interface SimulationControls { - mode: SimulationMode - priority: Priority - alpha: number - beta: number - gamma: number - time: string - batchSize: number -} diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx deleted file mode 100644 index bef5202..0000000 --- a/frontend/src/main.tsx +++ /dev/null @@ -1,10 +0,0 @@ -import { StrictMode } from 'react' -import { createRoot } from 'react-dom/client' -import './index.css' -import App from './App.tsx' - -createRoot(document.getElementById('root')!).render( - - - , -) diff --git a/frontend/src/vite-env.d.ts b/frontend/src/vite-env.d.ts deleted file mode 100644 index 11f02fe..0000000 --- a/frontend/src/vite-env.d.ts +++ /dev/null @@ -1 +0,0 @@ -/// diff --git a/frontend/tsconfig.app.json b/frontend/tsconfig.app.json deleted file mode 100644 index af516fc..0000000 --- a/frontend/tsconfig.app.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "compilerOptions": { - "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo", - "target": "ES2023", - "useDefineForClassFields": true, - "lib": ["ES2023", "DOM", "DOM.Iterable"], - "module": "ESNext", - "types": ["vite/client"], - "skipLibCheck": true, - - /* Bundler mode */ - "moduleResolution": "bundler", - "allowImportingTsExtensions": true, - "verbatimModuleSyntax": true, - "moduleDetection": "force", - "noEmit": true, - "jsx": "react-jsx", - - /* Linting */ - "strict": true, - "noUnusedLocals": true, - "noUnusedParameters": true, - "erasableSyntaxOnly": true, - "noFallthroughCasesInSwitch": true, - "noUncheckedSideEffectImports": true - }, - "include": ["src"] -} diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json deleted file mode 100644 index 1ffef60..0000000 --- a/frontend/tsconfig.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "files": [], - "references": [ - { "path": "./tsconfig.app.json" }, - { "path": "./tsconfig.node.json" } - ] -} diff --git a/frontend/tsconfig.node.json b/frontend/tsconfig.node.json deleted file mode 100644 index 8a67f62..0000000 --- a/frontend/tsconfig.node.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "compilerOptions": { - "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo", - "target": "ES2023", - "lib": ["ES2023"], - "module": "ESNext", - "types": ["node"], - "skipLibCheck": true, - - /* Bundler mode */ - "moduleResolution": "bundler", - "allowImportingTsExtensions": true, - "verbatimModuleSyntax": true, - "moduleDetection": "force", - "noEmit": true, - - /* Linting */ - "strict": true, - "noUnusedLocals": true, - "noUnusedParameters": true, - "erasableSyntaxOnly": true, - "noFallthroughCasesInSwitch": true, - "noUncheckedSideEffectImports": true - }, - "include": ["vite.config.ts"] -} diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts deleted file mode 100644 index 3a1ad90..0000000 --- a/frontend/vite.config.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { defineConfig } from 'vite' -import react from '@vitejs/plugin-react' - -// https://vite.dev/config/ -export default defineConfig({ - plugins: [react()], - server: { - port: 5173, - proxy: { - '/api': { - target: 'http://127.0.0.1:8000', - changeOrigin: true, - rewrite: (path) => path.replace(/^\/api/, ''), - }, - }, - }, -}) diff --git a/notebooks/eda.ipynb b/notebooks/eda.ipynb deleted file mode 100644 index 9e0dbc9..0000000 --- a/notebooks/eda.ipynb +++ /dev/null @@ -1,240 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "4dff1e0a", - "metadata": {}, - "source": [ - "# Exploratory Data Analysis on Sample Dataset" - ] - }, - { - "cell_type": "markdown", - "id": "2e6ad806", - "metadata": {}, - "source": [ - "\n", - "This notebook performs exploratory data analysis (EDA) on the sample dataset from the project. The goal is to understand the distribution of key variables, identify missing values, explore correlations, and investigate relationships between weather, energy mix, carbon intensity, and water intensity. Observations from previous analysis are included where relevant, with references to the figures and insights noted during earlier EDA (e.g. seasonal patterns【776473472252763†screenshot】, linear relationships between hydro share and water intensity【59990393566859†screenshot】, coal share vs. carbon intensity【810379427692406†screenshot】, and regional differences【265301069593362†screenshot】).\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5811af86", - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "import pandas as pd\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "\n", - "# Load dataset (adjust the path if necessary)\n", - "file_path = '../data/sample_dataset.xlsx'\n", - "df = pd.read_excel(file_path)\n", - "\n", - "# Display basic info\n", - "print('Dataset shape:', df.shape)\n", - "df.head()\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "47ffaedf", - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "# Calculate missing value percentages\n", - "missing_pct = df.isnull().mean().sort_values(ascending=False) * 100\n", - "missing_pct = missing_pct[missing_pct > 0]\n", - "print('Missing value percentages (sorted descending):')\n", - "print(missing_pct)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "49994c4c", - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "# Convert TIMESTAMP to pandas datetime (if not already)\n", - "df['TIMESTAMP'] = pd.to_datetime(df['TIMESTAMP'])\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "724a3ceb", - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "# Plot histograms of carbon intensity and water intensity\n", - "fig, axs = plt.subplots(1, 2, figsize=(12, 4))\n", - "\n", - "# Histogram for kgco2_per_kwh (carbon intensity)\n", - "axs[0].hist(df['co2_kg'] / df['total_gen_kwh'], bins=30)\n", - "axs[0].set_title('Distribution of Carbon Intensity (kgCO$_2$ per kWh)')\n", - "axs[0].set_xlabel('kgCO$_2$ per kWh')\n", - "axs[0].set_ylabel('Frequency')\n", - "\n", - "# Histogram for WUE_total (water intensity)\n", - "axs[1].hist(df['WUE_total'], bins=30)\n", - "axs[1].set_title('Distribution of Water Intensity (WUE\\_total)')\n", - "axs[1].set_xlabel('WUE\\_total (liters per kWh)')\n", - "axs[1].set_ylabel('Frequency')\n", - "\n", - "plt.tight_layout()\n", - "plt.show()\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2d1b3969", - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "# Compute daily averages of carbon intensity and water intensity\n", - "df['date'] = df['TIMESTAMP'].dt.date\n", - "\n", - "daily_df = df.groupby('date').agg({\n", - " 'co2_kg': 'sum',\n", - " 'total_gen_kwh': 'sum',\n", - " 'WUE_total': 'mean'\n", - "}).reset_index()\n", - "# Carbon intensity as daily total emissions divided by total generation\n", - "\n", - "# Avoid division by zero\n", - "mask = daily_df['total_gen_kwh'] != 0\n", - "daily_df.loc[mask, 'kgco2_per_kwh'] = daily_df.loc[mask, 'co2_kg'] / daily_df.loc[mask, 'total_gen_kwh']\n", - "\n", - "fig, axs = plt.subplots(2, 1, figsize=(12, 6), sharex=True)\n", - "axs[0].plot(daily_df['date'], daily_df['kgco2_per_kwh'])\n", - "axs[0].set_ylabel('kgCO$_2$ per kWh')\n", - "axs[0].set_title('Daily Average Carbon Intensity')\n", - "\n", - "axs[1].plot(daily_df['date'], daily_df['WUE_total'])\n", - "axs[1].set_ylabel('WUE\\_total (liters per kWh)')\n", - "axs[1].set_title('Daily Average Water Intensity')\n", - "\n", - "plt.tight_layout()\n", - "plt.show()\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2effef9c", - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "# Scatter plots of weather vs water intensity\n", - "\n", - "# Temperature vs WUE_total\n", - "plt.figure(figsize=(6,4))\n", - "plt.scatter(df['temp_c'], df['WUE_total'], alpha=0.5)\n", - "plt.title('Temperature (°C) vs WUE\\_total')\n", - "plt.xlabel('temp_c (°C)')\n", - "plt.ylabel('WUE\\_total (liters per kWh)')\n", - "plt.show()\n", - "\n", - "# Relative humidity vs WUE_total\n", - "plt.figure(figsize=(6,4))\n", - "plt.scatter(df['rh_pct'], df['WUE_total'], alpha=0.5)\n", - "plt.title('Relative Humidity (%) vs WUE\\_total')\n", - "plt.xlabel('rh_pct (%)')\n", - "plt.ylabel('WUE\\_total (liters per kWh)')\n", - "plt.show()\n", - "\n", - "# Coal share vs carbon intensity\n", - "plt.figure(figsize=(6,4))\n", - "# Compute coal share for scatter plot\n", - "coal_share = df['COAL'] / df['total_gen_mwh']\n", - "kgco2_per_kwh = df['co2_kg'] / df['total_gen_kwh']\n", - "plt.scatter(coal_share, kgco2_per_kwh, alpha=0.5)\n", - "plt.title('Coal Share vs Carbon Intensity')\n", - "plt.xlabel('Coal Share')\n", - "plt.ylabel('kgCO$_2$ per kWh')\n", - "plt.show()\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8f8d5c18", - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "# Correlation heatmap for fuel share percentages and intensities\n", - "fuel_cols = ['COAL', 'HYDRO', 'NATURALGAS', 'NUCLEAR', 'OTHER', 'PETROLEUM', 'SOLAR', 'WIND']\n", - "# Compute fuel shares\n", - "shares = pd.DataFrame()\n", - "for col in fuel_cols:\n", - " shares[col + '_share'] = df[col] / df['total_gen_mwh']\n", - "# Add intensities\n", - "shares['kgco2_per_kwh'] = df['co2_kg'] / df['total_gen_kwh']\n", - "shares['WUE_total'] = df['WUE_total']\n", - "\n", - "# Compute Spearman correlation\n", - "corr_matrix = shares.corr(method='spearman')\n", - "\n", - "# Plot heatmap using matplotlib\n", - "plt.figure(figsize=(10,8))\n", - "cax = plt.imshow(corr_matrix, cmap='viridis', aspect='auto')\n", - "plt.xticks(range(len(corr_matrix.columns)), corr_matrix.columns, rotation=90)\n", - "plt.yticks(range(len(corr_matrix.index)), corr_matrix.index)\n", - "plt.colorbar(cax)\n", - "plt.title('Spearman Correlation Heatmap (Fuel Shares & Intensities)')\n", - "plt.tight_layout()\n", - "plt.show()\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4083efa3", - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "# Median WUE_total by EGRID region\n", - "region_medians = df.groupby('EGRIDREGION')['WUE_total'].median().sort_values(ascending=False)\n", - "\n", - "plt.figure(figsize=(8,6))\n", - "plt.barh(region_medians.index, region_medians.values)\n", - "plt.xlabel('Median WUE\\_total (liters per kWh)')\n", - "plt.ylabel('EGRIDREGION')\n", - "plt.title('Median WUE\\_total by Region')\n", - "plt.gca().invert_yaxis()\n", - "plt.show()\n" - ] - }, - { - "cell_type": "markdown", - "id": "73bc5b67", - "metadata": {}, - "source": [ - "\n", - "### Observations\n", - "\n", - "* **Seasonal variation:** Water intensity shows clear seasonal patterns, peaking in the spring and dipping during the winter months【776473472252763†screenshot】. Carbon intensity tends to drop during the spring/early summer and rise toward late summer and autumn.\n", - "* **Fuel mix effects:** Regions with higher hydro generation share (e.g., NWPP, NYCW, CAMX) display higher median water intensities【265301069593362†screenshot】. The scatter plot of hydro share vs water intensity from previous analysis confirmed a nearly linear positive relationship【59990393566859†screenshot】. Conversely, coal share is strongly correlated with carbon intensity【810379427692406†screenshot】.\n", - "* **Weather influence:** Higher air temperatures and lower relative humidity correspond to higher water intensity due to increased cooling load, while precipitation appears weakly related (not shown here but observed in past EDA).\n", - "* **Regional disparity:** Western and northeastern regions have significantly higher water usage per kWh compared to southeastern regions【265301069593362†screenshot】.\n", - "\n", - "These insights inform which features and relationships to model in the predictive stage. We now proceed to prepare the data properly for modeling.\n" - ] - } - ], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7061e53 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,20 @@ +[project] +name = "carbon-water-scheduler-research" +version = "0.2.0" +description = "Reproducible research benchmark for carbon and water aware compute scheduling" +requires-python = ">=3.11" + +[tool.ruff] +line-length = 99 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "B", "UP", "RUF"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-q --strict-markers" +markers = [ + "unit: pure in-memory test", + "integration: end-to-end offline test", +] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index b6520b9..0000000 --- a/pytest.ini +++ /dev/null @@ -1,7 +0,0 @@ -[pytest] -testpaths = tests -addopts = -v --tb=short -markers = - unit: pure function tests, no file I/O - integration: tests that read real CSV/model files - api: tests that hit the FastAPI TestClient diff --git a/references.bib b/references.bib new file mode 100644 index 0000000..ed4d224 --- /dev/null +++ b/references.bib @@ -0,0 +1,99 @@ +@techreport{shehabi2024datacenter, + author = {Shehabi, Arman and Smith, Sarah Josephine and Hubbard, Alex and Newkirk, Alexander and Lei, Nuoa and Siddik, Md AbuBakar and Holecek, Billie and Koomey, Jonathan G. and Masanet, Eric R. and Sartor, Dale A.}, + title = {2024 United States Data Center Energy Usage Report}, + institution = {Lawrence Berkeley National Laboratory}, + year = {2024}, + doi = {10.71468/P1WC7Q} +} + +@article{radovanovic2023carbon, + author = {Radovanovic, Ana and others}, + title = {Carbon-Aware Computing for Datacenters}, + journal = {IEEE Transactions on Power Systems}, + volume = {38}, + number = {2}, + pages = {1270--1280}, + year = {2023}, + doi = {10.1109/TPWRS.2022.3173250} +} + +@inproceedings{wiesner2021wait, + author = {Wiesner, Philipp and Behnke, Ilja and Scheinert, Dominik and Gontarska, Kordian and Thamsen, Lauritz}, + title = {Let's Wait Awhile: How Temporal Workload Shifting Can Reduce Carbon Emissions in the Cloud}, + booktitle = {Proceedings of the 22nd International Middleware Conference}, + year = {2021}, + doi = {10.1145/3464298.3493399} +} + +@inproceedings{islam2014water, + author = {Islam, Mohammad A. and Ahmed, Kishwar and Ren, Shaolei and Quan, Gang}, + title = {Exploiting Temporal Diversity of Water Efficiency to Make Data Center Less Thirsty}, + booktitle = {11th International Conference on Autonomic Computing}, + pages = {145--152}, + year = {2014}, + url = {https://www.usenix.org/conference/icac14/technical-sessions/presentation/islam} +} + +@inproceedings{acun2023carbonexplorer, + author = {Acun, Bilge and Lee, Benjamin and Kazhamiaka, Fiodar and Maeng, Kiwan and Gupta, Udit and Chakkaravarthy, Manoj and Brooks, David and Wu, Carole-Jean}, + title = {Carbon Explorer: A Holistic Framework for Designing Carbon Aware Datacenters}, + booktitle = {Proceedings of the 28th ACM International Conference on Architectural Support for Programming Languages and Operating Systems}, + pages = {118--132}, + year = {2023}, + doi = {10.1145/3575693.3575754} +} + +@inproceedings{souza2023ecovisor, + author = {Souza, Abel and Bashir, Noman and Murillo, Jorge and Hanafy, Walid A. and Liang, Qianlin and Irwin, David and Shenoy, Prashant}, + title = {Ecovisor: A Virtual Energy System for Carbon-Efficient Applications}, + booktitle = {Proceedings of the 28th ACM International Conference on Architectural Support for Programming Languages and Operating Systems}, + pages = {252--265}, + year = {2023}, + doi = {10.1145/3575693.3575709} +} + +@article{hanafy2023carbonscaler, + author = {Hanafy, Walid A. and Liang, Qianlin and Bashir, Noman and Irwin, David and Shenoy, Prashant}, + title = {CarbonScaler: Leveraging Cloud Workload Elasticity for Optimizing Carbon-Efficiency}, + journal = {Proceedings of the ACM on Measurement and Analysis of Computing Systems}, + volume = {7}, + number = {3}, + year = {2023}, + doi = {10.1145/3626788} +} + +@inproceedings{qi2023shield, + author = {Qi, Sirui and Milojicic, Dejan and Bash, Cullen and Pasricha, Sudeep}, + title = {SHIELD: Sustainable Hybrid Evolutionary Learning Framework for Carbon, Wastewater, and Energy-Aware Data Center Management}, + booktitle = {14th International Green and Sustainable Computing Conference}, + year = {2023}, + doi = {10.1145/3634769.3634810} +} + +@inproceedings{li2024equitable, + author = {Li, Pengfei and Yang, Jianyi and Wierman, Adam and Ren, Shaolei}, + title = {Towards Environmentally Equitable AI via Geographical Load Balancing}, + booktitle = {15th ACM International Conference on Future and Sustainable Energy Systems}, + year = {2024}, + doi = {10.1145/3632775.3661938} +} + +@inproceedings{xu2025green, + author = {Xu, Kaiqiang and Sun, Decang and Tian, Han and Zhang, Junxue and Chen, Kai}, + title = {GREEN: Carbon-efficient Resource Scheduling for Machine Learning Clusters}, + booktitle = {22nd USENIX Symposium on Networked Systems Design and Implementation}, + pages = {999--1014}, + year = {2025}, + url = {https://www.usenix.org/conference/nsdi25/presentation/xu-kaiqiang} +} + +@article{li2025thirsty, + author = {Li, Pengfei and Yang, Jianyi and Islam, Mohammad A. and Ren, Shaolei}, + title = {Making AI Less Thirsty}, + journal = {Communications of the ACM}, + volume = {68}, + number = {7}, + pages = {54--61}, + year = {2025}, + doi = {10.1145/3724499} +} diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..9acee2a --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +pytest==9.1.1 +ruff==0.16.5 diff --git a/requirements.txt b/requirements.txt index eca81e8..ee5137d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,22 +1,5 @@ -# Pinned to stable releases that support Python 3.9. -pandas==2.3.3 -numpy==2.0.2 -matplotlib==3.9.4 -seaborn==0.13.2 -scikit-learn==1.6.1 -xgboost==2.1.4 -prophet==1.3.0 -pyarrow==21.0.0 - -# Project/runtime utilities. -joblib==1.5.3 -openpyxl==3.1.5 -jupyter==1.1.1 -requests==2.32.5 -statsmodels==0.14.6 -pulp==3.3.0 -plotly==6.6.0 -fastapi==0.128.8 -uvicorn==0.39.0 -shap==0.47.2 -optuna==4.3.0 +numpy==2.4.6 +pandas==3.0.5 +matplotlib==3.11.1 +requests==2.34.2 +defusedxml==0.7.1 diff --git a/results/README.md b/results/README.md new file mode 100644 index 0000000..e679746 --- /dev/null +++ b/results/README.md @@ -0,0 +1,15 @@ +# Generated Results + +These files are generated by `python -m src.experiments` with the configuration +in `run_metadata.json`. +`SHA256SUMS` records the generated artifact digests for transfer checks. +Verify it from this directory with `shasum -a 256 -c SHA256SUMS`. + +The committed benchmark is synthetic. Its percentages are useful for testing the +method and code, not for estimating the impact of a real data center portfolio. +See `docs/RESULTS.md` for interpretation and `docs/THREATS_TO_VALIDITY.md` for +limitations. + +Do not edit the CSV or JSON outputs by hand. Change the source or experiment +configuration, rerun the command, review the seed-level tables, and then update +the written interpretation. diff --git a/results/SHA256SUMS b/results/SHA256SUMS new file mode 100644 index 0000000..ec55a60 --- /dev/null +++ b/results/SHA256SUMS @@ -0,0 +1,9 @@ +3639bb2f8ac7e9feb7487fb8529dc2983a01ce4facee320952b8cdcd27092fe6 ../docs/figures/benchmark_overview.png +cc5a877b903b7bd50b8d015037831c361f70dfceacd75736ab5c2aeebc215402 example_schedule.csv +89b472dd83248c1c16dc9738b8c735615c2edc1eee4652eb36ddb4be2b994a32 run_metadata.json +10a0f871ee3c59d79310e8a4f6ddc7720ff41c64df363a5253c8ff8949d54958 paired_comparisons_by_seed.csv +c8c8532603152f4500bbd5acf6cdb234af67e405d9b1511c03d7db25c3200586 benchmark_summary.csv +9d33dbfbbbfe4c56f7a7881a0dd318b874304358cfda7ac435852015840a69e4 policy_summary_by_seed.csv +cc79a5446b6cd5f05ae30771d8b54854be1068e1bf429d0766b77263efcdba84 forecast_robustness.csv +9eb084432df179e48eccdd4d74bbf9dec737e9db69121aeafb1920e5e372d49b ../docs/figures/forecast_robustness.png +006db4cbf8fcb438db2dad46a0772eb8f9d01a57d761b812c98d8099292a2432 ../docs/figures/example_schedule.png diff --git a/results/benchmark_summary.csv b/results/benchmark_summary.csv new file mode 100644 index 0000000..f0c5e3b --- /dev/null +++ b/results/benchmark_summary.csv @@ -0,0 +1,4 @@ +baseline,co2_reduction_pct_mean,co2_reduction_pct_ci95,water_reduction_pct_mean,water_reduction_pct_ci95,scarcity_weighted_water_reduction_pct_mean,scarcity_weighted_water_reduction_pct_ci95,baseline_coverage_mean,baseline_coverage_ci95,proposed_coverage_mean,proposed_coverage_ci95,coverage_gap_percentage_points_mean,coverage_gap_percentage_points_ci95,common_jobs_mean,common_jobs_ci95,common_it_energy_kwh_mean,common_it_energy_kwh_ci95,seeds +carbon_greedy,-46.88448467,2.11249523,52.75314875,1.46869253,39.17078223,1.17262926,1.00000000,0.00000000,1.00000000,0.00000000,0.00000000,0.00000000,180.00000000,0.00000000,19073.60000000,721.27723043,10.00000000 +immediate_local,23.93681288,1.92347442,28.59738564,1.89421060,20.08039078,1.27511200,1.00000000,0.00000000,1.00000000,0.00000000,0.00000000,0.00000000,180.00000000,0.00000000,19073.60000000,721.27723043,10.00000000 +random,22.31396114,2.18234172,29.49830818,2.80955726,20.78337348,1.87185039,1.00000000,0.00000000,1.00000000,0.00000000,0.00000000,0.00000000,180.00000000,0.00000000,19073.60000000,721.27723043,10.00000000 diff --git a/results/example_schedule.csv b/results/example_schedule.csv new file mode 100644 index 0000000..d0e294a --- /dev/null +++ b/results/example_schedule.csv @@ -0,0 +1,26 @@ +job_id,origin_city,power_demand_kw,duration_hours,earliest_start,deadline,priority,assigned_dc_id,assigned_city,scheduled_start,scheduled_end,scheduler_score,expected_co2_kg,expected_water_liters,scarcity_weighted_water_liters,relocated,scheduled,failure_reason +job_042_0169,Zone West,23.00000000,1,2025-01-01 03:00:00,2025-01-02 03:00:00,0,dc_2,Zone West,2025-01-01 06:00:00,2025-01-01 07:00:00,0.07393650,5.10240097,6.62324628,11.92184330,0,1, +job_042_0068,Zone East,31.00000000,5,2025-01-01 10:00:00,2025-01-01 18:00:00,0,dc_2,Zone West,2025-01-01 10:00:00,2025-01-01 15:00:00,0.28535308,50.83680189,51.38439362,92.49190851,1,1, +job_042_0000,Zone South,19.00000000,6,2025-01-01 12:00:00,2025-01-02 15:00:00,0,dc_2,Zone West,2025-01-02 05:00:00,2025-01-02 11:00:00,0.19338336,27.94088946,34.28199374,61.70758872,1,1, +job_042_0132,Zone South,44.00000000,1,2025-01-01 12:00:00,2025-01-02 08:00:00,0,dc_2,Zone West,2025-01-02 06:00:00,2025-01-02 07:00:00,0.17858367,9.95071348,12.66802332,22.80244198,1,1, +job_042_0117,Zone South,21.00000000,3,2025-01-01 19:00:00,2025-01-02 01:00:00,0,dc_2,Zone West,2025-01-01 22:00:00,2025-01-02 01:00:00,0.40226954,27.27526555,24.69166837,44.44500306,1,1, +job_042_0177,Zone West,16.00000000,6,2025-01-01 22:00:00,2025-01-02 09:00:00,0,dc_2,Zone West,2025-01-02 03:00:00,2025-01-02 09:00:00,0.08425091,23.99753756,29.70005128,53.46009231,0,1, +job_042_0022,Zone West,34.00000000,3,2025-01-02 01:00:00,2025-01-03 03:00:00,0,dc_2,Zone West,2025-01-02 06:00:00,2025-01-02 09:00:00,0.06603484,23.20924421,30.28712264,54.51682075,0,1, +job_042_0101,Zone East,35.00000000,3,2025-01-02 03:00:00,2025-01-02 12:00:00,0,dc_2,Zone West,2025-01-02 06:00:00,2025-01-02 09:00:00,0.11151183,23.89186904,31.17792036,56.12025665,1,1, +job_042_0110,Zone West,22.00000000,5,2025-01-02 14:00:00,2025-01-03 14:00:00,0,dc_2,Zone West,2025-01-03 05:00:00,2025-01-03 10:00:00,0.06691208,25.36552635,32.77841168,59.00114103,0,1, +job_042_0087,Zone West,18.00000000,3,2025-01-02 19:00:00,2025-01-03 22:00:00,0,dc_2,Zone West,2025-01-03 06:00:00,2025-01-03 09:00:00,0.06281113,12.02447885,15.87519080,28.57534345,0,1, +job_042_0065,Zone South,33.00000000,1,2025-01-02 20:00:00,2025-01-03 04:00:00,0,dc_2,Zone West,2025-01-03 03:00:00,2025-01-03 04:00:00,0.24703428,10.06021725,11.00134362,19.80241852,1,1, +job_042_0168,Zone West,15.00000000,2,2025-01-02 22:00:00,2025-01-03 23:00:00,0,dc_2,Zone West,2025-01-03 06:00:00,2025-01-03 08:00:00,0.07367824,6.59889924,8.74204814,15.73568665,0,1, +job_042_0138,Zone South,35.00000000,6,2025-01-02 23:00:00,2025-01-04 04:00:00,0,dc_2,Zone West,2025-01-03 05:00:00,2025-01-03 11:00:00,0.18980074,49.98360565,62.90340834,113.22613502,1,1, +job_042_0062,Zone South,18.00000000,1,2025-01-03 02:00:00,2025-01-03 22:00:00,0,dc_2,Zone West,2025-01-03 06:00:00,2025-01-03 07:00:00,0.16263042,3.80456416,5.25177123,9.45318821,1,1, +job_042_0004,Zone East,39.00000000,3,2025-01-03 11:00:00,2025-01-03 19:00:00,0,dc_2,Zone West,2025-01-03 11:00:00,2025-01-03 14:00:00,0.30175712,39.26096973,38.50050564,69.30091014,1,1, +job_042_0003,Zone North,18.00000000,5,2025-01-03 12:00:00,2025-01-04 15:00:00,0,dc_2,Zone West,2025-01-04 04:00:00,2025-01-04 09:00:00,0.18577951,20.91405136,26.95380621,48.51685118,1,1, +job_042_0139,Zone East,39.00000000,5,2025-01-03 12:00:00,2025-01-04 05:00:00,0,dc_2,Zone West,2025-01-04 00:00:00,2025-01-04 05:00:00,0.27134595,60.75747528,68.38345073,123.09021131,1,1, +job_042_0072,Zone East,37.00000000,2,2025-01-03 15:00:00,2025-01-04 14:00:00,0,dc_2,Zone West,2025-01-04 06:00:00,2025-01-04 08:00:00,0.17706618,16.13318370,21.90442532,39.42796558,1,1, +job_042_0157,Zone South,38.00000000,2,2025-01-04 04:00:00,2025-01-04 10:00:00,0,dc_2,Zone West,2025-01-04 06:00:00,2025-01-04 08:00:00,0.10797890,16.56921569,22.49643682,40.49358627,1,1, +job_042_0136,Zone South,22.00000000,5,2025-01-04 11:00:00,2025-01-04 21:00:00,0,dc_2,Zone West,2025-01-04 11:00:00,2025-01-04 16:00:00,0.34008951,38.56271297,37.18162395,66.92692311,1,1, +job_042_0084,Zone South,37.00000000,6,2025-01-04 14:00:00,2025-01-05 10:00:00,0,dc_2,Zone West,2025-01-05 04:00:00,2025-01-05 10:00:00,0.18292036,47.35585890,68.42656271,123.16781287,1,1, +job_042_0059,Zone South,41.00000000,4,2025-01-04 22:00:00,2025-01-05 16:00:00,0,dc_2,Zone West,2025-01-05 05:00:00,2025-01-05 09:00:00,0.18308021,33.58177159,50.17767867,90.31982160,1,1, +job_042_0074,Zone South,28.00000000,4,2025-01-04 22:00:00,2025-01-05 14:00:00,0,dc_2,Zone West,2025-01-05 05:00:00,2025-01-05 09:00:00,0.18308021,22.93389279,34.26768299,61.68182938,1,1, +job_042_0097,Zone West,18.00000000,5,2025-01-04 22:00:00,2025-01-05 19:00:00,0,dc_2,Zone West,2025-01-05 05:00:00,2025-01-05 10:00:00,0.07790342,18.73845327,27.57610698,49.63699256,0,1, +job_042_0088,Zone South,27.00000000,4,2025-01-05 09:00:00,2025-01-06 03:00:00,0,dc_2,Zone West,2025-01-05 09:00:00,2025-01-05 13:00:00,0.24127574,28.31761721,33.15141854,59.67255337,1,1, diff --git a/results/forecast_robustness.csv b/results/forecast_robustness.csv new file mode 100644 index 0000000..1494728 --- /dev/null +++ b/results/forecast_robustness.csv @@ -0,0 +1,7 @@ +noise_level,forecast_rmse_mean,forecast_rmse_ci95,carbon_delta_pct_mean,carbon_delta_pct_ci95,coverage_mean,coverage_ci95,paired_jobs_mean,repetitions +0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,1.00000000,0.00000000,180.00000000,10.00000000 +0.05000000,0.01685129,0.00026484,0.63939671,0.20591409,1.00000000,0.00000000,180.00000000,10.00000000 +0.10000000,0.03417225,0.00109864,1.23210677,0.17460054,1.00000000,0.00000000,180.00000000,10.00000000 +0.20000000,0.06838686,0.00062730,3.65387962,0.53959767,1.00000000,0.00000000,180.00000000,10.00000000 +0.35000000,0.11961951,0.00243991,11.63000859,1.56038449,1.00000000,0.00000000,180.00000000,10.00000000 +0.50000000,0.16813510,0.00264690,21.41513613,1.70923390,1.00000000,0.00000000,180.00000000,10.00000000 diff --git a/results/paired_comparisons_by_seed.csv b/results/paired_comparisons_by_seed.csv new file mode 100644 index 0000000..05e992b --- /dev/null +++ b/results/paired_comparisons_by_seed.csv @@ -0,0 +1,31 @@ +seed,baseline,baseline_coverage,proposed_coverage,coverage_gap_percentage_points,common_jobs,common_it_energy_kwh,co2_reduction_pct,water_reduction_pct,scarcity_weighted_water_reduction_pct +42,immediate_local,1.00000000,1.00000000,0.00000000,180.00000000,17999.00000000,28.65809016,28.57049746,20.45931391 +42,carbon_greedy,1.00000000,1.00000000,0.00000000,180.00000000,17999.00000000,-39.61306712,50.71410341,37.91004765 +42,random,1.00000000,1.00000000,0.00000000,180.00000000,17999.00000000,24.59141960,28.06271634,20.12142493 +43,immediate_local,1.00000000,1.00000000,0.00000000,180.00000000,20791.00000000,21.18168348,29.76088937,21.12020550 +43,carbon_greedy,1.00000000,1.00000000,0.00000000,180.00000000,20791.00000000,-46.32491058,51.20233221,37.56823883 +43,random,1.00000000,1.00000000,0.00000000,180.00000000,20791.00000000,21.61195176,28.80334390,20.08409764 +44,immediate_local,1.00000000,1.00000000,0.00000000,180.00000000,19480.00000000,19.06299764,28.08595224,19.92186095 +44,carbon_greedy,1.00000000,1.00000000,0.00000000,180.00000000,19480.00000000,-44.92454499,51.41391939,38.29122074 +44,random,1.00000000,1.00000000,0.00000000,180.00000000,19480.00000000,24.57167607,25.85008132,18.76420725 +45,immediate_local,1.00000000,1.00000000,0.00000000,180.00000000,19443.00000000,24.17506012,27.85458699,19.61499737 +45,carbon_greedy,1.00000000,1.00000000,0.00000000,180.00000000,19443.00000000,-46.85898012,50.24845612,37.02031348 +45,random,1.00000000,1.00000000,0.00000000,180.00000000,19443.00000000,19.26239014,29.02721243,20.22158232 +46,immediate_local,1.00000000,1.00000000,0.00000000,180.00000000,17167.00000000,27.97169073,31.34137166,22.54608050 +46,carbon_greedy,1.00000000,1.00000000,0.00000000,180.00000000,17167.00000000,-49.66615129,55.81706962,42.01089722 +46,random,1.00000000,1.00000000,0.00000000,180.00000000,17167.00000000,27.16222971,32.23258897,22.97070821 +47,immediate_local,1.00000000,1.00000000,0.00000000,180.00000000,19383.00000000,24.80125328,21.26377474,15.10604404 +47,carbon_greedy,1.00000000,1.00000000,0.00000000,180.00000000,19383.00000000,-47.60822694,51.50547525,38.24682998 +47,random,1.00000000,1.00000000,0.00000000,180.00000000,19383.00000000,22.28354729,21.80210266,15.34991612 +48,immediate_local,1.00000000,1.00000000,0.00000000,180.00000000,19079.00000000,20.75697537,32.80481145,22.23524793 +48,carbon_greedy,1.00000000,1.00000000,0.00000000,180.00000000,19079.00000000,-49.44428267,56.74456945,41.95044220 +48,random,1.00000000,1.00000000,0.00000000,180.00000000,19079.00000000,15.32838984,36.24974603,24.88278240 +49,immediate_local,1.00000000,1.00000000,0.00000000,180.00000000,20771.00000000,24.14087243,27.54929287,19.09742370 +49,carbon_greedy,1.00000000,1.00000000,0.00000000,180.00000000,20771.00000000,-45.63586966,51.44108500,38.04828100 +49,random,1.00000000,1.00000000,0.00000000,180.00000000,20771.00000000,25.50722231,25.67409553,18.23561072 +50,immediate_local,1.00000000,1.00000000,0.00000000,180.00000000,18377.00000000,22.64750129,28.83213178,20.20195577 +50,carbon_greedy,1.00000000,1.00000000,0.00000000,180.00000000,18377.00000000,-46.33084130,52.99922463,39.31560031 +50,random,1.00000000,1.00000000,0.00000000,180.00000000,18377.00000000,19.49288329,35.59285027,24.88189130 +51,immediate_local,1.00000000,1.00000000,0.00000000,180.00000000,18246.00000000,25.97200431,29.91054784,20.50077809 +51,carbon_greedy,1.00000000,1.00000000,0.00000000,180.00000000,18246.00000000,-52.43797205,55.44525244,41.34595086 +51,random,1.00000000,1.00000000,0.00000000,180.00000000,18246.00000000,23.32790138,31.68834438,22.32151395 diff --git a/results/policy_summary_by_seed.csv b/results/policy_summary_by_seed.csv new file mode 100644 index 0000000..e2d3b07 --- /dev/null +++ b/results/policy_summary_by_seed.csv @@ -0,0 +1,41 @@ +seed,policy,total_jobs,scheduled_jobs,coverage,total_co2_kg,total_water_liters,total_scarcity_weighted_water_liters,relocation_rate +42,proposed,180.00000000,180.00000000,1.00000000,5080.35389962,8731.94450633,13082.37108102,0.62777778 +42,immediate_local,180.00000000,180.00000000,1.00000000,7121.13526365,12224.56295558,16447.39531855,0.00000000 +42,carbon_greedy,180.00000000,180.00000000,1.00000000,3638.88137728,17716.92332093,21070.02918649,0.78333333 +42,random,180.00000000,180.00000000,1.00000000,6737.10322178,12138.27387090,16377.82229950,0.77222222 +43,proposed,180.00000000,180.00000000,1.00000000,6303.47774088,9548.86673177,14658.98540794,0.59444444 +43,immediate_local,180.00000000,180.00000000,1.00000000,7997.47827603,13594.80017127,18583.95486668,0.00000000 +43,carbon_greedy,180.00000000,180.00000000,1.00000000,4307.86372315,19568.28505067,23480.01262997,0.72222222 +43,random,180.00000000,180.00000000,1.00000000,8041.37605490,13411.95956992,18343.01431206,0.76666667 +44,proposed,180.00000000,180.00000000,1.00000000,5780.51905809,9215.85168651,13923.62933704,0.52777778 +44,immediate_local,180.00000000,180.00000000,1.00000000,7141.99796085,12815.09242497,17387.55358526,0.00000000 +44,carbon_greedy,180.00000000,180.00000000,1.00000000,3988.64047384,18968.09038886,22563.44964805,0.74444444 +44,random,180.00000000,180.00000000,1.00000000,7663.59207872,12428.67402990,17139.77160281,0.76666667 +45,proposed,180.00000000,180.00000000,1.00000000,5666.82772484,9211.02991736,13968.34100255,0.56666667 +45,immediate_local,180.00000000,180.00000000,1.00000000,7473.56705316,12767.31192328,17376.79983250,0.00000000 +45,carbon_greedy,180.00000000,180.00000000,1.00000000,3858.68655787,18514.05845770,22179.12119549,0.71666667 +45,random,180.00000000,180.00000000,1.00000000,7018.82026798,12978.25579799,17508.92209766,0.81666667 +46,proposed,180.00000000,180.00000000,1.00000000,4922.34944688,7631.92940563,11831.70343957,0.59444444 +46,immediate_local,180.00000000,180.00000000,1.00000000,6833.90946786,11115.76154431,15275.79690728,0.00000000 +46,carbon_greedy,180.00000000,180.00000000,1.00000000,3288.88623418,17273.47946534,20403.32212970,0.76111111 +46,random,180.00000000,180.00000000,1.00000000,6757.96283642,11261.94625018,15360.00548009,0.71111111 +47,proposed,180.00000000,180.00000000,1.00000000,5752.35396800,9099.32945447,13806.74800611,0.54444444 +47,immediate_local,180.00000000,180.00000000,1.00000000,7649.53435960,11556.72554023,16263.52294420,0.00000000 +47,carbon_greedy,180.00000000,180.00000000,1.00000000,3897.04157219,18763.62228688,22357.95830736,0.80000000 +47,random,180.00000000,180.00000000,1.00000000,7401.71967130,11636.28404888,16310.37722894,0.80000000 +48,proposed,180.00000000,180.00000000,1.00000000,5867.41670603,8119.35100658,12943.29826183,0.59444444 +48,immediate_local,180.00000000,180.00000000,1.00000000,7404.33209563,12083.23271706,16644.17093576,0.00000000 +48,carbon_greedy,180.00000000,180.00000000,1.00000000,3926.15669278,18770.70902818,22296.97994619,0.78888889 +48,random,180.00000000,180.00000000,1.00000000,6929.61512704,12736.18613467,17230.80097315,0.75555556 +49,proposed,180.00000000,180.00000000,1.00000000,6248.35366294,9642.20078924,14700.24451161,0.61111111 +49,immediate_local,180.00000000,180.00000000,1.00000000,8236.78555694,13308.63585962,18170.30456169,0.00000000 +49,carbon_greedy,180.00000000,180.00000000,1.00000000,4290.39472038,19856.70558905,23728.54982672,0.75555556 +49,random,180.00000000,180.00000000,1.00000000,8387.86504754,12972.86707521,17978.78592538,0.71666667 +50,proposed,180.00000000,180.00000000,1.00000000,5337.85232733,8332.38137873,12852.53776346,0.56666667 +50,immediate_local,180.00000000,180.00000000,1.00000000,6900.68506710,11708.06655691,16106.33178809,0.00000000 +50,carbon_greedy,180.00000000,180.00000000,1.00000000,3647.79719704,17728.17855343,21179.31103931,0.76111111 +50,random,180.00000000,180.00000000,1.00000000,6630.28629694,12937.04412309,17109.77284425,0.74444444 +51,proposed,180.00000000,180.00000000,1.00000000,5387.87394093,8006.49832915,12498.36199315,0.61666667 +51,immediate_local,180.00000000,180.00000000,1.00000000,7278.15725744,11423.25711199,15721.36392397,0.00000000 +51,carbon_greedy,180.00000000,180.00000000,1.00000000,3534.46970483,17970.02287574,21308.60899905,0.75555556 +51,random,180.00000000,180.00000000,1.00000000,7027.16377662,11720.54498896,16089.86300914,0.74444444 diff --git a/results/run_metadata.json b/results/run_metadata.json new file mode 100644 index 0000000..94e66ba --- /dev/null +++ b/results/run_metadata.json @@ -0,0 +1,14 @@ +{ + "benchmark": "synthetic-v1", + "claim_scope": "simulation only", + "forecast_repetitions": 10, + "hours_per_seed": 168, + "jobs_per_seed": 180, + "seed_count": 10, + "seed_start": 42, + "weights": { + "carbon": 0.55, + "relocation": 0.1, + "water": 0.35 + } +} diff --git a/src/__init__.py b/src/__init__.py index ec91126..e92f56d 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -1,2 +1 @@ -"""Core package for the Capstone project pipeline.""" - +"""Research package for carbon and water aware compute scheduling.""" diff --git a/src/api.py b/src/api.py deleted file mode 100644 index 252ef5e..0000000 --- a/src/api.py +++ /dev/null @@ -1,281 +0,0 @@ -from __future__ import annotations - -from functools import lru_cache -from typing import Any, Optional - -import pandas as pd -from fastapi import FastAPI, HTTPException -from fastapi.middleware.cors import CORSMiddleware -from pydantic import BaseModel, Field - -from .app_backend import ( - BatchSchedulerInputs, - SchedulerInputs, - SustainabilitySchedulingBackend, - resolve_default_paths, -) -from .explain import compute_shap -from .baselines import ( - baseline_random, - baseline_nearest_neighbor, - baseline_time_of_day_heuristic, - BaselineResult, -) - - -class SimulationRequest(BaseModel): - priority: str = Field(default="medium", pattern="^(low|medium|high)$") - alpha: float = Field(default=1.0, ge=0.0, le=5.0) - beta: float = Field(default=1.0, ge=0.0, le=5.0) - gamma: float = Field(default=1.0, ge=0.0, le=5.0) - time: str - latency_sensitivity: float = Field(default=0.55, ge=0.0, le=1.0) - workload_size: Optional[float] = Field(default=None, gt=0.0) - origin_city: Optional[str] = None - top_k: int = Field(default=4, ge=1, le=8) - - -class BatchSimulationRequest(BaseModel): - priority: str = Field(default="medium", pattern="^(low|medium|high)$") - alpha: float = Field(default=1.0, ge=0.0, le=5.0) - beta: float = Field(default=1.0, ge=0.0, le=5.0) - gamma: float = Field(default=1.0, ge=0.0, le=5.0) - time: str - latency_sensitivity: float = Field(default=0.55, ge=0.0, le=1.0) - batch_size: int = Field(default=100, ge=50, le=200) - - -class BaselineComparisonRequest(BaseModel): - priority: str = Field(default="medium", pattern="^(low|medium|high)$") - time: str - batch_size: int = Field(default=50, ge=10, le=200) - - -app = FastAPI( - title="Sustainability-Aware Scheduling API", - version="1.0.0", - description="Simulation API for interactive workload routing demonstrations.", -) - -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - - -@lru_cache(maxsize=1) -def get_backend() -> SustainabilitySchedulingBackend: - defaults = resolve_default_paths() - return SustainabilitySchedulingBackend( - data_path=defaults["data_path"], - jobs_path=defaults["jobs_path"], - dc_config_path=defaults["dc_config_path"], - coords_path=defaults["coords_path"], - ) - - -def _serialize_nodes(nodes: pd.DataFrame) -> list[dict[str, Any]]: - if nodes.empty: - return [] - - frame = nodes.copy() - for column in [ - "avg_co2_intensity", - "avg_wue", - "avg_scarcity", - "records", - "latitude", - "longitude", - "has_data_center", - ]: - if column in frame.columns: - frame[column] = pd.to_numeric(frame[column], errors="coerce") - - return frame.fillna("").to_dict(orient="records") - - -def _route_label(path: dict[str, Any]) -> str: - return " -> ".join(path.get("path", [])) - - -def _choose_selected_path(paths: list[dict[str, Any]]) -> dict[str, Any] | None: - if not paths: - return None - - for path in paths: - if str(path.get("type")) == "balanced": - return path - return min(paths, key=lambda item: float(item.get("score", 0.0))) - - -@app.get("/health") -def healthcheck() -> dict[str, str]: - return {"status": "ok"} - - -@app.get("/context") -def context() -> dict[str, Any]: - backend = get_backend() - app_context = backend.context() - return { - "origin_city": app_context["default_origin_city"], - "time_min": pd.Timestamp(app_context["time_min"]).isoformat(), - "time_max": pd.Timestamp(app_context["time_max"]).isoformat(), - "default_time": pd.Timestamp(app_context["default_scheduler_timestamp"]).isoformat(), - "default_workload_size": float(app_context["default_workload_size"]), - "default_batch_size": int(app_context["default_batch_size"]), - "batch_size_min": int(app_context["batch_size_min"]), - "batch_size_max": int(app_context["batch_size_max"]), - "priority_counts": app_context["priority_counts"], - } - - -@app.post("/simulate") -def simulate(request: SimulationRequest) -> dict[str, Any]: - backend = get_backend() - app_context = backend.context() - - try: - start_time = pd.Timestamp(request.time).floor("h") - except Exception as exc: # pragma: no cover - request validation guard. - raise HTTPException(status_code=422, detail=f"Invalid simulation time: {exc}") from exc - - workload_size = ( - float(request.workload_size) - if request.workload_size is not None - else float(app_context["default_workload_size"]) - ) - origin_city = request.origin_city or str(app_context["default_origin_city"]) - - paths = backend.run_scheduler( - SchedulerInputs( - origin_city=origin_city, - priority=request.priority, - latency_sensitivity=float(request.latency_sensitivity), - workload_size=workload_size, - alpha=float(request.alpha), - beta=float(request.beta), - gamma=float(request.gamma), - start_time=start_time, - top_k=int(request.top_k), - ) - ) - - if not paths: - window_end = start_time + pd.Timedelta(hours=24) - return { - "time": start_time.isoformat(), - "origin_city": origin_city, - "paths": [], - "selected_path": None, - "baseline_path": None, - "metrics": { - "co2_reduction_pct": 0.0, - "water_reduction_pct": 0.0, - "latency_delta_ms": 0.0, - "latency_delta_pct": 0.0, - }, - "insight": backend.build_path_story({}, []), - "nodes": _serialize_nodes(backend.build_scheduler_nodes(start_time, window_end)), - } - - selected_path = _choose_selected_path(paths) - baseline_path = backend.select_baseline_path(paths) - metrics = backend.compare_paths(selected_path, baseline_path) - insight = backend.build_path_story(selected_path, paths, baseline_path=baseline_path) - - window_start = min(pd.Timestamp(path["scheduled_start"]) for path in paths) - window_end = max(pd.Timestamp(path["scheduled_end"]) for path in paths) - nodes = backend.build_scheduler_nodes(window_start, window_end) - - enriched_paths = [] - for path in paths: - enriched_path = dict(path) - enriched_path["route"] = _route_label(path) - enriched_paths.append(enriched_path) - - selected_payload = dict(selected_path) - selected_payload["route"] = _route_label(selected_path) - baseline_payload = dict(baseline_path) if baseline_path is not None else None - if baseline_payload is not None: - baseline_payload["route"] = _route_label(baseline_payload) - - return { - "time": start_time.isoformat(), - "origin_city": origin_city, - "paths": enriched_paths, - "selected_path": selected_payload, - "baseline_path": baseline_payload, - "metrics": metrics, - "insight": insight, - "nodes": _serialize_nodes(nodes), - } - - -@app.post("/simulate-batch") -def simulate_batch(request: BatchSimulationRequest) -> dict[str, Any]: - backend = get_backend() - try: - start_time = pd.Timestamp(request.time).floor("h") - except Exception as exc: # pragma: no cover - request validation guard. - raise HTTPException(status_code=422, detail=f"Invalid simulation time: {exc}") from exc - - payload = backend.run_batch_scheduler( - BatchSchedulerInputs( - priority=request.priority, - latency_sensitivity=float(request.latency_sensitivity), - alpha=float(request.alpha), - beta=float(request.beta), - gamma=float(request.gamma), - start_time=start_time, - batch_size=int(request.batch_size), - ) - ) - return payload - - -@app.get("/explain") -def explain(city: str, target: str, time: str) -> dict[str, Any]: - """ - Return SHAP feature importances for a city's CO2 or WUE prediction. - - Query params: - city — data center city name - target — 'co2' or 'wue' - time — ISO 8601 timestamp - """ - if target not in ("co2", "wue"): - raise HTTPException(status_code=422, detail="target must be 'co2' or 'wue'") - try: - return compute_shap(target=target, city=city, time=time) - except (FileNotFoundError, ModuleNotFoundError) as exc: - raise HTTPException(status_code=503, detail=str(exc)) from exc - except Exception as exc: - raise HTTPException(status_code=500, detail=f"Explanation failed: {exc}") from exc - - -@app.post("/compare-baselines") -def compare_baselines(request: BaselineComparisonRequest) -> dict[str, Any]: - """ - Compare optimal scheduler against three baselines (random, nearest, time-of-day). - - Returns carbon and water footprint for each strategy on the same batch. - """ - backend = get_backend() - try: - start_time = pd.Timestamp(request.time).floor("h") - except Exception as exc: # pragma: no cover - request validation guard. - raise HTTPException(status_code=422, detail=f"Invalid time: {exc}") from exc - - try: - results = backend.run_baseline_comparison( - start_time=start_time, - batch_size=int(request.batch_size), - priority=request.priority, - ) - return results - except Exception as exc: - raise HTTPException(status_code=500, detail=f"Comparison failed: {exc}") from exc diff --git a/src/app_backend.py b/src/app_backend.py deleted file mode 100644 index 4fb2bca..0000000 --- a/src/app_backend.py +++ /dev/null @@ -1,1200 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path -import math -from typing import Any - -import pandas as pd - -try: - from .data_loader import PROJECT_ROOT, load_dataset - from .scheduling import ( - DEFAULT_DC_CONFIG_PATH, - DEFAULT_JOBS_PATH, - build_job_record, - compare_summaries, - list_job_candidates, - load_dc_config, - load_jobs_dataset, - prepare_scheduler_state, - prepare_environment_frame, - schedule_jobs, - schedule_jobs_naive, - summarize_schedule, - ) - from .baselines import ( - baseline_random, - baseline_nearest_neighbor, - baseline_time_of_day_heuristic, - ) -except ImportError: - from data_loader import PROJECT_ROOT, load_dataset - from scheduling import ( - DEFAULT_DC_CONFIG_PATH, - DEFAULT_JOBS_PATH, - build_job_record, - compare_summaries, - list_job_candidates, - load_dc_config, - load_jobs_dataset, - prepare_scheduler_state, - prepare_environment_frame, - schedule_jobs, - schedule_jobs_naive, - summarize_schedule, - ) - from baselines import ( - baseline_random, - baseline_nearest_neighbor, - baseline_time_of_day_heuristic, - ) - - -DEFAULT_COORDINATES_PATH = PROJECT_ROOT / "data" / "reference" / "city_coordinates.csv" -DEFAULT_DATA_CANDIDATES = [ - PROJECT_ROOT / "data" / "base_data_with_metrics.parquet", - PROJECT_ROOT / "data" / "base_data_with_metrics.csv", - Path.home() / "Downloads" / "Archive 3.zip", - PROJECT_ROOT / "data" / "sample_dataset.xlsx", -] -PATH_TYPES = ["balanced", "low_carbon", "low_water", "low_latency"] -PATH_TYPE_LABELS = { - "balanced": "Balanced", - "low_carbon": "Low Carbon", - "low_water": "Low Water", - "low_latency": "Low Latency", -} -PRIORITY_TO_CODE = {"high": 0, "medium": 1, "low": 2} -INTERACTIVE_HORIZON_HOURS = 24 - - -def resolve_default_paths() -> dict[str, str]: - data_path = SustainabilitySchedulingBackend._resolve_data_path(None) - return { - "data_path": str(data_path), - "jobs_path": str(DEFAULT_JOBS_PATH), - "dc_config_path": str(DEFAULT_DC_CONFIG_PATH), - "coords_path": str(DEFAULT_COORDINATES_PATH), - } - - -@dataclass(frozen=True) -class SchedulerInputs: - origin_city: str - priority: str - latency_sensitivity: float - workload_size: float - alpha: float - beta: float - gamma: float - start_time: pd.Timestamp - top_k: int = 4 - - -@dataclass(frozen=True) -class BatchSchedulerInputs: - priority: str - latency_sensitivity: float - alpha: float - beta: float - gamma: float - start_time: pd.Timestamp - batch_size: int = 100 - - -@dataclass(frozen=True) -class OverviewSnapshot: - metrics: dict[str, float] - nodes: pd.DataFrame - start_time: pd.Timestamp - end_time: pd.Timestamp - - -class SustainabilitySchedulingBackend: - """Backend façade for API-backed scheduling simulations.""" - - def __init__( - self, - data_path: str | Path | None = None, - jobs_path: str | Path | None = None, - dc_config_path: str | Path | None = None, - coords_path: str | Path | None = None, - ) -> None: - self.data_path = self._resolve_data_path(data_path) - self.jobs_path = self._resolve_path(jobs_path, DEFAULT_JOBS_PATH) - self.dc_config_path = self._resolve_path(dc_config_path, DEFAULT_DC_CONFIG_PATH) - self.coords_path = self._resolve_path(coords_path, DEFAULT_COORDINATES_PATH) - - self.env_df = self._load_environment_frame(self.data_path, self.dc_config_path) - self.jobs_df = load_jobs_dataset(self.jobs_path) - self.dc_df = load_dc_config(self.dc_config_path) - self.coords_df = self._load_coordinates(self.coords_path) - self.dc_city_set = set(self.dc_df["city"].dropna().unique().tolist()) - self.cache_key = self._build_cache_key() - - @staticmethod - def _resolve_path(path: str | Path | None, fallback: Path) -> Path: - if path is None or str(path).strip() == "": - return fallback - - candidate = Path(path).expanduser() - if candidate.is_absolute(): - return candidate - return PROJECT_ROOT / candidate - - @classmethod - def _resolve_data_path(cls, data_path: str | Path | None) -> Path: - if data_path is not None and str(data_path).strip() != "": - return cls._resolve_path(data_path, DEFAULT_DATA_CANDIDATES[-1]) - - for candidate in DEFAULT_DATA_CANDIDATES: - if candidate.exists() and candidate.is_file() and candidate.stat().st_size > 0: - return candidate - return DEFAULT_DATA_CANDIDATES[-1] - - @staticmethod - def _load_coordinates(coords_path: Path) -> pd.DataFrame: - coords_df = pd.read_csv(coords_path) - coords_df["latitude"] = pd.to_numeric(coords_df["latitude"], errors="coerce") - coords_df["longitude"] = pd.to_numeric(coords_df["longitude"], errors="coerce") - coords_df = coords_df.dropna(subset=["city", "latitude", "longitude"]).copy() - return coords_df - - @staticmethod - def _load_environment_frame(data_path: Path, dc_config_path: Path) -> pd.DataFrame: - data_df = load_dataset(data_path) - return prepare_environment_frame(data_df) - - def _build_cache_key(self) -> str: - parts: list[str] = [] - for path in [self.data_path, self.jobs_path, self.dc_config_path, self.coords_path]: - resolved = Path(path) - if resolved.exists(): - stat = resolved.stat() - parts.append(f"{resolved}:{stat.st_mtime_ns}:{stat.st_size}") - else: - parts.append(str(resolved)) - return "|".join(parts) - - def time_bounds(self) -> tuple[pd.Timestamp, pd.Timestamp]: - return self.env_df["timestamp"].min(), self.env_df["timestamp"].max() - - def default_overview_window(self, window_days: int = 30) -> tuple[pd.Timestamp, pd.Timestamp]: - env_start, env_end = self.time_bounds() - jobs_start = self.jobs_df["earliest_start"].min() - start = max(env_start, jobs_start.floor("h")) - end = min(start + pd.Timedelta(days=window_days), env_end) - return start, end - - def default_scheduler_timestamp(self) -> pd.Timestamp: - start, _ = self.default_overview_window(window_days=1) - return start - - def context(self) -> dict[str, Any]: - duration_median = float(self.jobs_df["duration_hours"].median()) if "duration_hours" in self.jobs_df.columns else 1.0 - power_median = float(self.jobs_df["power_demand"].median()) if "power_demand" in self.jobs_df.columns else 100.0 - priority_counts = self.jobs_df.get("priority_label", pd.Series(dtype=str)).value_counts().to_dict() - env_start, env_end = self.time_bounds() - default_window_start, default_window_end = self.default_overview_window() - return { - "origin_cities": sorted(self.coords_df["city"].unique().tolist()), - "default_origin_city": str(self.dc_df["city"].iloc[0]), - "time_min": env_start.to_pydatetime(), - "time_max": env_end.to_pydatetime(), - "default_window_start": default_window_start.to_pydatetime(), - "default_window_end": default_window_end.to_pydatetime(), - "default_scheduler_timestamp": self.default_scheduler_timestamp().to_pydatetime(), - "default_workload_size": round(power_median, 2), - "default_duration_hours": max(1, int(round(duration_median))), - "default_batch_size": 100, - "batch_size_min": 50, - "batch_size_max": 200, - "priority_counts": {str(key): int(value) for key, value in priority_counts.items()}, - } - - @staticmethod - def _filter_environment_window( - env_df: pd.DataFrame, - start_time: pd.Timestamp, - end_time: pd.Timestamp, - ) -> pd.DataFrame: - start = pd.Timestamp(start_time).floor("h") - end = pd.Timestamp(end_time).floor("h") - return env_df[(env_df["timestamp"] >= start) & (env_df["timestamp"] <= end)].copy() - - def _shift_trace_jobs_to_window( - self, - start_time: pd.Timestamp, - end_time: pd.Timestamp, - ) -> pd.DataFrame: - jobs = self.jobs_df.copy() - if jobs.empty: - return jobs - - offset = pd.Timestamp(start_time).floor("h") - jobs["earliest_start"].min().floor("h") - jobs["earliest_start"] = jobs["earliest_start"] + offset - jobs["deadline"] = jobs["deadline"] + offset - - window_hours = max( - 1, - int( - ( - pd.Timestamp(end_time).floor("h") - - pd.Timestamp(start_time).floor("h") - ).total_seconds() - / 3600.0 - ), - ) - interactive_limit = max(1, min(window_hours, INTERACTIVE_HORIZON_HOURS)) - raw_slack = ( - (jobs["deadline"] - jobs["earliest_start"]).dt.total_seconds() / 3600.0 - - pd.to_numeric(jobs["duration_hours"], errors="coerce") - ) - jobs["duration_hours"] = ( - pd.to_numeric(jobs["duration_hours"], errors="coerce") - .clip(lower=1, upper=interactive_limit) - .round() - .astype(int) - ) - normalized_slack = raw_slack.clip(lower=1, upper=interactive_limit).round().astype(int) - jobs["deadline"] = jobs["earliest_start"] + pd.to_timedelta( - jobs["duration_hours"] + normalized_slack, - unit="h", - ) - - return jobs[ - (jobs["earliest_start"] <= pd.Timestamp(end_time)) - & (jobs["deadline"] >= pd.Timestamp(start_time)) - ].copy() - - def _derive_job_shape(self, priority: str) -> tuple[int, int]: - jobs = self.jobs_df.copy() - if "priority_label" in jobs.columns: - scoped = jobs[jobs["priority_label"].str.lower() == priority.lower()].copy() - if scoped.empty and priority.lower() == "medium": - scoped = jobs.copy() - if not scoped.empty: - jobs = scoped - - duration = int( - min( - INTERACTIVE_HORIZON_HOURS, - max( - 1, - round( - pd.to_numeric(jobs["duration_hours"], errors="coerce") - .clip(lower=1, upper=INTERACTIVE_HORIZON_HOURS) - .median() - ), - ), - ) - ) - slack_series = ( - (jobs["deadline"] - jobs["earliest_start"]).dt.total_seconds() / 3600.0 - - pd.to_numeric(jobs["duration_hours"], errors="coerce") - ) - slack = ( - int( - min( - INTERACTIVE_HORIZON_HOURS, - max( - 1, - round( - slack_series.dropna() - .clip(lower=1, upper=INTERACTIVE_HORIZON_HOURS) - .median() - ), - ), - ) - ) - if not slack_series.dropna().empty - else 6 - ) - return duration, slack - - def _build_node_frame(self, env_window: pd.DataFrame) -> pd.DataFrame: - if env_window.empty: - return pd.DataFrame( - columns=[ - "city", - "avg_co2_intensity", - "avg_wue", - "avg_scarcity", - "records", - "latitude", - "longitude", - "state", - "zip", - "has_data_center", - ] - ) - - nodes = ( - env_window.groupby("city", as_index=False) - .agg( - avg_co2_intensity=("predicted_co2_intensity", "mean"), - avg_wue=("predicted_water_intensity", "mean"), - avg_scarcity=("scarcity_index", "mean"), - records=("timestamp", "size"), - ) - .merge(self.coords_df, on="city", how="left") - .dropna(subset=["latitude", "longitude"]) - ) - nodes["has_data_center"] = nodes["city"].isin(self.dc_city_set).astype(int) - return nodes.sort_values("avg_co2_intensity", ascending=False).reset_index(drop=True) - - def build_overview_snapshot( - self, - start_time: pd.Timestamp, - end_time: pd.Timestamp, - ) -> OverviewSnapshot: - env_window = self._filter_environment_window(self.env_df, start_time, end_time) - jobs_window = self._shift_trace_jobs_to_window(start_time, end_time) - nodes = self._build_node_frame(env_window) - - metrics = { - "avg_co2_intensity": float(env_window["predicted_co2_intensity"].mean()) if not env_window.empty else 0.0, - "avg_wue": float(env_window["predicted_water_intensity"].mean()) if not env_window.empty else 0.0, - "total_jobs_processed": float(len(jobs_window)), - "co2_reduction_pct": 0.0, - "water_reduction_pct": 0.0, - "coverage_pct": 0.0, - } - - if not env_window.empty and not jobs_window.empty: - optimized = schedule_jobs(env_window, jobs_window, self.dc_df, alpha=1.0, beta=1.0, gamma=1.0) - baseline = schedule_jobs_naive(env_window, jobs_window, self.dc_df) - optimized_summary = summarize_schedule(optimized) - baseline_summary = summarize_schedule(baseline) - comparison = compare_summaries(baseline_summary, optimized_summary) - metrics.update( - { - "total_jobs_processed": float(optimized_summary["total_jobs"]), - "co2_reduction_pct": float(comparison["co2_reduction_pct"]), - "water_reduction_pct": float(comparison["water_reduction_pct"]), - "coverage_pct": float(optimized_summary["coverage"] * 100.0), - } - ) - - return OverviewSnapshot( - metrics=metrics, - nodes=nodes, - start_time=pd.Timestamp(start_time), - end_time=pd.Timestamp(end_time), - ) - - @staticmethod - def _haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float: - radius_km = 6371.0 - lat1_rad = math.radians(lat1) - lon1_rad = math.radians(lon1) - lat2_rad = math.radians(lat2) - lon2_rad = math.radians(lon2) - dlat = lat2_rad - lat1_rad - dlon = lon2_rad - lon1_rad - a = ( - math.sin(dlat / 2) ** 2 - + math.cos(lat1_rad) * math.cos(lat2_rad) * math.sin(dlon / 2) ** 2 - ) - c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a)) - return radius_km * c - - @staticmethod - def _minmax(series: pd.Series) -> pd.Series: - values = pd.to_numeric(series, errors="coerce") - if values.dropna().empty: - return pd.Series([0.0] * len(values), index=values.index) - low = values.min() - high = values.max() - if low == high: - return pd.Series([0.0] * len(values), index=values.index) - return (values - low) / (high - low) - - @staticmethod - def _select_candidate( - candidates: pd.DataFrame, - sort_column: str, - path_type: str, - used_indices: set[int], - used_cities: set[str], - ) -> dict[str, Any] | None: - if candidates.empty: - return None - ordered = candidates.sort_values(sort_column, ascending=True) - for index, row in ordered.iterrows(): - candidate_city = str(row["assigned_city"]) - if index in used_indices or candidate_city in used_cities: - continue - used_indices.add(index) - used_cities.add(candidate_city) - payload = row.to_dict() - payload["type"] = path_type - return payload - return None - - def _attach_route_metrics(self, candidates: pd.DataFrame, origin_city: str) -> pd.DataFrame: - coords = self.coords_df[["city", "latitude", "longitude"]].copy() - origin = coords.rename( - columns={ - "city": "origin_city", - "latitude": "origin_latitude", - "longitude": "origin_longitude", - } - ) - destination = coords.rename( - columns={ - "city": "assigned_city", - "latitude": "destination_latitude", - "longitude": "destination_longitude", - } - ) - enriched = candidates.merge(origin, on="origin_city", how="left").merge( - destination, on="assigned_city", how="left" - ) - enriched = enriched.dropna( - subset=["origin_latitude", "origin_longitude", "destination_latitude", "destination_longitude"] - ).copy() - - latency_ms: list[float] = [] - for _, row in enriched.iterrows(): - distance_km = self._haversine_km( - float(row["origin_latitude"]), - float(row["origin_longitude"]), - float(row["destination_latitude"]), - float(row["destination_longitude"]), - ) - latency_ms.append(5.0 + (distance_km / 200.0)) - enriched["latency"] = latency_ms - - return enriched - - @staticmethod - def _batch_profile(priority: str) -> dict[str, int]: - profiles = { - "high": {"spread_hours": 4, "max_duration": 8, "max_slack": 2}, - "medium": {"spread_hours": 8, "max_duration": 12, "max_slack": 4}, - "low": {"spread_hours": 12, "max_duration": 16, "max_slack": 6}, - } - return profiles.get(priority.lower(), profiles["medium"]) - - def _build_batch_jobs( - self, - start_time: pd.Timestamp, - batch_size: int, - priority: str, - ) -> pd.DataFrame: - jobs = self.jobs_df.copy() - if jobs.empty: - return jobs - - profile = self._batch_profile(priority) - sample_size = max(1, min(len(jobs), int(batch_size))) - batch = ( - jobs.sort_values(["earliest_start", "job_id"]) - .head(sample_size) - .reset_index(drop=True) - .copy() - ) - - original_earliest = batch["earliest_start"].copy() - original_slack = ( - (batch["deadline"] - batch["earliest_start"]).dt.total_seconds() / 3600.0 - - pd.to_numeric(batch["duration_hours"], errors="coerce") - ).clip(lower=1) - - offset_hours = ( - (original_earliest.dt.floor("h") - original_earliest.min().floor("h")) - .dt.total_seconds() - .div(3600.0) - .fillna(0) - .astype(int) - ) - batch["earliest_start"] = pd.Timestamp(start_time).floor("h") + pd.to_timedelta( - offset_hours.mod(profile["spread_hours"]), - unit="h", - ) - - duration_rank = pd.to_numeric(batch["duration_hours"], errors="coerce").rank( - method="average", - pct=True, - ) - batch["duration_hours"] = ( - (duration_rank.fillna(0.5) * profile["max_duration"]) - .clip(lower=1, upper=profile["max_duration"]) - .round() - .astype(int) - ) - - slack_rank = original_slack.rank(method="average", pct=True) - slack_hours = ( - (slack_rank.fillna(0.5) * profile["max_slack"]) - .clip(lower=1, upper=profile["max_slack"]) - .round() - .astype(int) - ) - batch["deadline"] = batch["earliest_start"] + pd.to_timedelta( - batch["duration_hours"] + slack_hours, - unit="h", - ) - - priority_code = PRIORITY_TO_CODE.get(priority.lower(), 1) - batch["priority"] = priority_code - batch["priority_label"] = priority.lower() - return batch - - def _attach_schedule_coordinates(self, schedule_df: pd.DataFrame) -> pd.DataFrame: - if schedule_df.empty: - return schedule_df.copy() - - coords = self.coords_df[["city", "latitude", "longitude"]].copy() - origin = coords.rename( - columns={ - "city": "origin_city", - "latitude": "origin_latitude", - "longitude": "origin_longitude", - } - ) - destination = coords.rename( - columns={ - "city": "assigned_city", - "latitude": "destination_latitude", - "longitude": "destination_longitude", - } - ) - enriched = schedule_df.merge(origin, on="origin_city", how="left").merge( - destination, - on="assigned_city", - how="left", - ) - enriched = enriched.dropna( - subset=["origin_latitude", "origin_longitude", "destination_latitude", "destination_longitude"] - ).copy() - latency_ms: list[float] = [] - for _, row in enriched.iterrows(): - latency_ms.append( - 5.0 - + ( - self._haversine_km( - float(row["origin_latitude"]), - float(row["origin_longitude"]), - float(row["destination_latitude"]), - float(row["destination_longitude"]), - ) - / 200.0 - ) - ) - enriched["latency_ms"] = latency_ms - return enriched - - def _aggregate_schedule_flows( - self, - schedule_df: pd.DataFrame, - scenario: str, - ) -> list[dict[str, Any]]: - if schedule_df.empty: - return [] - - scheduled = self._attach_schedule_coordinates(schedule_df[schedule_df["scheduled"] == 1].copy()) - if scheduled.empty: - return [] - - flows = ( - scheduled.groupby( - [ - "origin_city", - "assigned_city", - "origin_latitude", - "origin_longitude", - "destination_latitude", - "destination_longitude", - ], - as_index=False, - ) - .agg( - jobs=("job_id", "count"), - total_co2_kg=("expected_co2_kg", "sum"), - total_water_liters=("expected_water_liters", "sum"), - avg_scheduler_score=("scheduler_score", "mean"), - avg_latency_ms=("latency_ms", "mean"), - ) - .sort_values(["jobs", "total_co2_kg"], ascending=[False, False]) - .reset_index(drop=True) - ) - outputs: list[dict[str, Any]] = [] - for index, row in flows.iterrows(): - route = f"{row['origin_city']} -> {row['assigned_city']}" - outputs.append( - { - "flow_id": f"{scenario}:{index}:{row['origin_city']}:{row['assigned_city']}", - "route": route, - "origin_city": str(row["origin_city"]), - "assigned_city": str(row["assigned_city"]), - "origin_latitude": float(row["origin_latitude"]), - "origin_longitude": float(row["origin_longitude"]), - "destination_latitude": float(row["destination_latitude"]), - "destination_longitude": float(row["destination_longitude"]), - "jobs": int(row["jobs"]), - "co2": float(row["total_co2_kg"]), - "water_liters": float(row["total_water_liters"]), - "latency": float(row["avg_latency_ms"]), - "score": float(row["avg_scheduler_score"]), - "scenario": scenario, - } - ) - return outputs - - @staticmethod - def _weighted_average_latency(flows: list[dict[str, Any]]) -> float: - if not flows: - return 0.0 - total_jobs = sum(int(flow.get("jobs", 0)) for flow in flows) - if total_jobs <= 0: - return 0.0 - weighted_latency = sum( - float(flow.get("latency", 0.0)) * int(flow.get("jobs", 0)) - for flow in flows - ) - return weighted_latency / total_jobs - - def build_batch_story( - self, - comparison: dict[str, float], - optimized_flows: list[dict[str, Any]], - baseline_flows: list[dict[str, Any]], - batch_size: int, - ) -> dict[str, str]: - busiest_flow = optimized_flows[0] if optimized_flows else None - if busiest_flow is None: - return { - "title": "No feasible batch schedule", - "summary": "The current batch setup could not place any jobs inside the selected simulation window.", - } - - baseline_latency = self._weighted_average_latency(baseline_flows) - optimized_latency = self._weighted_average_latency(optimized_flows) - latency_delta = optimized_latency - baseline_latency - destination = busiest_flow["assigned_city"] - summary = ( - f"The optimized batch sends the heaviest traffic toward {destination}, where the system finds a better " - f"carbon and water profile for this {batch_size}-job window. Compared with the baseline schedule, " - f"the batch changes CO2 by {float(comparison.get('co2_reduction_pct', 0.0)):+.1f}% and water by " - f"{float(comparison.get('water_reduction_pct', 0.0)):+.1f}%, while average route latency moves " - f"{latency_delta:+.1f} ms." - ) - return { - "title": f"Why {destination} absorbs the busiest flow", - "summary": summary, - } - - def run_batch_scheduler(self, inputs: BatchSchedulerInputs) -> dict[str, Any]: - start_time = pd.Timestamp(inputs.start_time).floor("h") - end_time = start_time + pd.Timedelta(hours=INTERACTIVE_HORIZON_HOURS) - env_window = self._filter_environment_window(self.env_df, start_time, end_time) - if env_window.empty: - return { - "time": start_time.isoformat(), - "batch_size": int(inputs.batch_size), - "priority": inputs.priority, - "window_end": end_time.isoformat(), - "nodes": [], - "optimized_flows": [], - "baseline_flows": [], - "selected_flow": None, - "optimized_summary": { - "scheduled_jobs": 0.0, - "total_jobs": float(inputs.batch_size), - "coverage": 0.0, - "total_expected_co2_kg": 0.0, - "total_expected_water_liters": 0.0, - "avg_latency_ms": 0.0, - }, - "baseline_summary": { - "scheduled_jobs": 0.0, - "total_jobs": float(inputs.batch_size), - "coverage": 0.0, - "total_expected_co2_kg": 0.0, - "total_expected_water_liters": 0.0, - "avg_latency_ms": 0.0, - }, - "comparison": { - "co2_reduction_pct": 0.0, - "water_reduction_pct": 0.0, - "coverage_delta_pct": 0.0, - "scheduled_jobs_delta": 0.0, - "latency_delta_ms": 0.0, - }, - "insight": self.build_batch_story( - { - "co2_reduction_pct": 0.0, - "water_reduction_pct": 0.0, - "latency_delta_ms": 0.0, - }, - [], - [], - int(inputs.batch_size), - ), - } - - jobs_window = self._build_batch_jobs(start_time, inputs.batch_size, inputs.priority) - prepared_state = prepare_scheduler_state(env_window, self.dc_df) - optimized = schedule_jobs( - env_window, - jobs_window, - self.dc_df, - alpha=float(inputs.alpha), - beta=float(inputs.beta), - gamma=float(inputs.gamma) * float(inputs.latency_sensitivity), - prepared_state=prepared_state, - ) - baseline = schedule_jobs_naive( - env_window, - jobs_window, - self.dc_df, - prepared_state=prepared_state, - ) - - optimized_flows = self._aggregate_schedule_flows(optimized, scenario="optimized") - baseline_flows = self._aggregate_schedule_flows(baseline, scenario="baseline") - optimized_summary = summarize_schedule(optimized) - baseline_summary = summarize_schedule(baseline) - optimized_summary["avg_latency_ms"] = self._weighted_average_latency(optimized_flows) - baseline_summary["avg_latency_ms"] = self._weighted_average_latency(baseline_flows) - - comparison = compare_summaries(baseline_summary, optimized_summary) - comparison["latency_delta_ms"] = ( - float(optimized_summary["avg_latency_ms"]) - float(baseline_summary["avg_latency_ms"]) - ) - - nodes = self._build_node_frame(env_window) - selected_flow = optimized_flows[0] if optimized_flows else None - insight = self.build_batch_story( - comparison, - optimized_flows, - baseline_flows, - int(inputs.batch_size), - ) - - return { - "time": start_time.isoformat(), - "window_end": end_time.isoformat(), - "batch_size": int(inputs.batch_size), - "priority": inputs.priority, - "nodes": nodes.fillna("").to_dict(orient="records"), - "optimized_flows": optimized_flows, - "baseline_flows": baseline_flows, - "selected_flow": selected_flow, - "optimized_summary": optimized_summary, - "baseline_summary": baseline_summary, - "comparison": comparison, - "insight": insight, - } - - def run_scheduler(self, inputs: SchedulerInputs) -> list[dict[str, Any]]: - start_time = pd.Timestamp(inputs.start_time).floor("h") - duration_hours, slack_hours = self._derive_job_shape(inputs.priority) - deadline = start_time + pd.Timedelta(hours=duration_hours + slack_hours) - priority_code = PRIORITY_TO_CODE.get(inputs.priority.lower(), 1) - - env_window = self._filter_environment_window(self.env_df, start_time, deadline) - if env_window.empty: - return [] - - job = build_job_record( - origin_city=inputs.origin_city, - power_demand=float(inputs.workload_size), - duration_hours=duration_hours, - earliest_start=start_time, - deadline=deadline, - priority=priority_code, - job_id="interactive_simulation_job", - ) - candidates = list_job_candidates( - env_df=env_window, - dc_df=self.dc_df, - job=job, - alpha=1.0, - beta=1.0, - gamma=0.0, - ) - if candidates.empty: - return [] - - candidates = self._attach_route_metrics(candidates, inputs.origin_city) - if candidates.empty: - return [] - - workload_energy_kwh = float(inputs.workload_size) * float(duration_hours) - candidates["co2"] = pd.to_numeric(candidates["expected_co2_kg"], errors="coerce") - candidates["wue"] = pd.to_numeric(candidates["expected_water_liters"], errors="coerce") / max(workload_energy_kwh, 1e-9) - candidates["normalized_co2"] = self._minmax(candidates["co2"]) - candidates["normalized_wue"] = self._minmax(candidates["wue"]) - candidates["normalized_latency"] = self._minmax(candidates["latency"]) - candidates["score"] = ( - float(inputs.alpha) * candidates["normalized_co2"] - + float(inputs.beta) * candidates["normalized_wue"] - + float(inputs.gamma) * float(inputs.latency_sensitivity) * candidates["normalized_latency"] - ) - candidates = candidates.sort_values("score", ascending=True) - - used_indices: set[int] = set() - used_cities: set[str] = set() - selected: list[dict[str, Any]] = [] - selectors = [ - ("score", "balanced"), - ("co2", "low_carbon"), - ("wue", "low_water"), - ("latency", "low_latency"), - ] - for sort_column, path_type in selectors: - candidate = self._select_candidate( - candidates, - sort_column, - path_type, - used_indices, - used_cities, - ) - if candidate is not None: - selected.append(candidate) - - if not selected: - return [] - - outputs: list[dict[str, Any]] = [] - for candidate in selected[: inputs.top_k]: - outputs.append( - { - "path": [candidate["origin_city"], candidate["assigned_city"]], - "co2": float(candidate["co2"]), - "wue": float(candidate["wue"]), - "latency": float(candidate["latency"]), - "score": float(candidate["score"]), - "type": str(candidate["type"]), - "scheduled_start": pd.Timestamp(candidate["scheduled_start"]).isoformat(), - "scheduled_end": pd.Timestamp(candidate["scheduled_end"]).isoformat(), - "origin_city": str(candidate["origin_city"]), - "assigned_city": str(candidate["assigned_city"]), - "origin_latitude": float(candidate["origin_latitude"]), - "origin_longitude": float(candidate["origin_longitude"]), - "destination_latitude": float(candidate["destination_latitude"]), - "destination_longitude": float(candidate["destination_longitude"]), - "water_liters": float(candidate["expected_water_liters"]), - } - ) - return outputs - - def build_scheduler_nodes(self, start_time: pd.Timestamp, end_time: pd.Timestamp) -> pd.DataFrame: - env_window = self._filter_environment_window(self.env_df, start_time, end_time) - return self._build_node_frame(env_window) - - @staticmethod - def explain_path_choice( - selected_path: dict[str, Any], - all_paths: list[dict[str, Any]], - ) -> dict[str, str]: - if not selected_path: - return { - "title": "No route selected", - "summary": "Choose a candidate route to inspect its trade-offs.", - } - - frame = pd.DataFrame(all_paths) - best_co2 = frame["co2"].min() if not frame.empty else selected_path["co2"] - best_wue = frame["wue"].min() if not frame.empty else selected_path["wue"] - best_latency = frame["latency"].min() if not frame.empty else selected_path["latency"] - - def pct_delta(value: float, baseline: float) -> float: - if baseline == 0: - return 0.0 - return ((value - baseline) / baseline) * 100.0 - - path_label = PATH_TYPE_LABELS.get(selected_path["type"], selected_path["type"].replace("_", " ").title()) - summary = ( - f"{path_label} routes {selected_path['path'][0]} to {selected_path['path'][-1]} " - f"with score {selected_path['score']:.3f}. " - f"Its carbon impact is {pct_delta(selected_path['co2'], best_co2):+.1f}% versus the lowest-carbon option, " - f"water intensity is {pct_delta(selected_path['wue'], best_wue):+.1f}% versus the lowest-water option, " - f"and latency is {pct_delta(selected_path['latency'], best_latency):+.1f}% versus the lowest-latency option." - ) - - if selected_path["type"] == "balanced": - title = "Why the balanced route leads" - elif selected_path["type"] == "low_carbon": - title = "Why the low-carbon route wins" - elif selected_path["type"] == "low_water": - title = "Why the low-water route wins" - else: - title = "Why the low-latency route wins" - - return {"title": title, "summary": summary} - - @staticmethod - def select_baseline_path(paths: list[dict[str, Any]]) -> dict[str, Any] | None: - if not paths: - return None - - same_city_paths = [ - path for path in paths if path.get("path") and path["path"][0] == path["path"][-1] - ] - scoped = same_city_paths if same_city_paths else paths - return min( - scoped, - key=lambda path: ( - float(path.get("latency", 0.0)), - float(path.get("score", 0.0)), - ), - ) - - @staticmethod - def compare_paths( - selected_path: dict[str, Any], - baseline_path: dict[str, Any] | None, - ) -> dict[str, float]: - if not selected_path: - return { - "co2_reduction_pct": 0.0, - "water_reduction_pct": 0.0, - "latency_delta_ms": 0.0, - "latency_delta_pct": 0.0, - } - - baseline = baseline_path or selected_path - - def reduction(old: float, new: float) -> float: - if old == 0: - return 0.0 - return ((old - new) / old) * 100.0 - - def delta_pct(new: float, old: float) -> float: - if old == 0: - return 0.0 - return ((new - old) / old) * 100.0 - - selected_co2 = float(selected_path.get("co2", 0.0)) - selected_wue = float(selected_path.get("wue", 0.0)) - selected_latency = float(selected_path.get("latency", 0.0)) - baseline_co2 = float(baseline.get("co2", selected_co2)) - baseline_wue = float(baseline.get("wue", selected_wue)) - baseline_latency = float(baseline.get("latency", selected_latency)) - - return { - "co2_reduction_pct": reduction(baseline_co2, selected_co2), - "water_reduction_pct": reduction(baseline_wue, selected_wue), - "latency_delta_ms": selected_latency - baseline_latency, - "latency_delta_pct": delta_pct(selected_latency, baseline_latency), - } - - def run_baseline_comparison( - self, - start_time: pd.Timestamp, - batch_size: int = 50, - priority: str = "medium", - ) -> dict[str, Any]: - """Compare optimal scheduler vs three baselines on same batch.""" - from .scheduling import prepare_scheduler_state - - # Prepare state (self.env_df is already prepared) - prepared_state = prepare_scheduler_state(self.env_df, self.dc_df) - - # Load and shift jobs - end_time = start_time + pd.Timedelta(hours=48) - jobs = self._shift_trace_jobs_to_window(start_time, end_time) - - # Limit batch - jobs_batch = jobs.head(batch_size).copy() - if jobs_batch.empty: - return { - "timestamp": start_time.isoformat(), - "batch_size": 0, - "results": [], - "summary": "No jobs available for this time window", - } - - # Run each strategy - strategies = { - "optimal": { - "name": "Optimal (Your Method)", - "co2_total": 0.0, - "water_total": 0.0, - "jobs_scheduled": 0, - }, - "random": { - "name": "Random Routing", - "co2_total": 0.0, - "water_total": 0.0, - "jobs_scheduled": 0, - }, - "nearest": { - "name": "Nearest Data Center", - "co2_total": 0.0, - "water_total": 0.0, - "jobs_scheduled": 0, - }, - "time_of_day": { - "name": "Time-of-Day Heuristic", - "co2_total": 0.0, - "water_total": 0.0, - "jobs_scheduled": 0, - }, - } - - # Run optimal scheduler - for _, job in jobs_batch.iterrows(): - from .scheduling import list_job_candidates - candidates_df = list_job_candidates( - env_df=self.env_df, - dc_df=self.dc_df, - job=job, - alpha=1.0, - beta=1.0, - gamma=1.0, - prepared_state=prepared_state, - ) - if not candidates_df.empty: - best_row = candidates_df.loc[candidates_df["scheduler_score"].idxmin()] - strategies["optimal"]["co2_total"] += float(best_row["expected_co2_kg"]) - strategies["optimal"]["water_total"] += float(best_row["expected_water_liters"]) - strategies["optimal"]["jobs_scheduled"] += 1 - - # Run random baseline - for _, job in jobs_batch.iterrows(): - result = baseline_random(job, prepared_state) - if result: - strategies["random"]["co2_total"] += result.expected_co2_kg - strategies["random"]["water_total"] += result.expected_water_liters - strategies["random"]["jobs_scheduled"] += 1 - - # Run nearest baseline - for _, job in jobs_batch.iterrows(): - result = baseline_nearest_neighbor(job, prepared_state) - if result: - strategies["nearest"]["co2_total"] += result.expected_co2_kg - strategies["nearest"]["water_total"] += result.expected_water_liters - strategies["nearest"]["jobs_scheduled"] += 1 - - # Run time-of-day baseline - for _, job in jobs_batch.iterrows(): - result = baseline_time_of_day_heuristic(job, prepared_state) - if result: - strategies["time_of_day"]["co2_total"] += result.expected_co2_kg - strategies["time_of_day"]["water_total"] += result.expected_water_liters - strategies["time_of_day"]["jobs_scheduled"] += 1 - - # Calculate reductions vs baselines - optimal_co2 = strategies["optimal"]["co2_total"] - optimal_water = strategies["optimal"]["water_total"] - - def calc_reduction(baseline_total: float, optimal_total: float) -> float: - if baseline_total == 0: - return 0.0 - return ((baseline_total - optimal_total) / baseline_total) * 100.0 - - results = [] - for key, stats in strategies.items(): - result_item = { - "strategy": stats["name"], - "jobs_scheduled": stats["jobs_scheduled"], - "total_co2_kg": round(stats["co2_total"], 2), - "total_water_liters": round(stats["water_total"], 2), - "avg_co2_per_job": ( - round(stats["co2_total"] / stats["jobs_scheduled"], 4) - if stats["jobs_scheduled"] > 0 - else 0.0 - ), - "avg_water_per_job": ( - round(stats["water_total"] / stats["jobs_scheduled"], 4) - if stats["jobs_scheduled"] > 0 - else 0.0 - ), - } - - # Add reduction metrics relative to each baseline - if key != "optimal": - result_item["co2_reduction_vs_optimal_pct"] = calc_reduction( - stats["co2_total"], optimal_co2 - ) - result_item["water_reduction_vs_optimal_pct"] = calc_reduction( - stats["water_total"], optimal_water - ) - - results.append(result_item) - - # Calculate averages - avg_co2_random = ( - strategies["random"]["co2_total"] / strategies["random"]["jobs_scheduled"] - if strategies["random"]["jobs_scheduled"] > 0 - else 0.0 - ) - avg_co2_optimal = ( - strategies["optimal"]["co2_total"] / strategies["optimal"]["jobs_scheduled"] - if strategies["optimal"]["jobs_scheduled"] > 0 - else 0.0 - ) - - co2_improvement = calc_reduction(avg_co2_random, avg_co2_optimal) - - return { - "timestamp": start_time.isoformat(), - "batch_size": int(batch_size), - "results": results, - "summary": { - "total_jobs_in_batch": len(jobs_batch), - "co2_improvement_vs_random_pct": round(co2_improvement, 2), - "winner": "Optimal" if optimal_co2 <= strategies["random"]["co2_total"] else "Random", - }, - } - - @classmethod - def build_path_story( - cls, - selected_path: dict[str, Any], - all_paths: list[dict[str, Any]], - baseline_path: dict[str, Any] | None = None, - ) -> dict[str, str]: - if not selected_path: - return { - "title": "No recommendation available", - "summary": "The system could not generate a feasible route for the selected hour.", - } - - comparison = cls.compare_paths(selected_path, baseline_path) - selected_label = PATH_TYPE_LABELS.get( - str(selected_path.get("type", "balanced")), - str(selected_path.get("type", "balanced")).replace("_", " ").title(), - ) - destination = selected_path["path"][-1] - - strongest_driver = max( - [ - ("carbon", comparison["co2_reduction_pct"]), - ("water", comparison["water_reduction_pct"]), - ("latency", -comparison["latency_delta_pct"]), - ], - key=lambda item: item[1], - )[0] - - if strongest_driver == "carbon": - rationale = "lower carbon intensity at the destination" - elif strongest_driver == "water": - rationale = "lower water stress and better water efficiency" - else: - rationale = "a smaller latency penalty than the alternatives" - - summary = ( - f"The recommended route sends the workload to {destination} using the {selected_label.lower()} option. " - f"Compared with the baseline route, it changes CO2 by {comparison['co2_reduction_pct']:+.1f}% " - f"and water impact by {comparison['water_reduction_pct']:+.1f}%, with latency moving " - f"{comparison['latency_delta_ms']:+.1f} ms. The recommendation is driven mainly by {rationale}." - ) - return { - "title": f"Why {destination} is recommended", - "summary": summary, - } - - -def run_scheduler( - backend: SustainabilitySchedulingBackend, - inputs: SchedulerInputs, -) -> list[dict[str, Any]]: - """Module-level scheduler entry point for interactive simulation clients.""" - return backend.run_scheduler(inputs) diff --git a/src/baselines.py b/src/baselines.py index 810e4e5..4e6ad7c 100644 --- a/src/baselines.py +++ b/src/baselines.py @@ -1,279 +1,74 @@ -"""Baseline scheduling strategies for comparison with optimized scheduler.""" +"""Feasible reference policies used by the experiment suite.""" from __future__ import annotations -import random -from dataclasses import dataclass -from typing import Any +from collections.abc import Callable import numpy as np import pandas as pd -from .scheduling import PreparedSchedulerState, ScheduleCandidate +from .scheduling import ScheduleCandidate, schedule_with_policy -@dataclass -class BaselineResult: - """Aggregated result from a baseline strategy.""" - strategy_name: str - jobs_scheduled: int - total_co2_kg: float - total_water_liters: float - avg_co2_per_job: float - avg_water_per_job: float - jobs_failed: int - - -def baseline_random( +def _immediate_local_selector( job: pd.Series, - prepared_state: PreparedSchedulerState, -) -> ScheduleCandidate | None: - """Baseline 1: Random city selection. - - Picks any city uniformly at random and schedules at earliest available slot. - No optimization for carbon or water. - """ - if prepared_state.dc_records is None or len(prepared_state.dc_records) == 0: - return None - - duration_hours = int(job["duration_hours"]) - earliest_start = pd.Timestamp(job["earliest_start"]).floor("h") - deadline = pd.Timestamp(job["deadline"]) - - if earliest_start >= deadline: - return None - - # Pick random data center - dc_record = random.choice(prepared_state.dc_records) - city = str(dc_record["city"]) - dc_id = str(dc_record["dc_id"]) - - # Find earliest available start time - city_metrics = prepared_state.city_metrics.get(city) - if city_metrics is None: - return None - - start_idx = int( - np.searchsorted( - prepared_state.time_values, - np.datetime64(earliest_start.to_datetime64(), "ns"), - side="left" - ) - ) - - if start_idx + duration_hours >= len(prepared_state.time_index): - return None - - end_idx = start_idx + duration_hours - start_time = pd.Timestamp(prepared_state.time_index[start_idx]) - end_time = pd.Timestamp(prepared_state.time_index[end_idx]) - - # Calculate averages - co2_span = max(1, end_idx - start_idx) - water_span = max(1, end_idx - start_idx) - - co2_mean = (city_metrics.co2_prefix[end_idx] - city_metrics.co2_prefix[start_idx]) / co2_span - water_mean = (city_metrics.water_prefix[end_idx] - city_metrics.water_prefix[start_idx]) / water_span - - pue = float(dc_record.get("pue", 1.0)) - water_factor = float(dc_record.get("water_usage_factor", 1.0)) - scarcity = float(dc_record.get("region_water_scarcity", 1.0)) - - # Total energy consumption: power_demand (kW) * duration_hours (h) = kWh - power_demand = float(job.get("power_demand", 1.0)) - total_kwh = power_demand * duration_hours - - # Total CO2: intensity (kg/kWh) * total_kwh * PUE - expected_co2 = co2_mean * total_kwh * pue - # Total water: intensity * total_kwh * water_factor * scarcity - expected_water = water_mean * total_kwh * water_factor * scarcity - - # Dummy score (not used for ranking, just placeholder) - score = 0.0 - - return ScheduleCandidate( - dc_id=dc_id, - city=city, - start_time=start_time, - end_time=end_time, - score=score, - expected_co2_kg=expected_co2, - expected_water_liters=expected_water, - start_index=start_idx, - end_index=end_idx, + candidates: list[ScheduleCandidate], +) -> ScheduleCandidate: + """Prefer the origin city, then the earliest feasible placement.""" + origin = str(job["origin_city"]) + return min( + candidates, + key=lambda item: ( + item.city != origin, + item.start_time, + item.dc_id, + ), ) -def baseline_nearest_neighbor( - job: pd.Series, - prepared_state: PreparedSchedulerState, -) -> ScheduleCandidate | None: - """Baseline 2: Nearest data center (current industry practice). - - Schedules to closest DC by location, earliest available slot. - Ignores carbon and water completely. - """ - origin_city = str(job["origin_city"]) - duration_hours = int(job["duration_hours"]) - earliest_start = pd.Timestamp(job["earliest_start"]).floor("h") - deadline = pd.Timestamp(job["deadline"]) - - if earliest_start >= deadline: - return None - - # Find DC in same city, or pick first available if none exists - same_city_dc = None - for dc_record in prepared_state.dc_records: - if str(dc_record["city"]) == origin_city: - same_city_dc = dc_record - break - - dc_record = same_city_dc or prepared_state.dc_records[0] - city = str(dc_record["city"]) - dc_id = str(dc_record["dc_id"]) - - city_metrics = prepared_state.city_metrics.get(city) - if city_metrics is None: - return None - - start_idx = int( - np.searchsorted( - prepared_state.time_values, - np.datetime64(earliest_start.to_datetime64(), "ns"), - side="left" - ) +def _carbon_greedy_selector( + _job: pd.Series, + candidates: list[ScheduleCandidate], +) -> ScheduleCandidate: + """Minimize predicted operational carbon only.""" + return min( + candidates, + key=lambda item: (item.expected_co2_kg, item.start_time, item.dc_id), ) - if start_idx + duration_hours >= len(prepared_state.time_index): - return None - - end_idx = start_idx + duration_hours - start_time = pd.Timestamp(prepared_state.time_index[start_idx]) - end_time = pd.Timestamp(prepared_state.time_index[end_idx]) - - # Calculate averages - co2_span = max(1, end_idx - start_idx) - water_span = max(1, end_idx - start_idx) - - co2_mean = (city_metrics.co2_prefix[end_idx] - city_metrics.co2_prefix[start_idx]) / co2_span - water_mean = (city_metrics.water_prefix[end_idx] - city_metrics.water_prefix[start_idx]) / water_span - - pue = float(dc_record.get("pue", 1.0)) - water_factor = float(dc_record.get("water_usage_factor", 1.0)) - scarcity = float(dc_record.get("region_water_scarcity", 1.0)) - - # Total energy consumption: power_demand (kW) * duration_hours (h) = kWh - power_demand = float(job.get("power_demand", 1.0)) - total_kwh = power_demand * duration_hours - - # Total CO2: intensity (kg/kWh) * total_kwh * PUE - expected_co2 = co2_mean * total_kwh * pue - # Total water: intensity * total_kwh * water_factor * scarcity - expected_water = water_mean * total_kwh * water_factor * scarcity - - return ScheduleCandidate( - dc_id=dc_id, - city=city, - start_time=start_time, - end_time=end_time, - score=0.0, - expected_co2_kg=expected_co2, - expected_water_liters=expected_water, - start_index=start_idx, - end_index=end_idx, - ) - - -def baseline_time_of_day_heuristic( - job: pd.Series, - prepared_state: PreparedSchedulerState, -) -> ScheduleCandidate | None: - """Baseline 3: Time-of-day heuristic (simple rule-based approach). - - Picks city with lowest CO2 at that hour globally. - Still respects deadlines but ignores water. - """ - duration_hours = int(job["duration_hours"]) - earliest_start = pd.Timestamp(job["earliest_start"]).floor("h") - deadline = pd.Timestamp(job["deadline"]) - - if earliest_start >= deadline: - return None - - start_idx = int( - np.searchsorted( - prepared_state.time_values, - np.datetime64(earliest_start.to_datetime64(), "ns"), - side="left" - ) - ) - - if start_idx + duration_hours >= len(prepared_state.time_index): - return None - - end_idx = start_idx + duration_hours - - # Find city with lowest average CO2 during this window - best_city = None - best_co2 = float("inf") - - for city, city_metrics in prepared_state.city_metrics.items(): - co2_span = max(1, end_idx - start_idx) - co2_mean = (city_metrics.co2_prefix[end_idx] - city_metrics.co2_prefix[start_idx]) / co2_span - - if co2_mean < best_co2: - best_co2 = co2_mean - best_city = city - - if best_city is None: - return None - - # Find a DC in best_city - dc_record = None - for record in prepared_state.dc_records: - if str(record["city"]) == best_city: - dc_record = record - break - - if dc_record is None: - return None - - city = best_city - dc_id = str(dc_record["dc_id"]) - city_metrics = prepared_state.city_metrics[city] - - start_time = pd.Timestamp(prepared_state.time_index[start_idx]) - end_time = pd.Timestamp(prepared_state.time_index[end_idx]) - - # Calculate metrics - co2_span = max(1, end_idx - start_idx) - water_span = max(1, end_idx - start_idx) - - co2_mean = (city_metrics.co2_prefix[end_idx] - city_metrics.co2_prefix[start_idx]) / co2_span - water_mean = (city_metrics.water_prefix[end_idx] - city_metrics.water_prefix[start_idx]) / water_span - - pue = float(dc_record.get("pue", 1.0)) - water_factor = float(dc_record.get("water_usage_factor", 1.0)) - scarcity = float(dc_record.get("region_water_scarcity", 1.0)) - - # Total energy consumption: power_demand (kW) * duration_hours (h) = kWh - power_demand = float(job.get("power_demand", 1.0)) - duration_hours = int(job.get("duration_hours", 1)) - total_kwh = power_demand * duration_hours - - # Total CO2: intensity (kg/kWh) * total_kwh * PUE - expected_co2 = co2_mean * total_kwh * pue - # Total water: intensity * total_kwh * water_factor * scarcity - expected_water = water_mean * total_kwh * water_factor * scarcity - return ScheduleCandidate( - dc_id=dc_id, - city=city, - start_time=start_time, - end_time=end_time, - score=0.0, - expected_co2_kg=expected_co2, - expected_water_liters=expected_water, - start_index=start_idx, - end_index=end_idx, +def _random_selector(seed: int) -> Callable: + rng = np.random.default_rng(seed) + + def select( + _job: pd.Series, + candidates: list[ScheduleCandidate], + ) -> ScheduleCandidate: + return candidates[int(rng.integers(0, len(candidates)))] + + return select + + +def run_baseline( + strategy: str, + env_df: pd.DataFrame, + jobs_df: pd.DataFrame, + dc_df: pd.DataFrame, + seed: int = 0, +) -> pd.DataFrame: + """Run a named baseline under the same capacity and deadline constraints.""" + selectors = { + "immediate_local": _immediate_local_selector, + "carbon_greedy": _carbon_greedy_selector, + "random": _random_selector(seed), + } + if strategy not in selectors: + choices = ", ".join(sorted(selectors)) + raise ValueError(f"Unknown baseline {strategy!r}. Choose one of: {choices}") + + return schedule_with_policy( + env_df=env_df, + jobs_df=jobs_df, + dc_df=dc_df, + selector=selectors[strategy], ) diff --git a/src/build_dataset.py b/src/build_dataset.py deleted file mode 100644 index 387f03e..0000000 --- a/src/build_dataset.py +++ /dev/null @@ -1,263 +0,0 @@ -""" -build_dataset.py ----------------- -Merge EIA generation mix + Open-Meteo weather into one unified dataset -that the existing ML pipeline (train.py, scheduling.py) can read directly. - -Carbon intensity is computed from EIA hourly fuel mix + EPA eGRID emission -factors — the standard methodology used in peer-reviewed research on -grid carbon accounting. No paid API required. - -Sources: - - EPA eGRID 2022 emission factors (kg CO2/kWh per fuel type) - https://www.epa.gov/egrid - - EIA hourly generation by fuel type (free API) - https://www.eia.gov/opendata/ - - Open-Meteo historical weather archive (free, no key) - https://open-meteo.com/ - -Input files (data/processed/): - - eia_generation_mix.csv ← from fetch_eia.py - - dc_weather_historical.csv ← from fetch_weather.py - -Output: - - data/base_data_with_metrics.csv (main dataset the pipeline expects) - -Usage: - python -m src.build_dataset - python -m src.build_dataset --start 2023-01-01 --end 2023-12-31 -""" - -from __future__ import annotations - -import argparse -from pathlib import Path - -import pandas as pd -import numpy as np - -try: - from .data_loader import PROJECT_ROOT -except ImportError: - from data_loader import PROJECT_ROOT - -EIA_PATH = PROJECT_ROOT / "data" / "processed" / "eia_generation_mix.csv" -WEATHER_PATH = PROJECT_ROOT / "data" / "processed" / "dc_weather_historical.csv" -COORDS_PATH = PROJECT_ROOT / "data" / "reference" / "city_coordinates.csv" -OUTPUT_PATH = PROJECT_ROOT / "data" / "base_data_with_metrics.csv" - -# --------------------------------------------------------------------------- -# EPA eGRID 2022 emission factors — kg CO2 per kWh generated -# Source: EPA eGRID Summary Tables 2022 (Table 1, national averages by fuel) -# https://www.epa.gov/egrid/download-data -# --------------------------------------------------------------------------- -EGRID_EMISSION_FACTORS = { - "coal": 0.9552, # kg CO2/kWh — highest emitter - "natural_gas": 0.4031, # kg CO2/kWh — cleaner than coal, still significant - "petroleum": 0.6815, # kg CO2/kWh - "nuclear": 0.0055, # kg CO2/kWh — lifecycle emissions only - "hydro": 0.0067, # kg CO2/kWh — lifecycle emissions only - "solar": 0.0200, # kg CO2/kWh — lifecycle emissions only - "wind": 0.0073, # kg CO2/kWh — lifecycle emissions only - "other": 0.3500, # kg CO2/kWh — conservative estimate for mixed -} - -# BA region for each of the 28 DC locations -DC_BA_MAP = { - "Abilene": "ERCOT", - "Shackelford County": "ERCOT", - "Dona Ana County": "WACM", - "Lordstown": "PJM", - "Milam County": "ERCOT", - "Port Washington": "MISO", - "Mount Pleasant": "MISO", - "Atlanta": "SOCO", - "Wilbarger County": "ERCOT", - "Armstrong County": "ERCOT", - "Haskell County": "ERCOT", - "Pine Island": "MISO", - "Richland Parish": "MISO", - "Beaver Dam": "MISO", - "El Paso": "EPE", - "Bowling Green": "PJM", - "Carlisle": "MISO", - "Northern Indiana": "MISO", - "Memphis": "MISO", - "Lancaster": "PJM", - "Kenilworth": "PJM", - "West Texas": "ERCOT", - "Dallas": "ERCOT", - "Albany": "NYIS", - "Little Rock": "MISO", - "Meridian": "MISO", - "Rocky Mount": "PJM", - "Powhatan County": "PJM", -} - - -def _load_eia(path: Path) -> pd.DataFrame: - df = pd.read_csv(path) - df["timestamp"] = pd.to_datetime(df["timestamp"], utc=True).dt.tz_localize(None).dt.floor("h") - return df - - -def _load_weather(path: Path) -> pd.DataFrame: - df = pd.read_csv(path) - df["timestamp"] = pd.to_datetime(df["TIMESTAMP"]).dt.floor("h") - weather_cols = [c for c in df.columns if c.startswith("weather_")] - return df[["city", "timestamp"] + weather_cols] - - -def _compute_co2_intensity(eia_df: pd.DataFrame) -> pd.DataFrame: - """ - Compute hourly CO2 intensity (kg CO2/kWh) from EIA fuel mix shares - and EPA eGRID emission factors. - - co2_intensity = Σ (share_fuel_i × emission_factor_i) - - This is the standard average emissions intensity approach used in - academic carbon-aware computing research. - """ - df = eia_df.copy() - - df["co2_intensity_kg_per_kwh"] = sum( - df.get(f"{fuel}_share", pd.Series(0.0, index=df.index)) * factor - for fuel, factor in EGRID_EMISSION_FACTORS.items() - ) - - # co2_kg = intensity × total generation converted to kWh - df["total_gen_kwh"] = df["total_mwh"] * 1000.0 - df["co2_kg"] = df["co2_intensity_kg_per_kwh"] * df["total_gen_kwh"] - - return df - - -def _estimate_wue(df: pd.DataFrame) -> pd.DataFrame: - """ - Estimate Water Usage Effectiveness from ambient temperature and humidity. - - WUE increases in hot/dry conditions (more evaporative cooling needed). - Range: 1.0 (excellent) – 2.5 (poor). - - Derived from ASHRAE TC 9.9 guidelines and Green Grid WUE benchmarks. - """ - temp = df.get("weather_temp", pd.Series(20.0, index=df.index)) - rhum = df.get("weather_rhum", pd.Series(50.0, index=df.index)) - - wue = ( - 1.2 - + 0.015 * (temp - 20).clip(lower=0) - - 0.003 * (rhum - 50).clip(lower=0) - ).clip(lower=1.0, upper=2.5) - - df["WUE_total"] = wue.round(4) - return df - - -def build(start: str | None, end: str | None) -> pd.DataFrame: - print("Loading source files...") - - for label, path in [("EIA mix", EIA_PATH), ("Weather", WEATHER_PATH)]: - if not path.exists(): - raise FileNotFoundError( - f"{label} file not found: {path}\n" - f"Run the corresponding fetch script first:\n" - f" EIA: python -m src.fetch_eia --api-key YOUR_KEY\n" - f" Weather: python -m src.fetch_weather" - ) - - eia_df = _load_eia(EIA_PATH) - weather_df = _load_weather(WEATHER_PATH) - coords_df = pd.read_csv(COORDS_PATH) - - print(f" EIA: {len(eia_df):,} rows ({eia_df['ba_region'].nunique()} regions)") - print(f" Weather: {len(weather_df):,} rows ({weather_df['city'].nunique()} cities)") - - # --- Step 1: compute CO2 intensity from EIA fuel mix --- - print("\nComputing CO2 intensity from EIA fuel mix + EPA eGRID factors...") - eia_df = _compute_co2_intensity(eia_df) - print(f" CO2 range: {eia_df['co2_intensity_kg_per_kwh'].min():.4f} – " - f"{eia_df['co2_intensity_kg_per_kwh'].max():.4f} kg/kWh") - - # --- Step 2: attach BA region to each DC city --- - ba_map_df = pd.DataFrame( - {"city": list(DC_BA_MAP.keys()), "ba_region": list(DC_BA_MAP.values())} - ) - weather_df = weather_df.merge(ba_map_df, on="city", how="left") - - # --- Step 3: join EIA data onto weather by (ba_region, timestamp) --- - share_cols = [c for c in eia_df.columns if c.endswith("_share")] - eia_cols = ["ba_region", "timestamp", "total_mwh", "total_gen_kwh", - "co2_kg", "co2_intensity_kg_per_kwh"] + share_cols - - merged = weather_df.merge(eia_df[eia_cols], on=["ba_region", "timestamp"], how="left") - - # --- Step 4: estimate WUE from weather --- - print("Estimating WUE from temperature and humidity...") - merged = _estimate_wue(merged) - - # --- Step 5: attach coordinates and metadata --- - merged = merged.merge( - coords_df[["city", "latitude", "longitude", "capacity_mw", "category"]], - on="city", - how="left", - ) - - # --- Step 6: rename to match pipeline column expectations --- - merged = merged.rename(columns={ - "city": "CITY", - "timestamp": "TIMESTAMP", - "co2_intensity_kg_per_kwh": "co2_intensity", - }) - - # --- Step 7: optional date filter --- - if start: - merged = merged[merged["TIMESTAMP"] >= pd.Timestamp(start)] - if end: - merged = merged[merged["TIMESTAMP"] <= pd.Timestamp(end)] - - merged = merged.sort_values(["CITY", "TIMESTAMP"]).reset_index(drop=True) - - before = len(merged) - merged = merged.dropna(subset=["co2_kg", "total_gen_kwh", "WUE_total"]) - dropped = before - len(merged) - if dropped: - print(f" Dropped {dropped:,} rows with missing core values") - - # Clip any negative CO2 values (data anomalies from near-zero generation hours) - neg = (merged["co2_intensity"] < 0).sum() - if neg: - merged["co2_intensity"] = merged["co2_intensity"].clip(lower=0) - merged["co2_kg"] = merged["co2_intensity"] * merged["total_gen_kwh"] - print(f" Clipped {neg} negative CO2 rows to 0") - - return merged - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Build unified dataset from EIA + Weather (EPA eGRID emission factors)" - ) - parser.add_argument("--start", default=None, help="Filter start date YYYY-MM-DD") - parser.add_argument("--end", default=None, help="Filter end date YYYY-MM-DD") - parser.add_argument("--output", default=str(OUTPUT_PATH)) - args = parser.parse_args() - - df = build(args.start, args.end) - - output_path = Path(args.output) - output_path.parent.mkdir(parents=True, exist_ok=True) - df.to_csv(output_path, index=False) - - print(f"\nDataset built successfully") - print(f" Rows: {len(df):,}") - print(f" Cities: {df['CITY'].nunique()}") - print(f" Date range: {df['TIMESTAMP'].min()} → {df['TIMESTAMP'].max()}") - print(f" CO2 range: {df['co2_kg'].min():.1f} – {df['co2_kg'].max():.1f} kg") - print(f" WUE range: {df['WUE_total'].min():.2f} – {df['WUE_total'].max():.2f}") - print(f"\nSaved → {output_path}") - print("Ready to run: python -m src.train") - - -if __name__ == "__main__": - main() diff --git a/src/build_eu_dataset.py b/src/build_eu_dataset.py deleted file mode 100644 index 403f3d7..0000000 --- a/src/build_eu_dataset.py +++ /dev/null @@ -1,249 +0,0 @@ -""" -build_eu_dataset.py -------------------- -Merge ENTSO-E generation mix + Open-Meteo weather into the same column -schema as base_data_with_metrics.csv — so the existing ML pipeline -(train.py, scheduling.py, app_backend.py) can load EU data without -any code changes. - -Sources: - - ENTSO-E Transparency Platform (hourly generation per fuel type) - https://transparency.entsoe.eu/ - - Open-Meteo historical archive (free, no key needed) - https://open-meteo.com/ - -Input files (data/processed/): - - entso_e_generation_mix.csv ← from fetch_entso_e.py - - eu_weather_historical.csv ← from fetch_weather.py --region eu - -Output: - - data/eu_data_with_metrics.csv - -Usage: - python -m src.build_eu_dataset - python -m src.build_eu_dataset --start 2023-01-01 --end 2023-12-31 -""" - -from __future__ import annotations - -import argparse -from pathlib import Path - -import numpy as np -import pandas as pd - -try: - from .data_loader import PROJECT_ROOT -except ImportError: - from data_loader import PROJECT_ROOT - -ENTSO_PATH = PROJECT_ROOT / "data" / "processed" / "entso_e_generation_mix.csv" -WEATHER_PATH = PROJECT_ROOT / "data" / "processed" / "eu_weather_historical.csv" -COORDS_PATH = PROJECT_ROOT / "data" / "reference" / "eu_city_coordinates.csv" -OUTPUT_PATH = PROJECT_ROOT / "data" / "eu_data_with_metrics.csv" - -# IPCC 2014 lifecycle emission factors (kg CO2 / kWh) for EU -# Source: IPCC AR5 WG3, Annex III, Table A.III.2 -EMISSION_FACTORS = { - "coal": 0.8200, - "gas": 0.4900, - "oil": 0.6500, - "nuclear": 0.0120, - "hydro": 0.0040, - "solar": 0.0450, - "wind": 0.0110, - "biomass": 0.2300, - "other": 0.4000, -} - -# Map ENTSO-E bidding zone → city name (must match eu_city_coordinates.csv) -ZONE_CITY_MAP = { - "IE": "Dublin", - "NL": "Amsterdam", - "DE-LU": "Frankfurt", - "GB": "London", - "FR": "Paris", - "SE3": "Stockholm", - "FI": "Helsinki", - "ES": "Madrid", - "PL": "Warsaw", - "DK2": "Copenhagen", -} - -# EU drought stress approximation (0=no scarcity, 1=extreme scarcity) -# Based on EU JRC MARS drought data (2023 annual average) -CITY_WATER_SCARCITY = { - "Dublin": 0.05, - "Amsterdam": 0.12, - "Frankfurt": 0.18, - "London": 0.15, - "Paris": 0.22, - "Stockholm": 0.03, - "Helsinki": 0.02, - "Madrid": 0.68, - "Warsaw": 0.35, - "Copenhagen": 0.10, -} - - -# ─── Loaders ────────────────────────────────────────────────────────────────── - -def _load_entso(path: Path) -> pd.DataFrame: - df = pd.read_csv(path) - df["timestamp"] = pd.to_datetime(df["timestamp"]).dt.floor("h") - df["generation_mwh"] = pd.to_numeric(df["generation_mwh"], errors="coerce").fillna(0.0) - return df - - -def _load_weather(path: Path) -> pd.DataFrame: - df = pd.read_csv(path) - col = "TIMESTAMP" if "TIMESTAMP" in df.columns else "timestamp" - df["timestamp"] = pd.to_datetime(df[col]).dt.floor("h") - city_col = "CITY" if "CITY" in df.columns else "city" - df = df.rename(columns={city_col: "city"}) - weather_cols = [c for c in df.columns if c.startswith("weather_")] - return df[["city", "timestamp"] + weather_cols] - - -# ─── CO₂ intensity ──────────────────────────────────────────────────────────── - -def _compute_co2_intensity(entso_df: pd.DataFrame) -> pd.DataFrame: - """ - Compute hourly CO₂ intensity per ENTSO-E zone. - - co2_intensity = Σ (generation_fuel_i × emission_factor_i) / total_generation - - This is the standard marginal/average intensity methodology used in - EU academic carbon-aware computing papers (e.g. Wiesner et al., 2021). - """ - df = entso_df.copy() - df["co2_kg_per_row"] = df["fuel_type"].map(EMISSION_FACTORS) * df["generation_mwh"] * 1000 - - hourly = ( - df.groupby(["timestamp", "zone"]) - .agg( - total_gen_mwh=("generation_mwh", "sum"), - total_co2_kg=("co2_kg_per_row", "sum"), - ) - .reset_index() - ) - hourly["total_gen_kwh"] = hourly["total_gen_mwh"] * 1000.0 - hourly["co2_intensity_kg_per_kwh"] = ( - hourly["total_co2_kg"] - / hourly["total_gen_kwh"].replace(0, float("nan")) - ) - hourly["co2_kg"] = hourly["total_co2_kg"] - return hourly - - -# ─── WUE estimation ─────────────────────────────────────────────────────────── - -def _estimate_wue(df: pd.DataFrame) -> pd.DataFrame: - """ - Estimate Water Usage Effectiveness (WUE) from ambient temperature. - - Same formula as build_dataset.py so models trained on US data - can be compared apples-to-apples with EU data. - - WUE = 1.0 + 0.008 × max(0, T - 15°C) - This is a simplified engineering model consistent with published - EU data center water efficiency benchmarks (EU Code of Conduct 2022). - """ - df = df.copy() - temp_col = next( - (c for c in df.columns if "temperature" in c.lower()), - None, - ) - if temp_col: - excess_heat = np.maximum(0.0, df[temp_col].fillna(15.0) - 15.0) - df["WUE_total"] = 1.0 + 0.008 * excess_heat - else: - df["WUE_total"] = 1.2 # EU Code of Conduct 2022 average - return df - - -# ─── Builder ────────────────────────────────────────────────────────────────── - -def build_eu_dataset( - entso_path: Path = ENTSO_PATH, - weather_path: Path = WEATHER_PATH, - coords_path: Path = COORDS_PATH, - start: str | None = None, - end: str | None = None, -) -> pd.DataFrame: - """ - Merge ENTSO-E + weather data into a dataset with the same schema - as base_data_with_metrics.csv. - """ - # Load inputs - entso_df = _load_entso(entso_path) - - if start: - entso_df = entso_df[entso_df["timestamp"] >= pd.Timestamp(start)] - if end: - entso_df = entso_df[entso_df["timestamp"] <= pd.Timestamp(end)] - - # Compute CO₂ intensity per zone per hour - intensity_df = _compute_co2_intensity(entso_df) - - # Map zone → city - intensity_df["CITY"] = intensity_df["zone"].map(ZONE_CITY_MAP) - intensity_df = intensity_df.dropna(subset=["CITY"]) - - # Rename columns to match US schema - intensity_df = intensity_df.rename(columns={"timestamp": "TIMESTAMP"}) - - # Merge weather if available - if weather_path.exists(): - weather_df = _load_weather(weather_path) - weather_df = weather_df.rename(columns={"city": "CITY", "timestamp": "TIMESTAMP"}) - merged = intensity_df.merge(weather_df, on=["TIMESTAMP", "CITY"], how="left") - else: - print(f" [INFO] Weather file not found at {weather_path}; skipping weather merge.") - merged = intensity_df - - # Estimate WUE - merged = _estimate_wue(merged) - - # Add water scarcity index (0–500 scale to match US dsci_0_500 column) - merged["dsci_0_500"] = merged["CITY"].map( - {city: v * 500 for city, v in CITY_WATER_SCARCITY.items()} - ).fillna(250.0) - - # Add region label for traceability - merged["region"] = "EU" - - # Sort and return - merged = merged.sort_values(["CITY", "TIMESTAMP"]).reset_index(drop=True) - return merged - - -def main() -> None: - parser = argparse.ArgumentParser(description="Build EU dataset from ENTSO-E + weather data") - parser.add_argument("--start", default=None, help="Start date filter (YYYY-MM-DD)") - parser.add_argument("--end", default=None, help="End date filter (YYYY-MM-DD)") - parser.add_argument("--output", default=str(OUTPUT_PATH), help="Output CSV path") - args = parser.parse_args() - - print("Building EU dataset...") - for path, label in [(ENTSO_PATH, "ENTSO-E"), (COORDS_PATH, "EU coordinates")]: - if not path.exists(): - print(f" [ERROR] {label} file not found: {path}") - print(f" Run 'python -m src.fetch_entso_e --token YOUR_TOKEN' first.") - return - - df = build_eu_dataset(start=args.start, end=args.end) - - output_path = Path(args.output) - output_path.parent.mkdir(parents=True, exist_ok=True) - df.to_csv(output_path, index=False) - - print(f"\nEU dataset summary:") - print(f" Rows: {len(df):,}") - print(f" Cities: {sorted(df['CITY'].unique().tolist())}") - print(f" Dates: {df['TIMESTAMP'].min()} → {df['TIMESTAMP'].max()}") - print(f" Saved → {output_path}") - - -if __name__ == "__main__": - main() diff --git a/src/cross_region_comparison.py b/src/cross_region_comparison.py deleted file mode 100644 index d277297..0000000 --- a/src/cross_region_comparison.py +++ /dev/null @@ -1,246 +0,0 @@ -""" -cross_region_comparison.py --------------------------- -Compare scheduler performance on US vs EU grids using identical job batches. - -Research question: - Does carbon-aware workload routing provide greater benefit in a - heterogeneous grid (EU, 40× CO2 spread) vs a homogeneous grid - (US, 3× CO2 spread)? - -Methodology: - 1. Sample a synthetic job batch representative of both regions. - 2. Schedule on US data (optimal + naive baselines). - 3. Schedule on EU data (optimal + naive baselines). - 4. Report: CO2 reduction %, water reduction %, coverage, and CO2 savings - per job for both regions. - -Output: - - Console table - - data/results/cross_region_comparison.csv - -Usage: - python -m src.cross_region_comparison - python -m src.cross_region_comparison --jobs 200 -""" - -from __future__ import annotations - -import argparse -from pathlib import Path - -import numpy as np -import pandas as pd - -try: - from .scheduling import ( - load_dc_config, - prepare_environment_frame, - schedule_jobs, - schedule_jobs_naive, - summarize_schedule, - ) - from .data_loader import PROJECT_ROOT, load_dataset -except ImportError: - from scheduling import ( - load_dc_config, - prepare_environment_frame, - schedule_jobs, - schedule_jobs_naive, - summarize_schedule, - ) - from data_loader import PROJECT_ROOT, load_dataset - -EU_DATA_PATH = PROJECT_ROOT / "data" / "eu_data_with_metrics.csv" -EU_DC_PATH = PROJECT_ROOT / "data" / "templates" / "eu_dc_config.csv" -OUT_PATH = PROJECT_ROOT / "data" / "results" / "cross_region_comparison.csv" - - -def _build_jobs(dc_df: pd.DataFrame, n: int, data_start: str, data_end: str, seed: int) -> pd.DataFrame: - """Build a synthetic job batch whose origin cities come from dc_df.""" - rng = np.random.default_rng(seed) - cities = dc_df["city"].tolist() - return pd.DataFrame({ - "job_id": [f"job_{i:04d}" for i in range(n)], - "power_demand": rng.choice([10.0, 20.0, 50.0, 100.0], n).tolist(), - "duration_hours": rng.integers(2, 9, n).tolist(), - "earliest_start": data_start, - "deadline": data_end, - "origin_city": rng.choice(cities, n).tolist(), - "priority": 1, - }) - - -def _run_region( - label: str, - data_df: pd.DataFrame, - dc_df: pd.DataFrame, - jobs_df: pd.DataFrame, - alpha: float, - beta: float, - gamma: float, -) -> dict: - """Schedule one region, return summary metrics for both optimal and naive.""" - env = prepare_environment_frame(data_df) - - # CO2 grid characteristics - co2_values = env.groupby("city")["predicted_co2_intensity"].mean() - co2_min = co2_values.min() - co2_max = co2_values.max() - co2_spread = co2_max / co2_min - - # Optimal schedule - sched_opt = schedule_jobs(env, jobs_df, dc_df, alpha=alpha, beta=beta, gamma=gamma) - summary_opt = summarize_schedule(sched_opt) - - # Naive schedule (no spatial/temporal shifting) - sched_naive = schedule_jobs_naive(env, jobs_df, dc_df) - summary_naive = summarize_schedule(sched_naive) - - opt_co2 = summary_opt["total_expected_co2_kg"] - naive_co2 = summary_naive["total_expected_co2_kg"] - opt_water = summary_opt["total_expected_water_liters"] - naive_water = summary_naive["total_expected_water_liters"] - - n_placed = int(summary_opt["scheduled_jobs"]) - co2_reduction_pct = (1 - opt_co2 / naive_co2) * 100 if naive_co2 > 0 else 0.0 - water_reduction_pct = (1 - opt_water / naive_water) * 100 if naive_water > 0 else 0.0 - co2_saved_per_job = (naive_co2 - opt_co2) / n_placed if n_placed > 0 else 0.0 - - # City allocation: fraction of jobs sent to top-3 cleanest cities - scheduled_rows = sched_opt[sched_opt["scheduled"] == 1] - top3_clean = co2_values.nsmallest(3).index.tolist() - frac_to_clean = ( - scheduled_rows["assigned_city"].isin(top3_clean).sum() / n_placed - if n_placed > 0 else 0.0 - ) - - return { - "region": label, - "n_cities": len(dc_df), - "co2_min_kg_kwh": round(co2_min, 4), - "co2_max_kg_kwh": round(co2_max, 4), - "co2_spread_x": round(co2_spread, 1), - "cleanest_city": co2_values.idxmin(), - "dirtiest_city": co2_values.idxmax(), - "jobs_total": len(jobs_df), - "jobs_scheduled": n_placed, - "coverage_pct": round(summary_opt["coverage"] * 100, 1), - "naive_co2_kg": round(naive_co2, 2), - "optimal_co2_kg": round(opt_co2, 2), - "co2_reduction_pct": round(co2_reduction_pct, 1), - "co2_saved_kg": round(naive_co2 - opt_co2, 2), - "co2_saved_per_job_kg": round(co2_saved_per_job, 2), - "naive_water_l": round(naive_water, 1), - "optimal_water_l": round(opt_water, 1), - "water_reduction_pct": round(water_reduction_pct, 1), - "frac_to_top3_clean": round(frac_to_clean * 100, 1), - "top3_clean_cities": ", ".join(top3_clean), - } - - -def run_cross_region_comparison( - n_jobs: int = 100, - alpha: float = 0.7, - beta: float = 0.2, - seed: int = 42, -) -> pd.DataFrame: - gamma = round(1 - alpha - beta, 4) - - # ── Load US data ────────────────────────────────────────────────────────── - us_data = load_dataset() - us_dc = load_dc_config() - # Use 6-month window matching both datasets - us_window = us_data[ - (us_data["TIMESTAMP"] >= "2023-01-01") & - (us_data["TIMESTAMP"] < "2023-07-01") - ].copy() - us_jobs = _build_jobs(us_dc, n_jobs, "2023-01-01", "2023-06-30", seed) - - # ── Load EU data ────────────────────────────────────────────────────────── - eu_data = pd.read_csv(EU_DATA_PATH) - eu_dc = pd.read_csv(EU_DC_PATH) - eu_window = eu_data[ - (eu_data["TIMESTAMP"] >= "2023-01-01") & - (eu_data["TIMESTAMP"] < "2023-07-01") - ].copy() - eu_jobs = _build_jobs(eu_dc, n_jobs, "2023-01-01", "2023-06-30", seed) - - # ── Run both regions ────────────────────────────────────────────────────── - us_result = _run_region("US", us_window, us_dc, us_jobs, alpha, beta, gamma) - eu_result = _run_region("EU", eu_window, eu_dc, eu_jobs, alpha, beta, gamma) - - return pd.DataFrame([us_result, eu_result]) - - -def _print_report(df: pd.DataFrame, n_jobs: int, alpha: float, beta: float) -> None: - us = df[df["region"] == "US"].iloc[0] - eu = df[df["region"] == "EU"].iloc[0] - - print("=" * 70) - print(f"CROSS-REGION COMPARISON — {n_jobs} jobs | α={alpha}, β={beta}") - print("=" * 70) - - print(f"\n{'GRID CHARACTERISTICS':}") - print(f" {'Metric':<30} {'US':>12} {'EU':>12}") - print(f" {'-'*56}") - print(f" {'Cities':<30} {int(us['n_cities']):>12} {int(eu['n_cities']):>12}") - print(f" {'CO2 min (kg/kWh)':<30} {us['co2_min_kg_kwh']:>12.4f} {eu['co2_min_kg_kwh']:>12.4f}") - print(f" {'CO2 max (kg/kWh)':<30} {us['co2_max_kg_kwh']:>12.4f} {eu['co2_max_kg_kwh']:>12.4f}") - print(f" {'CO2 spread (max/min)':<30} {us['co2_spread_x']:>11.1f}× {eu['co2_spread_x']:>11.1f}×") - print(f" {'Cleanest city':<30} {us['cleanest_city']:>12} {eu['cleanest_city']:>12}") - print(f" {'Dirtiest city':<30} {us['dirtiest_city']:>12} {eu['dirtiest_city']:>12}") - - print(f"\n{'SCHEDULING RESULTS':}") - print(f" {'Metric':<30} {'US':>12} {'EU':>12}") - print(f" {'-'*56}") - print(f" {'Jobs scheduled':<30} {int(us['jobs_scheduled']):>12} {int(eu['jobs_scheduled']):>12}") - print(f" {'Coverage':<30} {us['coverage_pct']:>11.1f}% {eu['coverage_pct']:>11.1f}%") - print(f" {'Naive CO2 (kg)':<30} {us['naive_co2_kg']:>12,.1f} {eu['naive_co2_kg']:>12,.1f}") - print(f" {'Optimal CO2 (kg)':<30} {us['optimal_co2_kg']:>12,.1f} {eu['optimal_co2_kg']:>12,.1f}") - print(f" {'CO2 reduction':<30} {us['co2_reduction_pct']:>11.1f}% {eu['co2_reduction_pct']:>11.1f}%") - print(f" {'CO2 saved per job (kg)':<30} {us['co2_saved_per_job_kg']:>12,.1f} {eu['co2_saved_per_job_kg']:>12,.1f}") - print(f" {'Water reduction':<30} {us['water_reduction_pct']:>11.1f}% {eu['water_reduction_pct']:>11.1f}%") - print(f" {'Jobs to top-3 clean cities':<30} {us['frac_to_top3_clean']:>11.1f}% {eu['frac_to_top3_clean']:>11.1f}%") - - print(f"\n{'KEY FINDING':}") - eu_lift = eu['co2_reduction_pct'] - us['co2_reduction_pct'] - print(f" EU routing saves {eu['co2_reduction_pct']:.1f}% CO2 vs {us['co2_reduction_pct']:.1f}% for US") - print(f" → Smart routing is {eu_lift:.1f} percentage points more impactful in the EU") - print(f" → EU CO2 spread ({eu['co2_spread_x']:.0f}×) vs US ({us['co2_spread_x']:.0f}×) explains the gap") - print(f" → Top-3 clean EU cities: {eu['top3_clean_cities']}") - print("=" * 70) - - -def main() -> None: - parser = argparse.ArgumentParser( - description="US vs EU cross-region scheduling comparison" - ) - parser.add_argument("--jobs", type=int, default=100, help="Jobs per region") - parser.add_argument("--alpha", type=float, default=0.7, help="Carbon weight") - parser.add_argument("--beta", type=float, default=0.2, help="Water weight") - parser.add_argument("--output", default=str(OUT_PATH), help="Output CSV path") - args = parser.parse_args() - - if not EU_DATA_PATH.exists(): - print(f"[ERROR] EU dataset not found: {EU_DATA_PATH}") - print(" Run: python -m src.generate_eu_synthetic_data") - print(" Then: python -m src.build_eu_dataset") - return - - df = run_cross_region_comparison( - n_jobs=args.jobs, - alpha=args.alpha, - beta=args.beta, - ) - - _print_report(df, args.jobs, args.alpha, args.beta) - - out = Path(args.output) - out.parent.mkdir(parents=True, exist_ok=True) - df.to_csv(out, index=False) - print(f"\nSaved → {out}") - - -if __name__ == "__main__": - main() diff --git a/src/data_loader.py b/src/data_loader.py index 575c8f9..3feca07 100644 --- a/src/data_loader.py +++ b/src/data_loader.py @@ -1,133 +1,28 @@ +"""Small, explicit table loader used by optional data adapters.""" + from __future__ import annotations -from io import BytesIO from pathlib import Path -import zipfile import pandas as pd - PROJECT_ROOT = Path(__file__).resolve().parents[1] -DEFAULT_DATA_CANDIDATES = [ - PROJECT_ROOT / "data" / "base_data_with_metrics.parquet", - PROJECT_ROOT / "data" / "base_data_with_metrics.csv", - PROJECT_ROOT / "data" / "sample_dataset.xlsx", -] -ZIP_MEMBER_CANDIDATES = [ - "base_data_with_metrics.parquet", - "base_data_with_metrics.csv", -] - - -def _find_default_data_path() -> Path: - for candidate in DEFAULT_DATA_CANDIDATES: - if candidate.exists() and candidate.is_file() and candidate.stat().st_size > 0: - return candidate - return DEFAULT_DATA_CANDIDATES[-1] -def resolve_data_path(path: str | Path | None = None) -> Path: - """Resolve a dataset path against the project root.""" - if path is None: - return _find_default_data_path() - +def resolve_project_path(path: str | Path) -> Path: candidate = Path(path) - if candidate.is_absolute(): - return candidate - return PROJECT_ROOT / candidate - - -def _normalize_timestamp_columns(df: pd.DataFrame) -> pd.DataFrame: - for column in ["TIMESTAMP", "Timestamp", "timestamp"]: - if column in df.columns: - df[column] = pd.to_datetime(df[column], errors="coerce") - return df - - -def _read_csv(path_or_buffer) -> pd.DataFrame: - df = pd.read_csv(path_or_buffer, low_memory=False) - return _normalize_timestamp_columns(df) - - -def _read_excel(path: Path) -> pd.DataFrame: - df = pd.read_excel(path) - return _normalize_timestamp_columns(df) - - -def _read_parquet(path_or_buffer) -> pd.DataFrame: - df = pd.read_parquet(path_or_buffer) - return _normalize_timestamp_columns(df) - - -def _read_regular_file(data_path: Path) -> pd.DataFrame: - suffix = data_path.suffix.lower() - if suffix == ".xlsx": - return _read_excel(data_path) - if suffix == ".csv": - return _read_csv(data_path) - if suffix == ".parquet": - return _read_parquet(data_path) - - raise ValueError( - f"Unsupported dataset format: {data_path.suffix}. " - "Supported formats are .xlsx, .csv, .parquet, and .zip." - ) - - -def _select_zip_member(zf: zipfile.ZipFile) -> str: - members = [ - name - for name in zf.namelist() - if not name.endswith("/") and not name.startswith("__MACOSX/") - ] - - for candidate in ZIP_MEMBER_CANDIDATES: - if candidate in members: - return candidate - - for member in members: - if member.lower().endswith(".parquet"): - return member - - for member in members: - if member.lower().endswith(".csv"): - return member - - raise FileNotFoundError( - "No supported dataset file found inside ZIP archive. " - "Expected a .parquet or .csv member." - ) - - -def _read_from_zip(data_path: Path) -> pd.DataFrame: - with zipfile.ZipFile(data_path) as zf: - member = _select_zip_member(zf) - if member.lower().endswith(".parquet"): - buffer = BytesIO(zf.read(member)) - return _read_parquet(buffer) - with zf.open(member) as fh: - return _read_csv(fh) - - -def load_dataset(path: str | Path | None = None) -> pd.DataFrame: - """Load the dataset from xlsx, csv, parquet, or zip.""" - data_path = resolve_data_path(path) - - if not data_path.exists(): - raise FileNotFoundError( - f"Dataset not found at {data_path}. " - "Place your dataset in data/ or pass --data-path." - ) - - if data_path.is_file() and data_path.stat().st_size == 0: - raise ValueError( - f"Dataset file exists but is empty at {data_path}. " - "Replace it with a valid dataset file." - ) - - try: - if data_path.suffix.lower() == ".zip": - return _read_from_zip(data_path) - return _read_regular_file(data_path) - except Exception as exc: - raise RuntimeError(f"Failed to read dataset at {data_path}: {exc}") from exc + return candidate if candidate.is_absolute() else PROJECT_ROOT / candidate + + +def load_table(path: str | Path) -> pd.DataFrame: + """Load a CSV or Parquet table without developer-specific fallbacks.""" + source = resolve_project_path(path) + if not source.is_file(): + raise FileNotFoundError(f"Dataset not found: {source}") + if source.stat().st_size == 0: + raise ValueError(f"Dataset is empty: {source}") + if source.suffix.lower() == ".csv": + return pd.read_csv(source, low_memory=False) + if source.suffix.lower() == ".parquet": + return pd.read_parquet(source) + raise ValueError("Supported table formats are .csv and .parquet.") diff --git a/src/evaluate.py b/src/evaluate.py deleted file mode 100644 index 4003360..0000000 --- a/src/evaluate.py +++ /dev/null @@ -1,54 +0,0 @@ -from __future__ import annotations - -from math import sqrt - -from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score - - -def regression_metrics(y_true, y_pred) -> dict[str, float]: - """Return RMSE, MAE, and R2 for a regression prediction.""" - return { - "rmse": float(sqrt(mean_squared_error(y_true, y_pred))), - "mae": float(mean_absolute_error(y_true, y_pred)), - "r2": float(r2_score(y_true, y_pred)), - } - - -def compare_to_baseline( - model_metrics: dict[str, float], baseline_metrics: dict[str, float] -) -> dict[str, float]: - """Compare model performance against a baseline using RMSE.""" - rmse_delta = baseline_metrics["rmse"] - model_metrics["rmse"] - rmse_improvement_pct = ( - (rmse_delta / baseline_metrics["rmse"]) * 100 - if baseline_metrics["rmse"] - else 0.0 - ) - return { - "rmse_delta": float(rmse_delta), - "rmse_improvement_pct": float(rmse_improvement_pct), - } - - -def print_metrics(label: str, metrics: dict[str, float]) -> None: - """Pretty-print regression metrics.""" - print(f"{label} metrics:") - print(f" RMSE: {metrics['rmse']:.4f}") - print(f" MAE: {metrics['mae']:.4f}") - print(f" R2: {metrics['r2']:.4f}") - - -def print_baseline_comparison( - label: str, - model_metrics: dict[str, float], - baseline_metrics: dict[str, float], -) -> None: - """Print RMSE comparison between the trained model and the baseline.""" - comparison = compare_to_baseline(model_metrics, baseline_metrics) - direction = "better" if comparison["rmse_delta"] >= 0 else "worse" - - print(f"{label} RMSE comparison:") - print(f" Baseline RMSE: {baseline_metrics['rmse']:.4f}") - print(f" Model RMSE: {model_metrics['rmse']:.4f}") - print(f" Difference: {abs(comparison['rmse_delta']):.4f} ({direction} than baseline)") - print(f" Improvement: {comparison['rmse_improvement_pct']:.2f}%") diff --git a/src/experiments.py b/src/experiments.py new file mode 100644 index 0000000..aeb8c3b --- /dev/null +++ b/src/experiments.py @@ -0,0 +1,325 @@ +"""Run the complete offline benchmark and write auditable research artifacts.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +from pathlib import Path + +import matplotlib +import numpy as np +import pandas as pd + +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +from .baselines import run_baseline +from .forecast_robustness import run_forecast_robustness +from .scheduling import paired_comparison, schedule_jobs, summarize_schedule +from .synthetic_benchmark import generate_benchmark + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_RESULTS_DIR = PROJECT_ROOT / "results" +DEFAULT_FIGURES_DIR = PROJECT_ROOT / "docs" / "figures" +BASELINES = ("immediate_local", "carbon_greedy", "random") +PAIRED_METRICS = ( + "co2_reduction_pct", + "water_reduction_pct", + "scarcity_weighted_water_reduction_pct", + "baseline_coverage", + "proposed_coverage", + "coverage_gap_percentage_points", + "common_jobs", + "common_it_energy_kwh", +) + + +def _mean_ci95(values: pd.Series) -> tuple[float, float]: + numeric = pd.to_numeric(values, errors="coerce").dropna().to_numpy(dtype=float) + mean = float(numeric.mean()) + ci = ( + float(1.96 * numeric.std(ddof=1) / math.sqrt(len(numeric))) + if len(numeric) > 1 + else 0.0 + ) + return mean, ci + + +def _aggregate_paired(frame: pd.DataFrame) -> pd.DataFrame: + rows: list[dict[str, float | str]] = [] + for baseline, group in frame.groupby("baseline"): + row: dict[str, float | str] = {"baseline": str(baseline)} + for metric in PAIRED_METRICS: + mean, ci = _mean_ci95(group[metric]) + row[f"{metric}_mean"] = mean + row[f"{metric}_ci95"] = ci + row["seeds"] = float(len(group)) + rows.append(row) + return pd.DataFrame(rows).sort_values("baseline").reset_index(drop=True) + + +def _plot_benchmark(summary: pd.DataFrame, destination: Path) -> None: + labels = [value.replace("_", " ").title() for value in summary["baseline"]] + positions = np.arange(len(labels)) + width = 0.36 + fig, axes = plt.subplots(1, 2, figsize=(12, 4.8)) + + axes[0].bar( + positions - width / 2, + summary["co2_reduction_pct_mean"], + width, + yerr=summary["co2_reduction_pct_ci95"], + capsize=4, + label="Operational CO2e", + color="#2f6b3b", + ) + axes[0].bar( + positions + width / 2, + summary["scarcity_weighted_water_reduction_pct_mean"], + width, + yerr=summary["scarcity_weighted_water_reduction_pct_ci95"], + capsize=4, + label="Scarcity-weighted water", + color="#3478a8", + ) + axes[0].axhline(0, color="#222222", linewidth=0.8) + axes[0].set_ylabel("Paired-job reduction (%)") + axes[0].set_title("Proposed policy relative to each baseline") + axes[0].set_xticks(positions, labels, rotation=15, ha="right") + axes[0].legend(frameon=False) + axes[0].grid(axis="y", alpha=0.25) + + axes[1].errorbar( + positions, + 100 * summary["baseline_coverage_mean"], + yerr=100 * summary["baseline_coverage_ci95"], + marker="o", + linewidth=2, + capsize=4, + label="Baseline", + color="#777777", + ) + axes[1].errorbar( + positions, + 100 * summary["proposed_coverage_mean"], + yerr=100 * summary["proposed_coverage_ci95"], + marker="o", + linewidth=2, + capsize=4, + label="Proposed", + color="#7b3f98", + ) + axes[1].set_ylim(0, 105) + axes[1].set_ylabel("Jobs completed before deadline (%)") + axes[1].set_title("Coverage is reported separately") + axes[1].set_xticks(positions, labels, rotation=15, ha="right") + axes[1].legend(frameon=False) + axes[1].grid(axis="y", alpha=0.25) + + fig.suptitle("Synthetic benchmark, mean and approximate 95% CI across seeds") + fig.tight_layout() + fig.savefig(destination, dpi=180, bbox_inches="tight") + plt.close(fig) + + +def _plot_robustness(frame: pd.DataFrame, destination: Path) -> None: + x = 100 * frame["noise_level"].to_numpy(dtype=float) + y = frame["carbon_delta_pct_mean"].to_numpy(dtype=float) + ci = frame["carbon_delta_pct_ci95"].to_numpy(dtype=float) + fig, ax = plt.subplots(figsize=(7.5, 4.8)) + ax.plot(x, y, marker="o", linewidth=2.2, color="#9a4d1f") + ax.fill_between(x, y - ci, y + ci, alpha=0.20, color="#9a4d1f") + ax.axhline(0, color="#222222", linewidth=0.8) + ax.set_xlabel("Forecast noise standard deviation (% of city mean)") + ax.set_ylabel("CO2e delta vs perfect-forecast policy (%)") + ax.set_title("Carbon-only forecast stress test on paired completed jobs") + ax.grid(alpha=0.25) + fig.tight_layout() + fig.savefig(destination, dpi=180, bbox_inches="tight") + plt.close(fig) + + +def _plot_example_schedule(frame: pd.DataFrame, destination: Path) -> None: + display = frame.loc[ + frame["scheduled"].eq(1), + [ + "job_id", + "origin_city", + "assigned_city", + "scheduled_start", + "expected_co2_kg", + "expected_water_liters", + ], + ].head(8).copy() + display["scheduled_start"] = pd.to_datetime(display["scheduled_start"]).dt.strftime( + "%Y-%m-%d %H:%M" + ) + display["expected_co2_kg"] = display["expected_co2_kg"].map(lambda value: f"{value:.2f}") + display["expected_water_liters"] = display["expected_water_liters"].map( + lambda value: f"{value:.2f}" + ) + display.columns = [ + "Job", + "Origin", + "Assigned zone", + "UTC start", + "CO2e (kg)", + "Water (L)", + ] + fig, ax = plt.subplots(figsize=(12, 3.8)) + ax.axis("off") + table = ax.table( + cellText=display.values, + colLabels=display.columns, + loc="center", + cellLoc="left", + colLoc="left", + ) + table.auto_set_font_size(False) + table.set_fontsize(9) + table.scale(1, 1.6) + for column in range(len(display.columns)): + table[(0, column)].set_facecolor("#dbe9df") + table[(0, column)].set_text_props(weight="bold") + ax.set_title("Example proposed schedule, synthetic seed 42", pad=18, fontsize=14) + fig.tight_layout() + fig.savefig(destination, dpi=180, bbox_inches="tight") + plt.close(fig) + + +def _write_checksums(paths: dict[str, Path], destination: Path) -> None: + lines = [] + for label, path in sorted(paths.items()): + if label == "checksums": + continue + digest = hashlib.sha256(path.read_bytes()).hexdigest() + relative_path = os.path.relpath(path, start=destination.parent) + lines.append(f"{digest} {relative_path}") + destination.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def run_experiments( + results_dir: Path = DEFAULT_RESULTS_DIR, + figures_dir: Path = DEFAULT_FIGURES_DIR, + seeds: int = 10, + jobs: int = 180, + hours: int = 168, +) -> dict[str, Path]: + """Run all policies, paired comparisons, and forecast-error stress tests.""" + if seeds < 2: + raise ValueError("Use at least two seeds to estimate uncertainty.") + results_dir.mkdir(parents=True, exist_ok=True) + figures_dir.mkdir(parents=True, exist_ok=True) + + policy_rows: list[dict[str, float | str]] = [] + paired_rows: list[dict[str, float | str]] = [] + example_schedule: pd.DataFrame | None = None + + for seed in range(42, 42 + seeds): + env, jobs_df, dc_df = generate_benchmark(seed=seed, jobs=jobs, hours=hours) + proposed = schedule_jobs(env, jobs_df, dc_df) + if example_schedule is None: + example_schedule = proposed.head(25).copy() + proposed_summary = summarize_schedule(proposed) + policy_rows.append({"seed": seed, "policy": "proposed", **proposed_summary}) + + for baseline_name in BASELINES: + baseline = run_baseline( + baseline_name, env, jobs_df, dc_df, seed=seed + ) + policy_rows.append( + { + "seed": seed, + "policy": baseline_name, + **summarize_schedule(baseline), + } + ) + paired_rows.append( + { + "seed": seed, + "baseline": baseline_name, + **paired_comparison(baseline, proposed), + } + ) + + policy_frame = pd.DataFrame(policy_rows) + paired_frame = pd.DataFrame(paired_rows) + paired_summary = _aggregate_paired(paired_frame) + + env, jobs_df, dc_df = generate_benchmark(seed=42, jobs=jobs, hours=hours) + robustness = run_forecast_robustness( + env, + dc_df, + jobs_df, + repetitions=max(8, seeds), + seed=42, + ) + + paths = { + "policy_by_seed": results_dir / "policy_summary_by_seed.csv", + "paired_by_seed": results_dir / "paired_comparisons_by_seed.csv", + "paired_summary": results_dir / "benchmark_summary.csv", + "robustness": results_dir / "forecast_robustness.csv", + "example_schedule": results_dir / "example_schedule.csv", + "metadata": results_dir / "run_metadata.json", + "benchmark_figure": figures_dir / "benchmark_overview.png", + "robustness_figure": figures_dir / "forecast_robustness.png", + "schedule_figure": figures_dir / "example_schedule.png", + "checksums": results_dir / "SHA256SUMS", + } + policy_frame.to_csv(paths["policy_by_seed"], index=False, float_format="%.8f") + paired_frame.to_csv(paths["paired_by_seed"], index=False, float_format="%.8f") + paired_summary.to_csv(paths["paired_summary"], index=False, float_format="%.8f") + robustness.to_csv(paths["robustness"], index=False, float_format="%.8f") + assert example_schedule is not None + example_schedule.to_csv(paths["example_schedule"], index=False, float_format="%.8f") + paths["metadata"].write_text( + json.dumps( + { + "benchmark": "synthetic-v1", + "seed_start": 42, + "seed_count": seeds, + "jobs_per_seed": jobs, + "hours_per_seed": hours, + "weights": {"carbon": 0.55, "water": 0.35, "relocation": 0.10}, + "forecast_repetitions": max(8, seeds), + "claim_scope": "simulation only", + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + _plot_benchmark(paired_summary, paths["benchmark_figure"]) + _plot_robustness(robustness, paths["robustness_figure"]) + _plot_example_schedule(example_schedule, paths["schedule_figure"]) + _write_checksums(paths, paths["checksums"]) + return paths + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--seeds", type=int, default=10) + parser.add_argument("--jobs", type=int, default=180) + parser.add_argument("--hours", type=int, default=168) + parser.add_argument("--results-dir", type=Path, default=DEFAULT_RESULTS_DIR) + parser.add_argument("--figures-dir", type=Path, default=DEFAULT_FIGURES_DIR) + args = parser.parse_args() + paths = run_experiments( + results_dir=args.results_dir, + figures_dir=args.figures_dir, + seeds=args.seeds, + jobs=args.jobs, + hours=args.hours, + ) + print("Research artifacts written:") + for label, path in paths.items(): + print(f" {label}: {path}") + + +if __name__ == "__main__": + main() diff --git a/src/explain.py b/src/explain.py deleted file mode 100644 index d739b5e..0000000 --- a/src/explain.py +++ /dev/null @@ -1,135 +0,0 @@ -"""SHAP feature importance for CO2 and WUE XGBoost models.""" -from __future__ import annotations - -from functools import lru_cache -from pathlib import Path -from typing import Any - -import joblib -import numpy as np -import pandas as pd -# shap is imported lazily inside compute_shap() so the module can be imported -# even when shap is not installed (e.g., running unit tests without the full -# requirements.txt, or importing api.py to check routes). -# Lazy import pattern: `import shap` moves inside the function body. - -from .data_loader import PROJECT_ROOT -from .preprocessing import ( - FutureFeatureProfiles, - build_feature_matrix, - build_profile_feature_frame, -) - -MODELS_DIR = PROJECT_ROOT / "models" - -FEATURE_LABELS: dict[str, str] = { - "temperature_c": "Temperature (°C)", - "humidity_pct": "Humidity (%)", - "wind_speed_ms": "Wind Speed (m/s)", - "scarcity_index": "Water Scarcity", - "hour": "Hour of Day", - "day_of_week": "Day of Week", - "month": "Month", - "day_of_year": "Day of Year", - "is_weekend": "Is Weekend", - "hour_sin": "Hour (sin)", - "hour_cos": "Hour (cos)", - "day_of_week_sin": "Day of Week (sin)", - "day_of_week_cos": "Day of Week (cos)", - "month_sin": "Month (sin)", - "month_cos": "Month (cos)", -} - - -@lru_cache(maxsize=2) -def _load_bundle(target: str) -> dict[str, Any]: - """Load and cache a model bundle. target is 'co2' or 'wue'.""" - filename = "co2_model.pkl" if target == "co2" else "wue_model.pkl" - path = MODELS_DIR / filename - if not path.exists(): - raise FileNotFoundError(f"Model bundle not found: {path}") - return joblib.load(path) - - -@lru_cache(maxsize=2) -def _load_explainer(target: str) -> tuple[Any, dict[str, Any]]: - """Build and cache a TreeExplainer for the direct XGBoost model.""" - import shap # lazy import — only loaded when SHAP is actually needed - bundle = _load_bundle(target) - explainer = shap.TreeExplainer(bundle["direct_xgboost_model"]) - return explainer, bundle - - -def compute_shap( - target: str, - city: str, - time: str, - top_n: int = 8, -) -> dict[str, Any]: - """ - Compute SHAP feature importances for a single city at a given timestamp. - - Parameters - ---------- - target : 'co2' or 'wue' - city : data center city name (must match training data) - time : ISO 8601 timestamp string - top_n : number of top features to return (by |shap_value|) - - Returns - ------- - dict with keys: - city, target, prediction, base_value, features (list of dicts) - """ - if target not in ("co2", "wue"): - raise ValueError(f"target must be 'co2' or 'wue', got: {target!r}") - - explainer, bundle = _load_explainer(target) - - # Reconstruct FutureFeatureProfiles from the serialized dict stored in the bundle - serialized = bundle["future_feature_profiles"] - profiles = FutureFeatureProfiles( - hourly_profiles=pd.DataFrame(serialized["hourly_profiles"]), - city_profiles=pd.DataFrame(serialized["city_profiles"]), - global_profile=serialized["global_profile"], - profile_columns=serialized["profile_columns"], - profile_history_days=int(serialized["profile_history_days"]), - ) - - # Build feature row for this city + timestamp using stored historical profiles - time_city = pd.DataFrame([{"timestamp": pd.Timestamp(time).floor("h"), "city": city}]) - feature_frame = build_profile_feature_frame(time_city, profiles) - - X, _ = build_feature_matrix( - feature_frame, - feature_columns=bundle["feature_columns"], - encoded_columns=bundle["encoded_columns"], - ) - - shap_values = explainer.shap_values(X) # shape (1, n_features) - shap_row = np.array(shap_values[0]) if isinstance(shap_values, list) else shap_values[0] - - # Collect base features only (skip one-hot city columns — city identity is obvious) - encoded_columns: list[str] = bundle["encoded_columns"] - features: list[dict[str, Any]] = [] - for i, col in enumerate(encoded_columns): - if col.startswith("city_"): - continue - features.append({ - "name": FEATURE_LABELS.get(col, col), - "raw_name": col, - "value": float(X.iloc[0][col]), - "shap_value": float(shap_row[i]), - }) - - features.sort(key=lambda f: abs(f["shap_value"]), reverse=True) - - return { - "city": city, - "target": target, - "prediction": float(bundle["direct_xgboost_model"].predict(X)[0]), - "base_value": float(explainer.expected_value) - if not isinstance(explainer.expected_value, np.ndarray) - else float(explainer.expected_value[0]), - "features": features[:top_n], - } diff --git a/src/fetch_eia.py b/src/fetch_eia.py index 820b825..1a6b9d1 100644 --- a/src/fetch_eia.py +++ b/src/fetch_eia.py @@ -4,7 +4,7 @@ Fetch hourly electricity generation mix from the EIA (Energy Information Administration) for the 7 grid regions covering your 28 AI data center locations. -EIA API is completely free — register for a key at: +EIA API is free to use. Register for a key at: https://www.eia.gov/opendata/register.php (instant approval) Produces: data/processed/eia_generation_mix.csv @@ -86,7 +86,7 @@ def fetch_region_generation( resp = requests.get(GEN_URL, params=params, timeout=60) if resp.status_code != 200: - print(f" [WARN] EIA {ba}: {resp.status_code} — {resp.text[:120]}") + print(f" [WARN] EIA {ba}: {resp.status_code}: {resp.text[:120]}") return [] return resp.json().get("response", {}).get("data", []) @@ -115,13 +115,32 @@ def fetch_all_pages(api_key: str, ba: str, start: str, end: str) -> pd.DataFrame def process_region(df: pd.DataFrame, ba: str) -> pd.DataFrame: """Clean and pivot raw EIA rows into wide format with one column per fuel type.""" - df = df.rename(columns={"period": "timestamp", "type-name": "fuel_label"}) + df = df.copy().rename(columns={"period": "timestamp"}) df["timestamp"] = pd.to_datetime(df["timestamp"], utc=True) df["generation_mwh"] = pd.to_numeric(df["value"], errors="coerce").fillna(0) - df["fuel_code"] = df["type-name"] if "type-name" in df.columns else df.get("fueltype", "OTH") + + fuel_column = next( + (column for column in ("type", "fueltype", "type-name") if column in df.columns), + None, + ) + if fuel_column is None: + raise ValueError("EIA response is missing a fuel type column.") + df["fuel_code"] = df[fuel_column].astype(str).str.strip().str.upper() # Normalise fuel codes to our standard names - df["fuel"] = df["fuel_code"].map(FUEL_TYPES).fillna("other") + label_aliases = { + "COAL": "coal", + "NATURAL GAS": "natural_gas", + "NUCLEAR": "nuclear", + "SOLAR": "solar", + "WIND": "wind", + "HYDRO": "hydro", + "PETROLEUM": "petroleum", + "OTHER": "other", + } + df["fuel"] = ( + df["fuel_code"].map(FUEL_TYPES).fillna(df["fuel_code"].map(label_aliases)) + ).fillna("other") # Pivot: one row per timestamp, one column per fuel pivoted = ( @@ -138,7 +157,7 @@ def process_region(df: pd.DataFrame, ba: str) -> pd.DataFrame: pivoted["total_mwh"] = pivoted[[f for f in FUEL_TYPES.values()]].sum(axis=1) - # Compute fuel share fractions (0–1) + # Compute fuel share fractions from 0 to 1. for fuel in FUEL_TYPES.values(): pivoted[f"{fuel}_share"] = ( pivoted[fuel] / pivoted["total_mwh"].replace(0, float("nan")) @@ -153,11 +172,11 @@ def process_region(df: pd.DataFrame, ba: str) -> pd.DataFrame: def fetch_all(api_key: str, start: str, end: str) -> pd.DataFrame: all_frames: list[pd.DataFrame] = [] - print(f"Fetching EIA generation mix for {len(BA_REGIONS)} regions ({start} → {end})") + print(f"Fetching EIA generation mix for {len(BA_REGIONS)} regions ({start} to {end})") print("Source: EIA Open Data API (free)\n") for ba in BA_REGIONS: - print(f" → {ba}", end=" ... ", flush=True) + print(f" {ba}", end=" ... ", flush=True) raw = fetch_all_pages(api_key, ba, start, end) if raw.empty: @@ -166,11 +185,12 @@ def fetch_all(api_key: str, start: str, end: str) -> pd.DataFrame: processed = process_region(raw, ba) all_frames.append(processed) - print(f"{len(processed):,} hourly rows | fuels: {[c for c in processed.columns if '_share' in c]}") + fuel_columns = [column for column in processed if column.endswith("_share")] + print(f"{len(processed):,} hourly rows | fuels: {fuel_columns}") time.sleep(0.5) if not all_frames: - raise RuntimeError("No EIA data fetched — check your API key and connectivity.") + raise RuntimeError("No EIA data fetched. Check your API key and connectivity.") return pd.concat(all_frames, ignore_index=True) @@ -189,9 +209,9 @@ def main() -> None: output_path.parent.mkdir(parents=True, exist_ok=True) df.to_csv(output_path, index=False) - print(f"\nSaved {len(df):,} rows → {output_path}") + print(f"\nSaved {len(df):,} rows to {output_path}") print(f"Regions: {sorted(df['ba_region'].unique())}") - print(f"Date range: {df['timestamp'].min()} → {df['timestamp'].max()}") + print(f"Date range: {df['timestamp'].min()} to {df['timestamp'].max()}") if __name__ == "__main__": diff --git a/src/fetch_entso_e.py b/src/fetch_entso_e.py index 3638852..5fa44b1 100644 --- a/src/fetch_entso_e.py +++ b/src/fetch_entso_e.py @@ -4,7 +4,7 @@ Fetch hourly electricity generation mix from ENTSO-E Transparency Platform for the 10 EU data center zones. -ENTSO-E API is free — register for a token at: +ENTSO-E API is free to use. Register for a token at: https://transparency.entsoe.eu/usrm/user/createPublicUser Produces: data/processed/entso_e_generation_mix.csv @@ -13,9 +13,9 @@ timestamp, zone, fuel_type, generation_mwh Fuel types mapped from ENTSO-E psrType codes to common names: - B01=biomass, B02=coal, B04=gas, B09=hydro, B10=hydro_pumped, - B11=hydro_run, B16=solar, B18=wind_offshore, B19=wind_onshore, - B13=nuclear, B17=other + B01=biomass, B02/B03/B05=coal, B04=gas, B06/B07=oil, + B10/B11/B12/B13=hydro, B14=nuclear, B17=solar, + B19/B20=wind, and remaining categories=other. Usage: python -m src.fetch_entso_e --token YOUR_TOKEN @@ -26,11 +26,11 @@ import argparse import time -import xml.etree.ElementTree as ET from pathlib import Path import pandas as pd import requests +from defusedxml import ElementTree as ET try: from .data_loader import PROJECT_ROOT @@ -57,45 +57,30 @@ } # Map ENTSO-E psrType codes to readable fuel names -# Codes from: https://transparency.entsoe.eu/content/static_content/Static content/web api/Guide.html +# Codes follow the ENTSO-E Energy Identification Codes list. PSR_TYPE_MAP = { "B01": "biomass", "B02": "coal", - "B03": "gas", + "B03": "coal", "B04": "gas", - "B05": "coal", # hard coal + "B05": "coal", "B06": "oil", "B07": "oil", - "B08": "oil", - "B09": "hydro", # hydro water reservoir - "B10": "hydro", # hydro pumped storage - "B11": "hydro", # hydro run-of-river - "B12": "hydro", # hydro marine - "B13": "nuclear", - "B14": "other", + "B08": "coal", + "B09": "other", + "B10": "hydro", + "B11": "hydro", + "B12": "hydro", + "B13": "hydro", + "B14": "nuclear", "B15": "other", - "B16": "solar", - "B17": "other", - "B18": "wind", # wind offshore - "B19": "wind", # wind onshore - "B20": "other", + "B16": "other", + "B17": "solar", + "B18": "other", + "B19": "wind", + "B20": "wind", } -# EPA-style CO2 factors (kg CO2 per MWh) for ENTSO-E fuel types -# Source: IPCC 2014, European Environment Agency -CO2_KG_PER_MWH = { - "coal": 820.0, - "gas": 490.0, - "oil": 650.0, - "nuclear": 12.0, - "hydro": 4.0, - "solar": 45.0, - "wind": 11.0, - "biomass": 230.0, - "other": 400.0, -} - - def _parse_generation_xml(xml_text: str, zone: str) -> list[dict]: """Parse ENTSO-E ActualGenerationPerProductionType XML response.""" root = ET.fromstring(xml_text) @@ -119,16 +104,11 @@ def _parse_generation_xml(xml_text: str, zone: str) -> list[dict]: continue start_ts = pd.Timestamp(start_el.text, tz="UTC") - resolution = resolution_el.text # PT60M or PT15M - - if "60" in resolution: - freq_h = 1 - elif "30" in resolution: - freq_h = 0 - else: - freq_h = 0 # 15-minute; skip sub-hourly - - if freq_h == 0: + resolution = resolution_el.text or "" + resolution_minutes = {"PT60M": 60, "PT30M": 30, "PT15M": 15}.get( + resolution + ) + if resolution_minutes is None: continue for point in period.findall("ns:Point", ns): @@ -143,17 +123,28 @@ def _parse_generation_xml(xml_text: str, zone: str) -> list[dict]: except (TypeError, ValueError): continue - hour_offset = position - 1 - timestamp = start_ts + pd.Timedelta(hours=hour_offset) + timestamp = start_ts + pd.Timedelta( + minutes=(position - 1) * resolution_minutes + ) rows.append( { - "timestamp": timestamp.tz_localize(None), + "timestamp": timestamp.floor("h").tz_localize(None), "zone": zone, "fuel_type": fuel, - "generation_mwh": generation_mwh, + "generation_mwh": generation_mwh + * (resolution_minutes / 60.0), } ) - return rows + if not rows: + return [] + hourly = ( + pd.DataFrame(rows) + .groupby(["timestamp", "zone", "fuel_type"], as_index=False)[ + "generation_mwh" + ] + .sum() + ) + return hourly.to_dict(orient="records") def fetch_zone_generation( @@ -177,7 +168,7 @@ def fetch_zone_generation( response = requests.get(ENTSO_E_BASE, params=params, timeout=30) response.raise_for_status() except requests.RequestException as exc: - print(f" [WARN] {zone}: request failed — {exc}") + print(f" [WARN] {zone}: request failed: {exc}") return [] if " pd.DataFrame: - """Add co2_intensity_kg_per_kwh column to hourly generation totals.""" - gen_df = gen_df.copy() - gen_df["co2_kg"] = gen_df["fuel_type"].map(CO2_KG_PER_MWH) * gen_df["generation_mwh"] - - hourly = ( - gen_df.groupby(["timestamp", "zone"]) - .agg( - total_gen_mwh=("generation_mwh", "sum"), - total_co2_kg=("co2_kg", "sum"), - ) - .reset_index() - ) - hourly["co2_intensity_kg_per_kwh"] = ( - hourly["total_co2_kg"] / (hourly["total_gen_mwh"] * 1000).replace(0, float("nan")) - ) - return hourly - - def fetch_all_zones( token: str, start: str = "2023-01-01", @@ -218,7 +190,7 @@ def fetch_all_zones( for zone, eic in ZONE_EIC.items(): print(f" Fetching {zone} ({eic})...") rows = fetch_zone_generation(token, zone, eic, start, end) - print(f" → {len(rows):,} rows") + print(f" {len(rows):,} rows") all_rows.extend(rows) time.sleep(sleep_s) @@ -236,7 +208,7 @@ def main() -> None: parser.add_argument("--output", default=str(OUTPUT_PATH), help="Output CSV path") args = parser.parse_args() - print(f"Fetching ENTSO-E generation data: {args.start} → {args.end}") + print(f"Fetching ENTSO-E generation data: {args.start} to {args.end}") gen_df = fetch_all_zones(args.token, args.start, args.end) if gen_df.empty: @@ -246,7 +218,7 @@ def main() -> None: output_path = Path(args.output) output_path.parent.mkdir(parents=True, exist_ok=True) gen_df.to_csv(output_path, index=False) - print(f"\nSaved {len(gen_df):,} rows → {output_path}") + print(f"\nSaved {len(gen_df):,} rows to {output_path}") if __name__ == "__main__": diff --git a/src/fetch_watttime.py b/src/fetch_watttime.py deleted file mode 100644 index f4d86e5..0000000 --- a/src/fetch_watttime.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -fetch_watttime.py — NOT USED ------------------------------ -WattTime's /v3/historical endpoint requires a paid subscription. -The free tier only previews CAISO_NORTH (California), which covers -none of the 28 AI data center locations in this project. - -Carbon intensity is instead computed in build_dataset.py using: - - EIA hourly fuel mix (free) ← fetch_eia.py - - EPA eGRID 2022 emission factors ← hardcoded constants - -This is the standard methodology in peer-reviewed carbon-aware -computing research and is fully reproducible without any paid API. - -If you obtain a WattTime subscription in the future, marginal MOER -would be a stronger signal than average intensity — update the -co2_kg column in build_dataset.py with WattTime values at that point. - -Subscription info: https://watttime.org/solutions/ -""" diff --git a/src/fetch_weather.py b/src/fetch_weather.py deleted file mode 100644 index ca1b996..0000000 --- a/src/fetch_weather.py +++ /dev/null @@ -1,159 +0,0 @@ -""" -fetch_weather.py ----------------- -Fetch historical hourly weather data from Open-Meteo (free, no API key) -for the 28 real AI data center locations defined in city_coordinates.csv. - -Produces: data/processed/dc_weather_historical.csv - -Usage: - python -m src.fetch_weather # 2022-2023 (default) - python -m src.fetch_weather --start 2019-01-01 --end 2023-12-31 - python -m src.fetch_weather --start 2024-01-01 --end 2024-12-31 -""" - -from __future__ import annotations - -import argparse -import time -from pathlib import Path - -import pandas as pd -import requests - -try: - from .data_loader import PROJECT_ROOT -except ImportError: - from data_loader import PROJECT_ROOT - -COORDS_PATH = PROJECT_ROOT / "data" / "reference" / "city_coordinates.csv" -OUTPUT_PATH = PROJECT_ROOT / "data" / "processed" / "dc_weather_historical.csv" - -# Open-Meteo free historical archive endpoint — no API key required -OPEN_METEO_URL = "https://archive-api.open-meteo.com/v1/archive" - -# Variables to fetch (maps to weather_loader.py column names) -HOURLY_VARIABLES = [ - "temperature_2m", # → weather_temp (°C) - "relative_humidity_2m", # → weather_rhum (%) - "wind_speed_10m", # → weather_wspd (km/h) - "precipitation", # → weather_prcp (mm) - "surface_pressure", # → weather_pres (hPa) - "wind_direction_10m", # → weather_wdir (°) - "dew_point_2m", # → weather_dwpt (°C) -] - -COLUMN_RENAME = { - "temperature_2m": "weather_temp", - "relative_humidity_2m": "weather_rhum", - "wind_speed_10m": "weather_wspd", - "precipitation": "weather_prcp", - "surface_pressure": "weather_pres", - "wind_direction_10m": "weather_wdir", - "dew_point_2m": "weather_dwpt", -} - - -def fetch_city_weather( - city: str, - lat: float, - lng: float, - start_date: str, - end_date: str, -) -> pd.DataFrame | None: - params = { - "latitude": lat, - "longitude": lng, - "start_date": start_date, - "end_date": end_date, - "hourly": ",".join(HOURLY_VARIABLES), - "timezone": "America/Chicago", - "wind_speed_unit": "kmh", - } - - try: - resp = requests.get(OPEN_METEO_URL, params=params, timeout=30) - resp.raise_for_status() - data = resp.json() - except Exception as exc: - print(f" [WARN] {city}: request failed — {exc}") - return None - - hourly = data.get("hourly", {}) - if "time" not in hourly: - print(f" [WARN] {city}: no hourly data returned") - return None - - df = pd.DataFrame(hourly) - df = df.rename(columns={"time": "TIMESTAMP"}) - df["TIMESTAMP"] = pd.to_datetime(df["TIMESTAMP"]) - df = df.rename(columns=COLUMN_RENAME) - df.insert(0, "city", city) - - # Derived features (match weather_loader.py conventions) - df["weather_temp_dewpoint_gap"] = df["weather_temp"] - df["weather_dwpt"] - df["weather_rain_flag"] = (df["weather_prcp"].fillna(0) > 0).astype(int) - df["weather_humidity_temp_index"] = df["weather_temp"] * df["weather_rhum"] / 100.0 - - return df - - -def fetch_all( - start_date: str, - end_date: str, - coords_path: Path = COORDS_PATH, -) -> pd.DataFrame: - coords = pd.read_csv(coords_path) - all_frames: list[pd.DataFrame] = [] - - print(f"Fetching weather for {len(coords)} locations ({start_date} → {end_date})") - print("Source: Open-Meteo Historical Archive (free, no API key)\n") - - for _, row in coords.iterrows(): - city = row["city"] - lat = float(row["latitude"]) - lng = float(row["longitude"]) - print(f" → {city} ({lat:.3f}, {lng:.3f})", end=" ... ", flush=True) - - df = fetch_city_weather(city, lat, lng, start_date, end_date) - if df is not None: - all_frames.append(df) - print(f"{len(df):,} rows") - else: - print("skipped") - - # Be polite to the free API - time.sleep(0.4) - - if not all_frames: - raise RuntimeError("No weather data fetched — check connectivity.") - - combined = pd.concat(all_frames, ignore_index=True) - return combined - - -def main() -> None: - parser = argparse.ArgumentParser(description="Fetch historical weather for AI DC locations") - parser.add_argument("--start", default="2022-01-01", help="Start date YYYY-MM-DD") - parser.add_argument("--end", default="2023-12-31", help="End date YYYY-MM-DD") - parser.add_argument("--output", default=str(OUTPUT_PATH), help="Output CSV path") - parser.add_argument( - "--coords", - default=str(COORDS_PATH), - help="City coordinates CSV (use eu_city_coordinates.csv for EU weather)", - ) - args = parser.parse_args() - - df = fetch_all(args.start, args.end, coords_path=Path(args.coords)) - - output_path = Path(args.output) - output_path.parent.mkdir(parents=True, exist_ok=True) - df.to_csv(output_path, index=False) - - print(f"\nSaved {len(df):,} rows × {len(df.columns)} columns → {output_path}") - print(f"Cities covered: {df['city'].nunique()}") - print(f"Date range: {df['TIMESTAMP'].min()} → {df['TIMESTAMP'].max()}") - - -if __name__ == "__main__": - main() diff --git a/src/forecast_robustness.py b/src/forecast_robustness.py index 39ea42c..96c63b2 100644 --- a/src/forecast_robustness.py +++ b/src/forecast_robustness.py @@ -1,323 +1,155 @@ -""" -forecast_robustness.py ----------------------- -Evaluate how the carbon-aware scheduler degrades as CO2 forecast error -increases. Simulates the real-world scenario where a scheduler must act -on imperfect predictions (e.g., day-ahead forecasts vs realized values). - -Methodology: - 1. Take realized hourly CO2 intensity as ground truth. - 2. Add Gaussian noise (σ = noise_level × mean) to simulate forecast error. - 3. Schedule jobs using the noisy forecast. - 4. Measure actual CO2 by applying ground-truth intensity to scheduled slots. - 5. Compare against the oracle (scheduled on perfect forecast). - -Metric: "forecast regret" = (actual CO2 under noisy schedule) - (oracle CO2) - expressed as percentage of oracle CO2. - -Usage: - python -m src.forecast_robustness - python -m src.forecast_robustness --jobs 100 --region US - python -m src.forecast_robustness --jobs 50 --region EU -""" +"""Forecast-error stress test with paired jobs and full-window accounting.""" from __future__ import annotations -import argparse -from pathlib import Path +import math import numpy as np import pandas as pd -try: - from .scheduling import ( - prepare_environment_frame, - prepare_scheduler_state, - schedule_jobs, - summarize_schedule, - load_dc_config, - load_jobs_dataset, - ) - from .data_loader import PROJECT_ROOT, load_dataset -except ImportError: - from scheduling import ( - prepare_environment_frame, - prepare_scheduler_state, - schedule_jobs, - summarize_schedule, - load_dc_config, - load_jobs_dataset, - ) - from data_loader import PROJECT_ROOT, load_dataset - -# Noise levels to test: 0% = perfect forecast, 50% = very noisy -NOISE_LEVELS = [0.00, 0.05, 0.10, 0.20, 0.35, 0.50] +from .scheduling import prepare_environment_frame, schedule_jobs -EU_DATA_PATH = PROJECT_ROOT / "data" / "eu_data_with_metrics.csv" -EU_DC_PATH = PROJECT_ROOT / "data" / "templates" / "eu_dc_config.csv" +DEFAULT_NOISE_LEVELS = (0.0, 0.05, 0.10, 0.20, 0.35, 0.50) -def _add_forecast_noise( +def add_forecast_noise( env_df: pd.DataFrame, noise_level: float, rng: np.random.Generator, ) -> pd.DataFrame: - """ - Perturb predicted_co2_intensity with multiplicative Gaussian noise. - - noise_level=0.10 means σ = 10% of each city's mean intensity. - Clipped to [0, ∞) so intensity stays non-negative. - """ - df = env_df.copy() - if noise_level == 0.0: - return df - - city_means = df.groupby("city")["predicted_co2_intensity"].transform("mean") - sigma = noise_level * city_means - noise = rng.normal(loc=0.0, scale=sigma) - df["predicted_co2_intensity"] = np.maximum(0.0, df["predicted_co2_intensity"] + noise) - return df + """Add zero-mean Gaussian error scaled to each city's mean intensity.""" + if noise_level < 0: + raise ValueError("noise_level cannot be negative.") + noisy = prepare_environment_frame(env_df) + if noise_level == 0: + return noisy + city_mean = noisy.groupby("city")[ + "carbon_intensity_kg_per_kwh" + ].transform("mean") + error = rng.normal(0.0, noise_level * city_mean.to_numpy()) + noisy["carbon_intensity_kg_per_kwh"] = np.maximum( + 0.0, + noisy["carbon_intensity_kg_per_kwh"].to_numpy() + error, + ) + return noisy -def _evaluate_schedule_against_truth( +def evaluate_schedule_against_truth( schedule_df: pd.DataFrame, - truth_env: pd.DataFrame, + truth_env_df: pd.DataFrame, + dc_df: pd.DataFrame, ) -> float: - """ - Given a schedule (assignments made on noisy forecast), compute the - actual CO2 that would result using ground-truth intensities. - - Returns total actual CO2 in kg. - """ - scheduled = schedule_df[schedule_df["scheduled"] == 1].copy() + """Compute carbon from every execution hour using realized intensity and PUE.""" + scheduled = schedule_df[schedule_df["scheduled"].eq(1)] if scheduled.empty: return 0.0 - # Build a lookup: city × timestamp → true co2 intensity - truth_lookup = truth_env.set_index(["city", "timestamp"])["predicted_co2_intensity"] - - actual_co2_total = 0.0 + truth = prepare_environment_frame(truth_env_df).set_index(["city", "timestamp"]) + config = dc_df.copy().rename(columns={"max_power_per_hour": "max_power_kw"}) + pue_by_dc = config.set_index("dc_id")["pue"].astype(float).to_dict() + total = 0.0 for _, row in scheduled.iterrows(): - city = row["assigned_city"] - slot = pd.Timestamp(row["scheduled_start"]).floor("h") - power = float(row["power_demand"]) + start = pd.to_datetime(row["scheduled_start"], utc=True).tz_convert(None) duration = int(row["duration_hours"]) + timestamps = pd.date_range(start, periods=duration, freq="h") + keys = pd.MultiIndex.from_product( + [[str(row["assigned_city"])], timestamps], names=["city", "timestamp"] + ) + intensities = truth.reindex(keys)["carbon_intensity_kg_per_kwh"] + if intensities.isna().any(): + raise ValueError( + f"Missing realized carbon data for job {row['job_id']} execution window." + ) + pue = float(pue_by_dc[str(row["assigned_dc_id"])]) + total += float(row["power_demand_kw"]) * pue * float(intensities.sum()) + return total - # Sum true intensity over the duration window - total_kwh = power * duration - true_intensity = truth_lookup.get((city, slot), np.nan) - if np.isnan(true_intensity): - # Fall back to city mean if exact slot missing - city_mask = truth_env["city"] == city - true_intensity = truth_env.loc[city_mask, "predicted_co2_intensity"].mean() - - actual_co2_total += true_intensity * total_kwh - return actual_co2_total +def _mean_ci95(values: list[float]) -> tuple[float, float]: + array = np.asarray(values, dtype=float) + mean = float(array.mean()) + if len(array) < 2: + return mean, 0.0 + return mean, float(1.96 * array.std(ddof=1) / math.sqrt(len(array))) def run_forecast_robustness( - data_df: pd.DataFrame, + env_df: pd.DataFrame, dc_df: pd.DataFrame, jobs_df: pd.DataFrame, - noise_levels: list[float] = NOISE_LEVELS, - alpha: float = 0.7, - beta: float = 0.2, - gamma: float = 0.1, + noise_levels: tuple[float, ...] | list[float] = DEFAULT_NOISE_LEVELS, + repetitions: int = 8, seed: int = 42, ) -> pd.DataFrame: - """ - Run forecast robustness experiment across noise levels. - - Returns a DataFrame with columns: - noise_level, forecast_rmse, oracle_co2_kg, actual_co2_kg, regret_kg, regret_pct, - jobs_scheduled, coverage - """ - rng = np.random.default_rng(seed) - - # Ground-truth environment frame (noise=0) - truth_env = prepare_environment_frame(data_df) - truth_state = prepare_scheduler_state(truth_env, dc_df) + """Compare noisy carbon-only schedules with a perfect-forecast reference.""" + if repetitions <= 0: + raise ValueError("repetitions must be positive.") + truth = prepare_environment_frame(env_df) + reference = schedule_jobs(truth, jobs_df, dc_df, alpha=1.0, beta=0.0, gamma=0.0) + rows: list[dict[str, float]] = [] - rows = [] for noise_level in noise_levels: - # Create noisy forecast - noisy_env = _add_forecast_noise(truth_env, noise_level, rng) - - # Schedule on noisy forecast - noisy_state = prepare_scheduler_state(noisy_env, dc_df) - schedule = schedule_jobs( - noisy_env, - jobs_df, - dc_df, - alpha=alpha, - beta=beta, - gamma=gamma, - prepared_state=noisy_state, + rmse_values: list[float] = [] + delta_values: list[float] = [] + coverage_values: list[float] = [] + paired_counts: list[float] = [] + for repetition in range(repetitions): + rng = np.random.default_rng(seed + 10_000 * repetition + int(noise_level * 1000)) + forecast = add_forecast_noise(truth, float(noise_level), rng) + schedule = schedule_jobs( + forecast, jobs_df, dc_df, alpha=1.0, beta=0.0, gamma=0.0 + ) + + reference_ids = set( + reference.loc[reference["scheduled"].eq(1), "job_id"].astype(str) + ) + schedule_ids = set( + schedule.loc[schedule["scheduled"].eq(1), "job_id"].astype(str) + ) + common_ids = reference_ids & schedule_ids + reference_paired = reference[ + reference["job_id"].astype(str).isin(common_ids) + ] + schedule_paired = schedule[schedule["job_id"].astype(str).isin(common_ids)] + reference_carbon = evaluate_schedule_against_truth( + reference_paired, truth, dc_df + ) + actual_carbon = evaluate_schedule_against_truth(schedule_paired, truth, dc_df) + delta = ( + 100.0 * (actual_carbon - reference_carbon) / reference_carbon + if reference_carbon > 0 + else 0.0 + ) + rmse = float( + np.sqrt( + np.mean( + ( + forecast["carbon_intensity_kg_per_kwh"].to_numpy() + - truth["carbon_intensity_kg_per_kwh"].to_numpy() + ) + ** 2 + ) + ) + ) + rmse_values.append(rmse) + delta_values.append(delta) + coverage_values.append(float(schedule["scheduled"].mean())) + paired_counts.append(float(len(common_ids))) + + rmse_mean, rmse_ci = _mean_ci95(rmse_values) + delta_mean, delta_ci = _mean_ci95(delta_values) + coverage_mean, coverage_ci = _mean_ci95(coverage_values) + rows.append( + { + "noise_level": float(noise_level), + "forecast_rmse_mean": rmse_mean, + "forecast_rmse_ci95": rmse_ci, + "carbon_delta_pct_mean": delta_mean, + "carbon_delta_pct_ci95": delta_ci, + "coverage_mean": coverage_mean, + "coverage_ci95": coverage_ci, + "paired_jobs_mean": float(np.mean(paired_counts)), + "repetitions": float(repetitions), + } ) - if schedule.empty or "scheduled" not in schedule.columns: - continue - summary = summarize_schedule(schedule) - - # Measure actual CO2 using truth intensities - actual_co2 = _evaluate_schedule_against_truth(schedule, truth_env) - - # Forecast RMSE relative to truth - if noise_level == 0.0: - oracle_co2 = actual_co2 - rmse = 0.0 - else: - merged = truth_env.set_index(["city", "timestamp"])["predicted_co2_intensity"].rename("truth") - noisy_indexed = noisy_env.set_index(["city", "timestamp"])["predicted_co2_intensity"] - aligned = pd.concat([merged, noisy_indexed], axis=1).dropna() - aligned.columns = ["truth", "forecast"] - rmse = float(np.sqrt(((aligned["forecast"] - aligned["truth"]) ** 2).mean())) - - regret_kg = actual_co2 - oracle_co2 - regret_pct = (regret_kg / oracle_co2 * 100) if oracle_co2 > 0 else 0.0 - - rows.append({ - "noise_level": noise_level, - "noise_pct": f"{noise_level * 100:.0f}%", - "forecast_rmse": round(rmse, 6), - "oracle_co2_kg": round(oracle_co2, 2), - "actual_co2_kg": round(actual_co2, 2), - "regret_kg": round(regret_kg, 2), - "regret_pct": round(regret_pct, 2), - "jobs_scheduled": int(summary["scheduled_jobs"]), - "coverage": round(summary["coverage"], 3), - }) - return pd.DataFrame(rows) - - -def _build_eu_jobs(dc_df: pd.DataFrame, n: int, seed: int = 42) -> pd.DataFrame: - """Build synthetic EU jobs matching EU city names.""" - rng = np.random.default_rng(seed) - cities = dc_df["city"].tolist() - return pd.DataFrame({ - "job_id": [f"eu_job_{i}" for i in range(n)], - "power_demand": rng.choice([10.0, 20.0, 50.0, 100.0], n), - "duration_hours": rng.integers(2, 9, n).tolist(), - "earliest_start": "2023-06-01 00:00:00", - "deadline": "2023-06-30 00:00:00", - "origin_city": rng.choice(cities, n).tolist(), - "priority": 1, - }) - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Forecast robustness experiment for carbon-aware scheduler" - ) - parser.add_argument( - "--region", - choices=["US", "EU"], - default="US", - help="Dataset region (default: US)", - ) - parser.add_argument( - "--jobs", - type=int, - default=100, - help="Number of jobs to schedule (default: 100)", - ) - parser.add_argument( - "--alpha", type=float, default=0.7, help="Carbon weight (default: 0.7)" - ) - parser.add_argument( - "--beta", type=float, default=0.2, help="Water weight (default: 0.2)" - ) - parser.add_argument( - "--output", - default=None, - help="Save results to CSV path (optional)", - ) - args = parser.parse_args() - - print(f"Forecast Robustness Experiment") - print(f" Region: {args.region}") - print(f" Jobs: {args.jobs}") - print(f" Weights: α={args.alpha}, β={args.beta}, γ={1 - args.alpha - args.beta:.2f}") - print() - - if args.region == "EU": - if not EU_DATA_PATH.exists(): - print(f"[ERROR] EU dataset not found: {EU_DATA_PATH}") - print(" Run: python -m src.generate_eu_synthetic_data") - print(" Then: python -m src.build_eu_dataset") - return - data_df = pd.read_csv(EU_DATA_PATH) - dc_df = pd.read_csv(EU_DC_PATH) - jobs_df = _build_eu_jobs(dc_df, args.jobs) - # Filter to a representative month - data_df = data_df[ - (data_df["TIMESTAMP"] >= "2023-06-01") & - (data_df["TIMESTAMP"] < "2023-07-01") - ].copy() - else: - data_df = load_dataset() - dc_df = load_dc_config() - all_jobs = load_jobs_dataset() - jobs_df = all_jobs.head(args.jobs).copy() - # Shift job timestamps to align with dataset window - data_start = pd.Timestamp(data_df["TIMESTAMP"].min()) - data_end = pd.Timestamp(data_df["TIMESTAMP"].max()) - jobs_df["earliest_start"] = data_start - jobs_df["deadline"] = data_end - data_df = data_df[ - (data_df["TIMESTAMP"] >= "2023-01-01") & - (data_df["TIMESTAMP"] < "2023-07-01") - ].copy() - - print(f" Dataset rows: {len(data_df):,}") - print(f" DCs: {len(dc_df)}") - print(f" Jobs: {len(jobs_df)}") - print() - - results = run_forecast_robustness( - data_df=data_df, - dc_df=dc_df, - jobs_df=jobs_df, - alpha=args.alpha, - beta=args.beta, - gamma=round(1 - args.alpha - args.beta, 4), - ) - - # Print table - print("=" * 75) - print(f"{'Noise':>6} {'RMSE':>10} {'Oracle CO2':>12} {'Actual CO2':>12} {'Regret':>10} {'Regret%':>9} {'Coverage':>9}") - print("-" * 75) - oracle_co2 = results.loc[results["noise_level"] == 0.0, "oracle_co2_kg"].iloc[0] - for _, row in results.iterrows(): - print( - f"{row['noise_pct']:>6} " - f"{row['forecast_rmse']:>9.5f} " - f"{row['oracle_co2_kg']:>11.2f} " - f"{row['actual_co2_kg']:>11.2f} " - f"{row['regret_kg']:>9.2f} " - f"{row['regret_pct']:>8.1f}% " - f"{row['coverage']:>8.3f}" - ) - print("=" * 75) - - # Robustness verdict - p50_regret = results.loc[results["noise_level"] == 0.50, "regret_pct"].iloc[0] - p20_regret = results.loc[results["noise_level"] == 0.20, "regret_pct"].iloc[0] - print(f"\nKey findings:") - print(f" At 20% forecast error: +{p20_regret:.1f}% CO2 regret") - print(f" At 50% forecast error: +{p50_regret:.1f}% CO2 regret") - if p20_regret < 10.0: - print(f" → Scheduler is ROBUST to typical day-ahead forecast errors (<10% regret at 20% noise)") - else: - print(f" → Scheduler shows meaningful sensitivity to forecast error") - - if args.output: - out_path = Path(args.output) - out_path.parent.mkdir(parents=True, exist_ok=True) - results.to_csv(out_path, index=False) - print(f"\nResults saved → {out_path}") - - -if __name__ == "__main__": - main() diff --git a/src/generate_eu_synthetic_data.py b/src/generate_eu_synthetic_data.py deleted file mode 100644 index 3642741..0000000 --- a/src/generate_eu_synthetic_data.py +++ /dev/null @@ -1,234 +0,0 @@ -""" -generate_eu_synthetic_data.py ------------------------------ -Generate realistic synthetic EU hourly generation data from published 2023 statistics. - -Sources (all public): - - ENTSO-E Annual Report 2023 (hourly generation mix by country) - - Ember Climate hourly data (average profiles) - - EU Code of Conduct 2022 (water efficiency benchmarks) - -This allows full end-to-end testing without requiring ENTSO-E API token. -Suitable for research (methodology is transparent and reproducible). - -Usage: - python -m src.generate_eu_synthetic_data --output data/processed/entso_e_generation_mix.csv -""" - -from __future__ import annotations - -import argparse -from pathlib import Path - -import numpy as np -import pandas as pd - -try: - from .data_loader import PROJECT_ROOT -except ImportError: - from data_loader import PROJECT_ROOT - -OUTPUT_PATH = PROJECT_ROOT / "data" / "processed" / "entso_e_generation_mix.csv" - -# Published 2023 annual generation mix by EU zone (%) -# Source: ENTSO-E Statistical Yearbook 2023, Ember Climate 2024 -ZONE_ANNUAL_MIX_2023 = { - "IE": { # Ireland — 38% wind is real, high renewable - "wind": 0.38, - "gas": 0.35, - "coal": 0.05, - "hydro": 0.12, - "biomass": 0.10, - }, - "NL": { # Netherlands — gas heavy, growing wind/solar - "gas": 0.45, - "wind": 0.30, - "coal": 0.10, - "solar": 0.08, - "nuclear": 0.03, - "biomass": 0.04, - }, - "DE-LU": { # Germany-Luxembourg — coal + renewables mix - "wind": 0.32, - "coal": 0.24, - "gas": 0.18, - "nuclear": 0.11, - "solar": 0.10, - "hydro": 0.03, - "biomass": 0.02, - }, - "GB": { # Great Britain — gas + nuclear + wind - "gas": 0.38, - "nuclear": 0.17, - "wind": 0.25, - "coal": 0.02, - "solar": 0.03, - "hydro": 0.04, - "biomass": 0.11, - }, - "FR": { # France — 65% nuclear is correct - "nuclear": 0.65, - "hydro": 0.15, - "wind": 0.10, - "solar": 0.05, - "biomass": 0.03, - "gas": 0.02, - }, - "SE3": { # Sweden SE3 (Stockholm) — hydro + nuclear dominant - "hydro": 0.55, - "nuclear": 0.28, - "wind": 0.12, - "biomass": 0.03, - "solar": 0.02, - }, - "FI": { # Finland — nuclear + hydro + biomass - "nuclear": 0.35, - "hydro": 0.25, - "biomass": 0.20, - "wind": 0.12, - "coal": 0.05, - "gas": 0.03, - }, - "ES": { # Spain — balanced mix - "wind": 0.28, - "solar": 0.15, - "nuclear": 0.21, - "gas": 0.22, - "hydro": 0.10, - "coal": 0.02, - "biomass": 0.02, - }, - "PL": { # Poland — coal heavy - "coal": 0.65, - "gas": 0.10, - "wind": 0.15, - "biomass": 0.05, - "nuclear": 0.03, - "hydro": 0.02, - }, - "DK2": { # Denmark East (Copenhagen) — wind dominant - "wind": 0.55, - "gas": 0.22, - "biomass": 0.15, - "solar": 0.05, - "hydro": 0.03, - }, -} - -# Intra-daily variation profiles (wind is higher at night, solar at day) -# Normalized to 1.0 at average hour -HOURLY_PROFILE = { - "wind": np.array([1.3, 1.4, 1.35, 1.2, 0.9, 0.7, 0.6, 0.6, 0.7, 0.8, 0.85, 0.9, - 0.95, 0.95, 0.9, 0.85, 0.8, 0.85, 0.95, 1.1, 1.2, 1.35, 1.4, 1.3]), - "solar": np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.1, 0.4, 0.8, 1.3, 1.5, 1.6, - 1.5, 1.3, 0.9, 0.4, 0.1, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]), - "hydro": np.ones(24) * 1.0, # Roughly constant (storage) - "nuclear": np.ones(24) * 1.0, # Baseload - "coal": np.ones(24) * 1.0, # Baseload - "gas": np.ones(24) * 1.0, # Flexible, assume constant for this sim - "biomass": np.ones(24) * 1.0, # Assume constant -} - -# Seasonal variation (approximate) -def seasonal_factor(month: int, fuel: str) -> float: - """Seasonal adjustment for fuel generation.""" - if fuel == "solar": - # Solar peaks in June (summer), low in December - return 0.5 + 0.5 * np.cos((month - 6) * np.pi / 6) - elif fuel == "wind": - # Wind stronger in winter - return 1.0 + 0.3 * np.cos((month - 1) * np.pi / 6) - elif fuel == "hydro": - # Hydro varies seasonally (snow melt in spring) - return 0.9 + 0.4 * np.sin((month - 3) * np.pi / 6) - else: - return 1.0 # No seasonal adjustment - - -def generate_eu_data( - start_date: str = "2023-01-01", - end_date: str = "2023-12-31", - total_gen_mw: float = 500.0, -) -> pd.DataFrame: - """ - Generate synthetic hourly EU generation data from published annual statistics. - - Args: - start_date: Start date (YYYY-MM-DD) - end_date: End date (YYYY-MM-DD) - total_gen_mw: Average MW per zone per hour - - Returns: - DataFrame with columns: timestamp, zone, fuel_type, generation_mwh - """ - np.random.seed(2023) # Reproducible randomness - - dates = pd.date_range(start_date, end_date, freq="h") - rows = [] - - for zone, fuel_mix in ZONE_ANNUAL_MIX_2023.items(): - for ts in dates: - hour = ts.hour - month = ts.month - - for fuel, annual_share in fuel_mix.items(): - # Intra-daily profile - hourly_factor = HOURLY_PROFILE.get(fuel, np.ones(24))[hour] - - # Seasonal adjustment - seasonal = seasonal_factor(month, fuel) - - # Generation = total × share × hourly_factor × seasonal ± noise - base_gen = total_gen_mw * annual_share * hourly_factor * seasonal - noise = np.random.normal(0, base_gen * 0.05) # ±5% noise - generation_mwh = max(0, base_gen + noise) - - rows.append({ - "timestamp": ts, - "zone": zone, - "fuel_type": fuel, - "generation_mwh": generation_mwh, - }) - - return pd.DataFrame(rows) - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Generate realistic synthetic EU generation data from 2023 statistics" - ) - parser.add_argument("--start", default="2023-01-01", help="Start date YYYY-MM-DD") - parser.add_argument("--end", default="2023-12-31", help="End date YYYY-MM-DD") - parser.add_argument("--output", default=str(OUTPUT_PATH), help="Output CSV path") - parser.add_argument( - "--mw", - default=500.0, - type=float, - help="Average MW per zone per hour (default 500)", - ) - args = parser.parse_args() - - print("Generating realistic synthetic EU generation data...") - print(f" Date range: {args.start} → {args.end}") - print(f" Source: ENTSO-E 2023 Annual Report + Ember Climate") - df = generate_eu_data(args.start, args.end, args.mw) - - output_path = Path(args.output) - output_path.parent.mkdir(parents=True, exist_ok=True) - df.to_csv(output_path, index=False) - - print(f"\nGenerated {len(df):,} rows ({len(df) // 24} days × 10 zones)") - print(f"Saved → {output_path}") - - # Show summary statistics - summary = ( - df.groupby("zone")["generation_mwh"] - .agg(["count", "mean", "std"]) - .round(2) - ) - print("\nGeneration summary (MWh) by zone:") - print(summary) - - -if __name__ == "__main__": - main() diff --git a/src/main.py b/src/main.py deleted file mode 100644 index 8295f96..0000000 --- a/src/main.py +++ /dev/null @@ -1,136 +0,0 @@ -from __future__ import annotations - -import argparse - -if __package__: - from .train import TARGET_SPECS, train_forecasting_models -else: - from train import TARGET_SPECS, train_forecasting_models - - -def run_pipeline( - data_path: str | None = None, - weather_path: str | None = None, - history_days: int = 365, - test_horizon_hours: int = 48, - forecast_horizon_hours: int = 48, - random_state: int = 42, - save_models: bool = True, - city_limit: int | None = None, -): - """Execute the full forecasting pipeline from loading through future predictions.""" - return train_forecasting_models( - data_path=data_path, - weather_path=weather_path, - history_days=history_days, - test_horizon_hours=test_horizon_hours, - forecast_horizon_hours=forecast_horizon_hours, - random_state=random_state, - save_models=save_models, - city_limit=city_limit, - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Run the hybrid forecasting pipeline for WUE and CO2 intensity." - ) - parser.add_argument( - "--data-path", - default=None, - help="Path to the master dataset (.zip, .parquet, .csv, or .xlsx).", - ) - parser.add_argument( - "--weather-path", - default=None, - help="Optional path to external weather data for feature enrichment.", - ) - parser.add_argument( - "--history-days", - type=int, - default=365, - help="Number of trailing historical days to use for training.", - ) - parser.add_argument( - "--test-horizon-hours", - type=int, - default=48, - help="Number of holdout hours used for time-based evaluation.", - ) - parser.add_argument( - "--forecast-horizon-hours", - type=int, - default=48, - help="Number of future hours to predict for each city.", - ) - parser.add_argument( - "--random-state", - type=int, - default=42, - help="Random seed for reproducible XGBoost training.", - ) - parser.add_argument( - "--city-limit", - type=int, - default=None, - help="Optional number of cities to train for faster smoke tests.", - ) - parser.add_argument( - "--no-save-models", - action="store_true", - help="Disable saving model bundles and Prophet artifacts.", - ) - return parser.parse_args() - - -def main() -> None: - args = parse_args() - results = run_pipeline( - data_path=args.data_path, - weather_path=args.weather_path, - history_days=args.history_days, - test_horizon_hours=args.test_horizon_hours, - forecast_horizon_hours=args.forecast_horizon_hours, - random_state=args.random_state, - save_models=not args.no_save_models, - city_limit=args.city_limit, - ) - - if results["weather"] is not None: - print("Weather enrichment") - print(f" Join strategy: {results['weather']['join_strategy']}") - print( - " Matched rows: " - f"{results['weather']['matched_rows']}/{results['weather']['total_rows']}" - ) - print(f" Coverage: {results['weather']['coverage']:.2%}") - print("") - - print("Forecasting split") - print(f" Train start: {results['split']['train_start']}") - print(f" Train end: {results['split']['train_end']}") - print(f" Test start: {results['split']['test_start']}") - print(f" Test end: {results['split']['test_end']}") - - for target_key, target_result in results["targets"].items(): - print(f"\n{TARGET_SPECS[target_key]['label']} results") - for model_name, metrics in target_result["metrics"].items(): - print( - f" {model_name:>7} -> RMSE {metrics['rmse']:.4f}, " - f"MAE {metrics['mae']:.4f}, R2 {metrics['r2']:.4f}" - ) - print( - " Hybrid vs Prophet RMSE improvement: " - f"{target_result['comparisons']['hybrid_vs_prophet']['rmse_improvement_pct']:.2f}%" - ) - if not args.no_save_models: - print(f" Saved model bundle: {target_result['saved_model_path']}") - print(f" Saved Prophet models: {target_result['saved_prophet_dir']}") - - print(f"\nSaved forecasts to: {results['forecast_path']}") - print("Forecast preview") - print(results["forecast_preview"].to_string(index=False)) - - -if __name__ == "__main__": - main() diff --git a/src/preprocessing.py b/src/preprocessing.py deleted file mode 100644 index 1103e02..0000000 --- a/src/preprocessing.py +++ /dev/null @@ -1,341 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -import numpy as np -import pandas as pd - - -FUEL_COLUMNS = [ - "COAL", - "HYDRO", - "NATURALGAS", - "NUCLEAR", - "OTHER", - "PETROLEUM", - "SOLAR", - "WIND", -] -WEATHER_CANDIDATES = { - "temperature_c": ["temp_c", "weather_temp", "temp_c_from_water", "TEMPERATUREF"], - "humidity_pct": ["rh_pct", "weather_rhum", "rh_pct_from_water", "HUMIDITY"], - "wind_speed_ms": ["wspd_ms", "weather_wspd"], -} -BASE_FEATURE_COLUMNS = [ - "temperature_c", - "humidity_pct", - "wind_speed_ms", - "scarcity_index", - "hour", - "day_of_week", - "month", - "day_of_year", - "is_weekend", - "hour_sin", - "hour_cos", - "day_of_week_sin", - "day_of_week_cos", - "month_sin", - "month_cos", -] -TARGET_COLUMNS = ["WUE_total", "carbon_intensity_kg_per_kwh"] - - -@dataclass(frozen=True) -class TimeSplit: - train: pd.DataFrame - test: pd.DataFrame - test_start: pd.Timestamp - test_end: pd.Timestamp - - -@dataclass(frozen=True) -class FutureFeatureProfiles: - hourly_profiles: pd.DataFrame - city_profiles: pd.DataFrame - global_profile: dict[str, float] - profile_columns: list[str] - profile_history_days: int - - -def _add_time_features(frame: pd.DataFrame) -> pd.DataFrame: - enriched = frame.copy() - enriched["hour"] = enriched["timestamp"].dt.hour - enriched["day_of_week"] = enriched["timestamp"].dt.dayofweek - enriched["month"] = enriched["timestamp"].dt.month - enriched["day_of_year"] = enriched["timestamp"].dt.dayofyear - enriched["is_weekend"] = (enriched["day_of_week"] >= 5).astype(int) - enriched["hour_sin"] = np.sin(2 * np.pi * enriched["hour"] / 24.0) - enriched["hour_cos"] = np.cos(2 * np.pi * enriched["hour"] / 24.0) - enriched["day_of_week_sin"] = np.sin(2 * np.pi * enriched["day_of_week"] / 7.0) - enriched["day_of_week_cos"] = np.cos(2 * np.pi * enriched["day_of_week"] / 7.0) - enriched["month_sin"] = np.sin(2 * np.pi * enriched["month"] / 12.0) - enriched["month_cos"] = np.cos(2 * np.pi * enriched["month"] / 12.0) - return enriched - - -def _safe_ratio(numerator: pd.Series, denominator: pd.Series) -> pd.Series: - denominator = denominator.replace(0, np.nan) - return numerator / denominator - - -def resolve_timestamp_column(df: pd.DataFrame) -> str: - for candidate in ["TIMESTAMP", "Timestamp", "timestamp"]: - if candidate in df.columns: - return candidate - raise ValueError("Dataset is missing a timestamp column.") - - -def resolve_city_column(df: pd.DataFrame) -> str: - for candidate in ["CITY", "city"]: - if candidate in df.columns: - return candidate - raise ValueError("Dataset is missing a city column.") - - -def _coerce_numeric_columns(df: pd.DataFrame, columns: list[str]) -> None: - for column in columns: - if column in df.columns: - df[column] = pd.to_numeric(df[column], errors="coerce") - - -def _extract_weather_feature(df: pd.DataFrame, candidates: list[str], convert_fahrenheit: bool = False) -> pd.Series: - for candidate in candidates: - if candidate in df.columns: - series = pd.to_numeric(df[candidate], errors="coerce") - if convert_fahrenheit: - return (series - 32.0) * (5.0 / 9.0) - return series - return pd.Series(np.nan, index=df.index, dtype=float) - - -def _generation_total_mwh(df: pd.DataFrame) -> pd.Series: - if "total_gen_mwh" in df.columns: - return pd.to_numeric(df["total_gen_mwh"], errors="coerce") - if "total_gen_kwh" in df.columns: - return pd.to_numeric(df["total_gen_kwh"], errors="coerce") / 1000.0 - - available_fuels = [column for column in FUEL_COLUMNS if column in df.columns] - if available_fuels: - return df[available_fuels].apply(pd.to_numeric, errors="coerce").sum(axis=1) - return pd.Series(np.nan, index=df.index, dtype=float) - - -def prepare_forecasting_frame( - df: pd.DataFrame, - city_limit: int | None = None, -) -> pd.DataFrame: - """Build the canonical city-hour forecasting frame used by XGBoost and Prophet.""" - data = df.copy() - timestamp_col = resolve_timestamp_column(data) - city_col = resolve_city_column(data) - - data[timestamp_col] = pd.to_datetime(data[timestamp_col], errors="coerce").dt.floor("h") - data[city_col] = data[city_col].astype(str).str.strip() - - numeric_candidates = [ - "WUE_total", - "co2_kg", - "total_gen_kwh", - "total_gen_mwh", - "dsci_0_500", - *FUEL_COLUMNS, - *sum(WEATHER_CANDIDATES.values(), []), - ] - _coerce_numeric_columns(data, numeric_candidates) - - required_targets = {"WUE_total", "co2_kg", "total_gen_kwh"} - missing_targets = sorted(required_targets - set(data.columns)) - if missing_targets: - raise ValueError( - "Dataset is missing required target columns: " + ", ".join(missing_targets) - ) - - frame = pd.DataFrame( - { - "timestamp": data[timestamp_col], - "city": data[city_col], - "WUE_total": data["WUE_total"], - "carbon_intensity_kg_per_kwh": _safe_ratio(data["co2_kg"], data["total_gen_kwh"]), - } - ) - - frame["temperature_c"] = _extract_weather_feature( - data, - WEATHER_CANDIDATES["temperature_c"], - convert_fahrenheit="TEMPERATUREF" in data.columns and "temp_c" not in data.columns, - ) - frame["humidity_pct"] = _extract_weather_feature(data, WEATHER_CANDIDATES["humidity_pct"]) - frame["wind_speed_ms"] = _extract_weather_feature(data, WEATHER_CANDIDATES["wind_speed_ms"]) - - scarcity = pd.to_numeric(data.get("dsci_0_500"), errors="coerce") if "dsci_0_500" in data.columns else pd.Series(np.nan, index=data.index) - frame["scarcity_index"] = scarcity / 500.0 - - generation_total = _generation_total_mwh(data) - for fuel_column in FUEL_COLUMNS: - if fuel_column in data.columns: - frame[f"{fuel_column.lower()}_share"] = _safe_ratio(data[fuel_column], generation_total) - else: - frame[f"{fuel_column.lower()}_share"] = np.nan - - frame = _add_time_features(frame) - - frame = frame.replace([np.inf, -np.inf], np.nan) - frame = frame.dropna(subset=["timestamp", "city", *TARGET_COLUMNS]).sort_values(["timestamp", "city"]) - - if city_limit is not None: - selected_cities = sorted(frame["city"].dropna().unique().tolist())[: int(city_limit)] - frame = frame[frame["city"].isin(selected_cities)].copy() - - return frame.reset_index(drop=True) - - -def feature_columns_for_frame(frame: pd.DataFrame) -> list[str]: - fuel_share_columns = [f"{column.lower()}_share" for column in FUEL_COLUMNS] - ordered = BASE_FEATURE_COLUMNS + fuel_share_columns - return [column for column in ordered if column in frame.columns] - - -def build_feature_matrix( - frame: pd.DataFrame, - feature_columns: list[str] | None = None, - encoded_columns: list[str] | None = None, -) -> tuple[pd.DataFrame, list[str]]: - """Create an aligned feature matrix with one-hot encoded cities.""" - if feature_columns is None: - feature_columns = feature_columns_for_frame(frame) - - base = frame[["city", *feature_columns]].copy() - encoded = pd.get_dummies(base, columns=["city"], prefix="city", dtype=float) - - if encoded_columns is None: - encoded_columns = encoded.columns.tolist() - else: - missing = [column for column in encoded_columns if column not in encoded.columns] - for column in missing: - encoded[column] = 0.0 - extra = [column for column in encoded.columns if column not in encoded_columns] - if extra: - encoded = encoded.drop(columns=extra) - encoded = encoded[encoded_columns] - - return encoded.astype(float), encoded_columns - - -def split_time_train_test( - frame: pd.DataFrame, - test_horizon_hours: int = 48, - history_days: int = 365, -) -> TimeSplit: - """Hold out the final horizon using a strict time-based split.""" - timestamps = pd.Index(sorted(frame["timestamp"].dropna().unique())) - if len(timestamps) <= test_horizon_hours: - raise ValueError("Not enough hourly observations for the requested test horizon.") - - test_timestamps = timestamps[-int(test_horizon_hours):] - test_start = pd.Timestamp(test_timestamps[0]) - test_end = pd.Timestamp(test_timestamps[-1]) - history_start = test_start - pd.Timedelta(days=int(history_days)) - - scoped = frame[(frame["timestamp"] >= history_start) & (frame["timestamp"] <= test_end)].copy() - train = scoped[scoped["timestamp"] < test_start].copy() - test = scoped[scoped["timestamp"] >= test_start].copy() - - if train.empty or test.empty: - raise ValueError("Time split produced an empty train or test partition.") - - return TimeSplit(train=train, test=test, test_start=test_start, test_end=test_end) - - -def select_recent_history( - frame: pd.DataFrame, - history_days: int = 365, - end_time: pd.Timestamp | None = None, -) -> pd.DataFrame: - if end_time is None: - end_time = pd.Timestamp(frame["timestamp"].max()) - history_start = pd.Timestamp(end_time) - pd.Timedelta(days=int(history_days)) - scoped = frame[(frame["timestamp"] >= history_start) & (frame["timestamp"] <= pd.Timestamp(end_time))].copy() - if scoped.empty: - raise ValueError("No history rows are available for the requested training window.") - return scoped.sort_values(["timestamp", "city"]).reset_index(drop=True) - - -def build_future_feature_profiles( - history_frame: pd.DataFrame, - profile_history_days: int = 30, -) -> FutureFeatureProfiles: - profile_columns = feature_columns_for_frame(history_frame) - recent_start = pd.Timestamp(history_frame["timestamp"].max()) - pd.Timedelta(days=int(profile_history_days)) - recent = history_frame[history_frame["timestamp"] >= recent_start].copy() - if recent.empty: - recent = history_frame.copy() - - hourly_profiles = ( - recent.groupby(["city", "hour"], as_index=False)[profile_columns].median(numeric_only=True) - ) - city_profiles = recent.groupby("city", as_index=False)[profile_columns].median(numeric_only=True) - global_profile = recent[profile_columns].median(numeric_only=True).to_dict() - return FutureFeatureProfiles( - hourly_profiles=hourly_profiles, - city_profiles=city_profiles, - global_profile={key: float(value) for key, value in global_profile.items()}, - profile_columns=profile_columns, - profile_history_days=int(profile_history_days), - ) - - -def build_profile_feature_frame( - time_city_frame: pd.DataFrame, - profiles: FutureFeatureProfiles, -) -> pd.DataFrame: - """Populate exogenous features from historical profiles without using future observations.""" - if not {"timestamp", "city"}.issubset(time_city_frame.columns): - raise ValueError("The profile feature frame requires `timestamp` and `city` columns.") - - future = time_city_frame[["timestamp", "city"]].copy() - future["timestamp"] = pd.to_datetime(future["timestamp"], errors="coerce").dt.floor("h") - future["city"] = future["city"].astype(str).str.strip() - future = _add_time_features(future) - - future = future.merge(profiles.hourly_profiles, on=["city", "hour"], how="left") - city_profiles = profiles.city_profiles.set_index("city") - for column in profiles.profile_columns: - if column not in future.columns: - future[column] = np.nan - future[column] = future[column].fillna(future["city"].map(city_profiles[column])) - future[column] = future[column].fillna(profiles.global_profile.get(column, np.nan)) - - return future.sort_values(["timestamp", "city"]).reset_index(drop=True) - - -def build_future_feature_frame( - history_frame: pd.DataFrame, - horizon_hours: int = 48, - profiles: FutureFeatureProfiles | None = None, -) -> pd.DataFrame: - """Project the next horizon of city-hour features from historical hourly profiles.""" - if profiles is None: - profiles = build_future_feature_profiles(history_frame) - - last_timestamp = pd.Timestamp(history_frame["timestamp"].max()).floor("h") - future_timestamps = pd.date_range( - start=last_timestamp + pd.Timedelta(hours=1), - periods=int(horizon_hours), - freq="h", - ) - cities = sorted(history_frame["city"].dropna().unique().tolist()) - time_city_frame = pd.MultiIndex.from_product( - [future_timestamps, cities], - names=["timestamp", "city"], - ).to_frame(index=False) - return build_profile_feature_frame(time_city_frame, profiles=profiles) - - -def build_modeling_frame( - df: pd.DataFrame, -) -> tuple[pd.DataFrame, pd.Series, pd.Series]: - """Backward-compatible helper for legacy regression code paths.""" - frame = prepare_forecasting_frame(df) - X, _ = build_feature_matrix(frame) - return X, frame["WUE_total"], frame["carbon_intensity_kg_per_kwh"] diff --git a/src/runtime.py b/src/runtime.py deleted file mode 100644 index 390c969..0000000 --- a/src/runtime.py +++ /dev/null @@ -1,51 +0,0 @@ -from __future__ import annotations - -import ctypes -import os -import platform -from pathlib import Path - - -def _candidate_openmp_paths() -> list[Path]: - candidates = [ - os.environ.get("LIBOMP_PATH"), - "/opt/homebrew/opt/libomp/lib/libomp.dylib", - "/usr/local/opt/libomp/lib/libomp.dylib", - "/Library/Frameworks/R.framework/Resources/lib/libomp.dylib", - ] - - r_framework = Path("/Library/Frameworks/R.framework/Versions") - if r_framework.exists(): - candidates.extend( - str(path) - for path in sorted(r_framework.glob("*/Resources/lib/libomp.dylib"), reverse=True) - ) - - return [Path(path) for path in candidates if path] - - -def ensure_openmp_runtime() -> str | None: - """Load libomp on macOS before importing XGBoost when Homebrew is unavailable.""" - if platform.system() != "Darwin": - return None - - for candidate in _candidate_openmp_paths(): - if not candidate.exists(): - continue - try: - existing_library_path = os.environ.get("DYLD_LIBRARY_PATH", "") - existing_fallback_path = os.environ.get("DYLD_FALLBACK_LIBRARY_PATH", "") - library_dir = str(candidate.parent) - if library_dir not in existing_library_path.split(":"): - os.environ["DYLD_LIBRARY_PATH"] = ":".join( - [part for part in [library_dir, existing_library_path] if part] - ) - if library_dir not in existing_fallback_path.split(":"): - os.environ["DYLD_FALLBACK_LIBRARY_PATH"] = ":".join( - [part for part in [library_dir, existing_fallback_path] if part] - ) - ctypes.CDLL(str(candidate), mode=ctypes.RTLD_GLOBAL) - return str(candidate) - except OSError: - continue - return None diff --git a/src/scheduling.py b/src/scheduling.py index 3d8e56e..3354bb8 100644 --- a/src/scheduling.py +++ b/src/scheduling.py @@ -1,175 +1,219 @@ +"""Core scheduler for reproducible carbon and water aware experiments. + +All timestamps are interpreted as UTC. Capacity is expressed as IT power in kW. +Carbon intensity is kg CO2e per facility kWh. Water Usage Effectiveness is +liters per IT kWh, so PUE is applied to carbon but not to onsite water use. +""" + from __future__ import annotations -import argparse -from dataclasses import dataclass +from collections.abc import Callable +from dataclasses import dataclass, replace from pathlib import Path -from typing import Any import numpy as np import pandas as pd -try: - from .data_loader import PROJECT_ROOT, load_dataset -except ImportError: - from data_loader import PROJECT_ROOT, load_dataset - - -DEFAULT_JOBS_PATH = PROJECT_ROOT / "data" / "processed" / "azure_jobs_sample.csv" -DEFAULT_DC_CONFIG_PATH = PROJECT_ROOT / "data" / "templates" / "dc_config_template.csv" - -REQUIRED_JOB_COLUMNS = [ +REQUIRED_JOB_COLUMNS = { "job_id", "origin_city", - "power_demand", + "power_demand_kw", "duration_hours", "earliest_start", "deadline", -] -REQUIRED_DC_COLUMNS = [ +} +REQUIRED_DC_COLUMNS = { "dc_id", "city", - "max_power_per_hour", + "max_power_kw", "pue", "water_usage_factor", - "region_water_scarcity", - "is_new", -] + "water_scarcity_index", +} -@dataclass +@dataclass(frozen=True) class ScheduleCandidate: + """One feasible placement for one non-preemptive job.""" + dc_id: str city: str start_time: pd.Timestamp end_time: pd.Timestamp - score: float - expected_co2_kg: float - expected_water_liters: float start_index: int end_index: int + expected_co2_kg: float + expected_water_liters: float + scarcity_weighted_water_liters: float + relocation_penalty: float + score: float = np.nan -@dataclass +@dataclass(frozen=True) class PreparedCityMetrics: - co2_prefix: np.ndarray + carbon_prefix: np.ndarray water_prefix: np.ndarray valid_prefix: np.ndarray -@dataclass +@dataclass(frozen=True) class PreparedSchedulerState: time_index: pd.DatetimeIndex time_values: np.ndarray city_metrics: dict[str, PreparedCityMetrics] - dc_records: list[dict[str, Any]] + dc_records: tuple[dict[str, float | str], ...] + + +CandidateSelector = Callable[[pd.Series, list[ScheduleCandidate]], ScheduleCandidate] -@dataclass -class RouteFlow: - route: str - origin_city: str - assigned_city: str - jobs: int - total_co2_kg: float - total_water_liters: float - avg_scheduler_score: float - avg_latency_ms: float +def _utc_naive_timestamp(value: object) -> pd.Timestamp: + timestamp = pd.to_datetime(value, utc=True) + if pd.isna(timestamp): + raise ValueError(f"Invalid timestamp: {value!r}") + return pd.Timestamp(timestamp).tz_convert(None) + + +def _utc_naive_series(series: pd.Series) -> pd.Series: + return pd.to_datetime(series, errors="coerce", utc=True).dt.tz_convert(None) + + +def _require_columns(frame: pd.DataFrame, required: set[str], label: str) -> None: + missing = sorted(required - set(frame.columns)) + if missing: + raise ValueError(f"{label} is missing required columns: {', '.join(missing)}") def build_job_record( origin_city: str, - power_demand: float, + power_demand_kw: float, duration_hours: int, earliest_start: str | pd.Timestamp, deadline: str | pd.Timestamp, - priority: int = 0, - job_id: str = "simulated_job", + priority: int = 1, + job_id: str = "job", ) -> pd.Series: - """Construct a scheduler-compatible job record for interactive simulations.""" - priority_value = int(priority) - priority_label = {0: "high", 1: "medium", 2: "low"}.get(priority_value, "low") - return pd.Series( + """Build one validated job record.""" + record = pd.Series( { - "job_id": job_id, - "origin_city": origin_city, - "power_demand": float(power_demand), + "job_id": str(job_id), + "origin_city": str(origin_city), + "power_demand_kw": float(power_demand_kw), "duration_hours": int(duration_hours), - "earliest_start": pd.Timestamp(earliest_start), - "deadline": pd.Timestamp(deadline), - "priority": priority_value, - "priority_label": priority_label, + "earliest_start": _utc_naive_timestamp(earliest_start), + "deadline": _utc_naive_timestamp(deadline), + "priority": int(priority), } ) - - -def _resolve_path(path: str | Path) -> Path: - candidate = Path(path) - if candidate.is_absolute(): - return candidate - return PROJECT_ROOT / candidate - - -def load_jobs_dataset(path: str | Path = DEFAULT_JOBS_PATH) -> pd.DataFrame: - jobs_path = _resolve_path(path) - jobs_df = pd.read_csv( - jobs_path, parse_dates=["earliest_start", "deadline"], low_memory=False + if record["power_demand_kw"] <= 0 or record["duration_hours"] <= 0: + raise ValueError("Power demand and duration must be positive.") + if record["deadline"] <= record["earliest_start"]: + raise ValueError("Deadline must be later than earliest_start.") + return record + + +def load_jobs_dataset(path: str | Path) -> pd.DataFrame: + """Load a canonical jobs CSV.""" + jobs = pd.read_csv(path) + if "power_demand" in jobs.columns and "power_demand_kw" not in jobs.columns: + jobs = jobs.rename(columns={"power_demand": "power_demand_kw"}) + _require_columns(jobs, REQUIRED_JOB_COLUMNS, "Jobs dataset") + jobs["earliest_start"] = _utc_naive_series(jobs["earliest_start"]) + jobs["deadline"] = _utc_naive_series(jobs["deadline"]) + return jobs + + +def load_dc_config(path: str | Path) -> pd.DataFrame: + """Load a data center configuration and normalize legacy column names.""" + config = pd.read_csv(path) + config = config.rename( + columns={ + "max_power_per_hour": "max_power_kw", + "region_water_scarcity": "water_scarcity_index", + } ) - missing = [col for col in REQUIRED_JOB_COLUMNS if col not in jobs_df.columns] - if missing: - raise ValueError(f"Jobs dataset is missing required columns: {', '.join(missing)}") - return jobs_df - - -def load_dc_config(path: str | Path = DEFAULT_DC_CONFIG_PATH) -> pd.DataFrame: - config_path = _resolve_path(path) - dc_df = pd.read_csv(config_path, low_memory=False) - missing = [col for col in REQUIRED_DC_COLUMNS if col not in dc_df.columns] - if missing: - raise ValueError(f"DC config is missing required columns: {', '.join(missing)}") - return dc_df + _require_columns(config, REQUIRED_DC_COLUMNS, "Data center configuration") + return config def prepare_environment_frame( - source_df: pd.DataFrame, city_subset: list[str] | None = None + source_df: pd.DataFrame, + city_subset: list[str] | None = None, ) -> pd.DataFrame: - """Create the city-hour environmental intensity frame used by the scheduler.""" - df = source_df.copy() + """Normalize source data to one environmental row per city and UTC hour.""" + data = source_df.copy() + timestamp_col = next( + (column for column in ("timestamp", "TIMESTAMP", "Timestamp") if column in data), + None, + ) + city_col = next((column for column in ("city", "CITY") if column in data), None) + if timestamp_col is None or city_col is None: + raise ValueError("Environment data requires timestamp and city columns.") + + data["timestamp"] = _utc_naive_series(data[timestamp_col]).dt.floor("h") + data["city"] = data[city_col].astype(str).str.strip() + + if "carbon_intensity_kg_per_kwh" in data: + carbon = pd.to_numeric(data["carbon_intensity_kg_per_kwh"], errors="coerce") + elif {"co2_kg", "total_gen_kwh"}.issubset(data.columns): + numerator = pd.to_numeric(data["co2_kg"], errors="coerce") + denominator = pd.to_numeric(data["total_gen_kwh"], errors="coerce").replace(0, np.nan) + carbon = numerator / denominator + else: + raise ValueError( + "Environment data requires carbon_intensity_kg_per_kwh or " + "co2_kg plus total_gen_kwh." + ) - timestamp_col = "TIMESTAMP" if "TIMESTAMP" in df.columns else "Timestamp" - city_col = "CITY" if "CITY" in df.columns else "city" - df[timestamp_col] = pd.to_datetime(df[timestamp_col], errors="coerce").dt.floor("h") + if "water_intensity_l_per_kwh" in data: + water = pd.to_numeric(data["water_intensity_l_per_kwh"], errors="coerce") + elif "WUE_total" in data: + water = pd.to_numeric(data["WUE_total"], errors="coerce") + else: + raise ValueError( + "Environment data requires water_intensity_l_per_kwh or WUE_total." + ) - for column in ["co2_kg", "total_gen_kwh", "WUE_total", "dsci_0_500"]: - if column in df.columns: - df[column] = pd.to_numeric(df[column], errors="coerce") + if "scarcity_index" in data: + scarcity = pd.to_numeric(data["scarcity_index"], errors="coerce") + elif "dsci_0_500" in data: + scarcity = pd.to_numeric(data["dsci_0_500"], errors="coerce") / 500.0 + else: + scarcity = pd.Series(0.0, index=data.index) + normalized = pd.DataFrame( + { + "timestamp": data["timestamp"], + "city": data["city"], + "carbon_intensity_kg_per_kwh": carbon, + "water_intensity_l_per_kwh": water, + "scarcity_index": scarcity, + } + ) if city_subset is not None: - df = df[df[city_col].isin(city_subset)].copy() - - if "co2_kg" not in df.columns or "total_gen_kwh" not in df.columns: - raise ValueError("Dataset must contain co2_kg and total_gen_kwh for scheduling.") - if "WUE_total" not in df.columns: - raise ValueError("Dataset must contain WUE_total for scheduling.") - - df["predicted_co2_intensity"] = df["co2_kg"] / df["total_gen_kwh"].replace(0, np.nan) - df["predicted_water_intensity"] = df["WUE_total"] - if "dsci_0_500" in df.columns: - df["scarcity_index"] = df["dsci_0_500"] / 500.0 - else: - df["scarcity_index"] = 1.0 - - env_df = ( - df[[timestamp_col, city_col, "predicted_co2_intensity", "predicted_water_intensity", "scarcity_index"]] - .dropna(subset=[timestamp_col, city_col, "predicted_co2_intensity", "predicted_water_intensity"]) - .groupby([timestamp_col, city_col], as_index=False) - .agg( - predicted_co2_intensity=("predicted_co2_intensity", "mean"), - predicted_water_intensity=("predicted_water_intensity", "mean"), - scarcity_index=("scarcity_index", "mean"), - ) - .rename(columns={timestamp_col: "timestamp", city_col: "city"}) + normalized = normalized[normalized["city"].isin(city_subset)] + + normalized = normalized.dropna( + subset=[ + "timestamp", + "city", + "carbon_intensity_kg_per_kwh", + "water_intensity_l_per_kwh", + ] + ) + if (normalized["carbon_intensity_kg_per_kwh"] < 0).any(): + raise ValueError("Carbon intensity cannot be negative.") + if (normalized["water_intensity_l_per_kwh"] < 0).any(): + raise ValueError("Water intensity cannot be negative.") + if not normalized["scarcity_index"].dropna().between(0, 1).all(): + raise ValueError("scarcity_index must be between 0 and 1.") + + return ( + normalized.groupby(["timestamp", "city"], as_index=False) + .mean(numeric_only=True) + .sort_values(["timestamp", "city"]) + .reset_index(drop=True) ) - return env_df def _prefix_sum(values: np.ndarray) -> np.ndarray: @@ -180,56 +224,82 @@ def prepare_scheduler_state( env_df: pd.DataFrame, dc_df: pd.DataFrame, ) -> PreparedSchedulerState: - if env_df.empty: - raise ValueError("Environment frame is empty; cannot prepare scheduler state.") + """Validate inputs and precompute hourly prefix sums.""" + env = prepare_environment_frame(env_df) + config = dc_df.copy().rename( + columns={ + "max_power_per_hour": "max_power_kw", + "region_water_scarcity": "water_scarcity_index", + } + ) + _require_columns(config, REQUIRED_DC_COLUMNS, "Data center configuration") + if config["dc_id"].duplicated().any(): + raise ValueError("dc_id values must be unique.") + + numeric_columns = [ + "max_power_kw", + "pue", + "water_usage_factor", + "water_scarcity_index", + ] + for column in numeric_columns: + config[column] = pd.to_numeric(config[column], errors="raise") + if (config["max_power_kw"] <= 0).any(): + raise ValueError("max_power_kw must be positive.") + if (config["pue"] < 1).any(): + raise ValueError("PUE must be at least 1.") + if (config["water_usage_factor"] < 0).any(): + raise ValueError("water_usage_factor cannot be negative.") + if not config["water_scarcity_index"].between(0, 1).all(): + raise ValueError("water_scarcity_index must be between 0 and 1.") + + missing_cities = sorted(set(config["city"].astype(str)) - set(env["city"])) + if missing_cities: + raise ValueError( + "No environmental data for configured cities: " + ", ".join(missing_cities) + ) time_index = pd.date_range( - start=pd.Timestamp(env_df["timestamp"].min()).floor("h"), - end=pd.Timestamp(env_df["timestamp"].max()).floor("h"), - freq="h", + env["timestamp"].min(), env["timestamp"].max(), freq="h" ) city_metrics: dict[str, PreparedCityMetrics] = {} - - for city, city_df in env_df.groupby("city"): - aligned = city_df.set_index("timestamp").sort_index().reindex(time_index) - co2 = pd.to_numeric(aligned["predicted_co2_intensity"], errors="coerce").to_numpy(dtype=float) - water = pd.to_numeric(aligned["predicted_water_intensity"], errors="coerce").to_numpy(dtype=float) - valid = np.isfinite(co2) & np.isfinite(water) + for city, city_frame in env.groupby("city"): + aligned = city_frame.set_index("timestamp").sort_index().reindex(time_index) + carbon = aligned["carbon_intensity_kg_per_kwh"].to_numpy(dtype=float) + water = aligned["water_intensity_l_per_kwh"].to_numpy(dtype=float) + valid = np.isfinite(carbon) & np.isfinite(water) city_metrics[str(city)] = PreparedCityMetrics( - co2_prefix=_prefix_sum(np.where(valid, co2, 0.0)), + carbon_prefix=_prefix_sum(np.where(valid, carbon, 0.0)), water_prefix=_prefix_sum(np.where(valid, water, 0.0)), valid_prefix=_prefix_sum(valid.astype(float)), ) - dc_records: list[dict[str, Any]] = [] - for _, row in dc_df.iterrows(): - dc_records.append( - { - "dc_id": str(row["dc_id"]), - "city": str(row["city"]), - "max_power_per_hour": float(row["max_power_per_hour"]), - "pue": float(row["pue"]), - "water_usage_factor": float(row["water_usage_factor"]), - "region_water_scarcity": float(row["region_water_scarcity"]), - } - ) - + records = tuple(config[list(REQUIRED_DC_COLUMNS)].to_dict(orient="records")) return PreparedSchedulerState( time_index=time_index, time_values=time_index.to_numpy(dtype="datetime64[ns]"), city_metrics=city_metrics, - dc_records=dc_records, + dc_records=records, ) -def _initialize_capacity_state(state: PreparedSchedulerState) -> dict[str, np.ndarray]: - horizon = len(state.time_index) +def initialize_capacity_state(state: PreparedSchedulerState) -> dict[str, np.ndarray]: return { - str(record["dc_id"]): np.zeros(horizon, dtype=np.float32) + str(record["dc_id"]): np.zeros(len(state.time_index), dtype=np.float64) for record in state.dc_records } +def reserve_capacity( + used_capacity: dict[str, np.ndarray], + candidate: ScheduleCandidate, + power_demand_kw: float, +) -> None: + used_capacity[candidate.dc_id][candidate.start_index : candidate.end_index] += ( + power_demand_kw + ) + + def _candidate_start_positions( state: PreparedSchedulerState, earliest_start: pd.Timestamp, @@ -238,520 +308,307 @@ def _candidate_start_positions( ) -> np.ndarray: if duration_hours <= 0 or duration_hours > len(state.time_index): return np.empty(0, dtype=np.int32) - - start_value = np.datetime64(pd.Timestamp(earliest_start).floor("h").to_datetime64(), "ns") - end_value = np.datetime64(pd.Timestamp(latest_start).floor("h").to_datetime64(), "ns") - lower = int(np.searchsorted(state.time_values, start_value, side="left")) - upper = int(np.searchsorted(state.time_values, end_value, side="right")) - 1 + lower = int( + np.searchsorted( + state.time_values, + np.datetime64(earliest_start.floor("h"), "ns"), + side="left", + ) + ) + upper = int( + np.searchsorted( + state.time_values, + np.datetime64(latest_start.floor("h"), "ns"), + side="right", + ) + ) - 1 upper = min(upper, len(state.time_index) - duration_hours) if lower > upper: return np.empty(0, dtype=np.int32) return np.arange(lower, upper + 1, dtype=np.int32) -def _window_is_valid(city_metrics: PreparedCityMetrics, start_index: int, end_index: int) -> bool: - valid_count = city_metrics.valid_prefix[end_index] - city_metrics.valid_prefix[start_index] - return bool(valid_count >= (end_index - start_index)) - - -def _window_mean(prefix: np.ndarray, start_index: int, end_index: int) -> float: - span = max(1, end_index - start_index) - return float((prefix[end_index] - prefix[start_index]) / span) - - -def _candidate_score( - co2_mean: float, - water_mean: float, - dc_record: dict[str, Any], - origin_city: str, - alpha: float, - beta: float, - gamma: float, -) -> tuple[float, float, float]: - co2_component = co2_mean * float(dc_record["pue"]) - water_component = ( - water_mean - * float(dc_record["water_usage_factor"]) - * float(dc_record["region_water_scarcity"]) - ) - latency_penalty = gamma if origin_city != str(dc_record["city"]) else 0.0 - score = alpha * co2_component + beta * water_component + latency_penalty - return float(score), float(co2_component), float(water_component) - - -def _capacity_available( - used_capacity: dict[str, np.ndarray], - dc_id: str, - start_index: int, - end_index: int, - required_power: float, - max_power_per_hour: float, -) -> bool: - window = used_capacity[dc_id][start_index:end_index] - if window.size == 0: - return False - return float(window.max()) + required_power <= max_power_per_hour - - -def _reserve_capacity( - used_capacity: dict[str, np.ndarray], - dc_id: str, - start_index: int, - end_index: int, - required_power: float, -) -> None: - used_capacity[dc_id][start_index:end_index] += required_power +def _window_mean(prefix: np.ndarray, start: int, end: int) -> float: + return float((prefix[end] - prefix[start]) / (end - start)) -def _enumerate_candidates( +def enumerate_feasible_candidates( job: pd.Series, - prepared_state: PreparedSchedulerState, - used_capacity: dict[str, np.ndarray] | None, - alpha: float, - beta: float, - gamma: float, + state: PreparedSchedulerState, + used_capacity: dict[str, np.ndarray], ) -> list[ScheduleCandidate]: - duration_hours = int(job["duration_hours"]) - earliest_start = pd.Timestamp(job["earliest_start"]).floor("h") - latest_start = (pd.Timestamp(job["deadline"]) - pd.Timedelta(hours=duration_hours)).floor("h") - required_power = float(job["power_demand"]) - - if latest_start < earliest_start: - return [] - - start_positions = _candidate_start_positions( - prepared_state, - earliest_start, - latest_start, - duration_hours, - ) - if start_positions.size == 0: + """Enumerate all deadline, data, and capacity feasible placements.""" + duration = int(job["duration_hours"]) + power = float(job["power_demand_kw"]) + earliest = _utc_naive_timestamp(job["earliest_start"]).floor("h") + deadline = _utc_naive_timestamp(job["deadline"]) + latest_start = (deadline - pd.Timedelta(hours=duration)).floor("h") + positions = _candidate_start_positions(state, earliest, latest_start, duration) + if positions.size == 0 or power <= 0: return [] - capacity_state = used_capacity or _initialize_capacity_state(prepared_state) candidates: list[ScheduleCandidate] = [] - origin_city = str(job["origin_city"]) - - for dc_record in prepared_state.dc_records: - city = str(dc_record["city"]) - city_metrics = prepared_state.city_metrics.get(city) - if city_metrics is None: - continue - - dc_id = str(dc_record["dc_id"]) - for start_index in start_positions: - end_index = int(start_index + duration_hours) - if not _window_is_valid(city_metrics, int(start_index), end_index): + for record in state.dc_records: + city = str(record["city"]) + dc_id = str(record["dc_id"]) + metrics = state.city_metrics[city] + max_power = float(record["max_power_kw"]) + for start in positions: + start_index = int(start) + end_index = start_index + duration + valid_count = metrics.valid_prefix[end_index] - metrics.valid_prefix[start_index] + if valid_count != duration: continue - if not _capacity_available( - capacity_state, - dc_id, - int(start_index), - end_index, - required_power, - float(dc_record["max_power_per_hour"]), - ): + capacity_window = used_capacity[dc_id][start_index:end_index] + if not np.all(capacity_window + power <= max_power + 1e-12): continue - co2_mean = _window_mean(city_metrics.co2_prefix, int(start_index), end_index) - water_mean = _window_mean(city_metrics.water_prefix, int(start_index), end_index) - score, co2_component, water_component = _candidate_score( - co2_mean, - water_mean, - dc_record, - origin_city, - alpha, - beta, - gamma, + carbon_intensity = _window_mean( + metrics.carbon_prefix, start_index, end_index + ) + water_intensity = _window_mean(metrics.water_prefix, start_index, end_index) + it_energy_kwh = power * duration + expected_co2 = it_energy_kwh * float(record["pue"]) * carbon_intensity + expected_water = ( + it_energy_kwh + * water_intensity + * float(record["water_usage_factor"]) ) - start_time = pd.Timestamp(prepared_state.time_index[int(start_index)]) - end_time = pd.Timestamp(prepared_state.time_index[end_index - 1]) + pd.Timedelta(hours=1) + scarcity_water = expected_water * ( + 1.0 + float(record["water_scarcity_index"]) + ) + start_time = pd.Timestamp(state.time_index[start_index]) candidates.append( ScheduleCandidate( dc_id=dc_id, city=city, start_time=start_time, - end_time=end_time, - score=score, - expected_co2_kg=required_power * co2_component * duration_hours, - expected_water_liters=required_power * water_component * duration_hours, - start_index=int(start_index), + end_time=start_time + pd.Timedelta(hours=duration), + start_index=start_index, end_index=end_index, + expected_co2_kg=float(expected_co2), + expected_water_liters=float(expected_water), + scarcity_weighted_water_liters=float(scarcity_water), + relocation_penalty=float(str(job["origin_city"]) != city), ) ) + return candidates - return sorted(candidates, key=lambda candidate: candidate.score) +def _minmax(values: np.ndarray) -> np.ndarray: + minimum = float(values.min()) + spread = float(values.max() - minimum) + if spread <= 1e-12: + return np.zeros_like(values, dtype=float) + return (values - minimum) / spread -def _find_best_assignment( - job: pd.Series, - prepared_state: PreparedSchedulerState, - used_capacity: dict[str, np.ndarray], + +def score_candidates( + candidates: list[ScheduleCandidate], alpha: float, beta: float, gamma: float, -) -> ScheduleCandidate | None: - candidates = _enumerate_candidates( - job=job, - prepared_state=prepared_state, - used_capacity=used_capacity, - alpha=alpha, - beta=beta, - gamma=gamma, - ) - return candidates[0] if candidates else None - - -def rank_job_candidates( - env_df: pd.DataFrame, - dc_df: pd.DataFrame, - job: pd.Series, - alpha: float = 1.0, - beta: float = 1.0, - gamma: float = 0.05, - top_k: int = 5, -) -> pd.DataFrame: - """Return the best feasible routing options for a single job.""" - candidates_df = list_job_candidates( - env_df=env_df, - dc_df=dc_df, - job=job, - alpha=alpha, - beta=beta, - gamma=gamma, - ) - if candidates_df.empty: - return candidates_df +) -> list[ScheduleCandidate]: + """Score candidates after per-job min-max normalization of physical impacts.""" + weights = np.asarray([alpha, beta, gamma], dtype=float) + if (weights < 0).any() or not np.isclose(weights.sum(), 1.0, atol=1e-8): + raise ValueError("alpha, beta, and gamma must be non-negative and sum to 1.") + if not candidates: + return [] - unique_candidates = ( - candidates_df.sort_values("scheduler_score") - .drop_duplicates(subset=["assigned_city"], keep="first") - .head(top_k) - .reset_index(drop=True) + carbon = _minmax(np.asarray([item.expected_co2_kg for item in candidates])) + water = _minmax( + np.asarray([item.scarcity_weighted_water_liters for item in candidates]) ) - unique_candidates["rank"] = np.arange(1, len(unique_candidates) + 1) - ordered_columns = [ - "rank", - "job_id", - "origin_city", - "assigned_dc_id", - "assigned_city", - "scheduled_start", - "scheduled_end", - "scheduler_score", - "expected_co2_kg", - "expected_water_liters", - "same_city", + scored = [ + replace( + item, + score=float( + alpha * carbon[index] + + beta * water[index] + + gamma * item.relocation_penalty + ), + ) + for index, item in enumerate(candidates) ] - return unique_candidates[ordered_columns] + return sorted( + scored, + key=lambda item: (item.score, item.start_time, item.dc_id), + ) -def list_job_candidates( - env_df: pd.DataFrame, - dc_df: pd.DataFrame, - job: pd.Series, - alpha: float = 1.0, - beta: float = 1.0, - gamma: float = 0.05, - prepared_state: PreparedSchedulerState | None = None, -) -> pd.DataFrame: - """Return all feasible routing options for a single job.""" - state = prepared_state or prepare_scheduler_state(env_df, dc_df) - candidates = _enumerate_candidates( - job=job, - prepared_state=state, - used_capacity=_initialize_capacity_state(state), - alpha=alpha, - beta=beta, - gamma=gamma, - ) - ranked_rows: list[dict[str, object]] = [] - for candidate in candidates: - ranked_rows.append( - { - "job_id": job.get("job_id", "simulated_job"), - "origin_city": str(job["origin_city"]), - "assigned_dc_id": candidate.dc_id, - "assigned_city": candidate.city, - "scheduled_start": candidate.start_time, - "scheduled_end": candidate.end_time, - "scheduler_score": candidate.score, - "expected_co2_kg": candidate.expected_co2_kg, - "expected_water_liters": candidate.expected_water_liters, - "same_city": int(str(job["origin_city"]) == candidate.city), - } - ) - return pd.DataFrame(ranked_rows) +def _validate_jobs(jobs_df: pd.DataFrame) -> pd.DataFrame: + jobs = jobs_df.copy() + if "power_demand" in jobs and "power_demand_kw" not in jobs: + jobs = jobs.rename(columns={"power_demand": "power_demand_kw"}) + _require_columns(jobs, REQUIRED_JOB_COLUMNS, "Jobs dataset") + jobs["earliest_start"] = _utc_naive_series(jobs["earliest_start"]) + jobs["deadline"] = _utc_naive_series(jobs["deadline"]) + jobs["duration_hours"] = pd.to_numeric(jobs["duration_hours"], errors="raise").astype(int) + jobs["power_demand_kw"] = pd.to_numeric(jobs["power_demand_kw"], errors="raise") + jobs["priority"] = pd.to_numeric(jobs.get("priority", 1), errors="coerce").fillna(1) + if jobs["job_id"].duplicated().any(): + raise ValueError("job_id values must be unique.") + return jobs.sort_values(["priority", "earliest_start", "job_id"]).reset_index(drop=True) + + +def _schedule_row(job: pd.Series, candidate: ScheduleCandidate | None) -> dict[str, object]: + row = job.to_dict() + if candidate is None: + return { + **row, + "assigned_dc_id": None, + "assigned_city": None, + "scheduled_start": pd.NaT, + "scheduled_end": pd.NaT, + "scheduler_score": np.nan, + "expected_co2_kg": np.nan, + "expected_water_liters": np.nan, + "scarcity_weighted_water_liters": np.nan, + "relocated": np.nan, + "scheduled": 0, + "failure_reason": "no_feasible_slot", + } + return { + **row, + "assigned_dc_id": candidate.dc_id, + "assigned_city": candidate.city, + "scheduled_start": candidate.start_time, + "scheduled_end": candidate.end_time, + "scheduler_score": candidate.score, + "expected_co2_kg": candidate.expected_co2_kg, + "expected_water_liters": candidate.expected_water_liters, + "scarcity_weighted_water_liters": candidate.scarcity_weighted_water_liters, + "relocated": int(candidate.relocation_penalty > 0), + "scheduled": 1, + "failure_reason": None, + } -def schedule_jobs( +def schedule_with_policy( env_df: pd.DataFrame, jobs_df: pd.DataFrame, dc_df: pd.DataFrame, - alpha: float = 1.0, - beta: float = 1.0, - gamma: float = 0.05, - prepared_state: PreparedSchedulerState | None = None, + selector: CandidateSelector, + alpha: float = 0.55, + beta: float = 0.35, + gamma: float = 0.10, ) -> pd.DataFrame: - """Greedy multi-objective scheduler with hourly capacity constraints.""" - if jobs_df.empty: - return jobs_df.copy() - - state = prepared_state or prepare_scheduler_state(env_df, dc_df) - used_capacity = _initialize_capacity_state(state) - - jobs = jobs_df.copy() - jobs["priority_sort"] = jobs["priority"].fillna(1) - jobs = jobs.sort_values(["priority_sort", "earliest_start", "job_id"]).reset_index(drop=True) - - scheduled_rows: list[dict[str, object]] = [] - for _, job in jobs.iterrows(): - assignment = _find_best_assignment( - job, - state, - used_capacity, - alpha, - beta, - gamma, - ) + """Run a deterministic greedy policy with a caller-supplied selector.""" + state = prepare_scheduler_state(env_df, dc_df) + used_capacity = initialize_capacity_state(state) + rows: list[dict[str, object]] = [] + for _, job in _validate_jobs(jobs_df).iterrows(): + raw_candidates = enumerate_feasible_candidates(job, state, used_capacity) + candidates = score_candidates(raw_candidates, alpha, beta, gamma) + candidate = selector(job, candidates) if candidates else None + if candidate is not None: + reserve_capacity(used_capacity, candidate, float(job["power_demand_kw"])) + rows.append(_schedule_row(job, candidate)) + return pd.DataFrame(rows) - if assignment is None: - scheduled_rows.append( - { - **job.to_dict(), - "assigned_dc_id": None, - "assigned_city": None, - "scheduled_start": pd.NaT, - "scheduled_end": pd.NaT, - "scheduler_score": np.nan, - "expected_co2_kg": np.nan, - "expected_water_liters": np.nan, - "scheduled": 0, - } - ) - continue - - _reserve_capacity( - used_capacity, - assignment.dc_id, - assignment.start_index, - assignment.end_index, - float(job["power_demand"]), - ) - scheduled_rows.append( - { - **job.to_dict(), - "assigned_dc_id": assignment.dc_id, - "assigned_city": assignment.city, - "scheduled_start": assignment.start_time, - "scheduled_end": assignment.end_time, - "scheduler_score": assignment.score, - "expected_co2_kg": assignment.expected_co2_kg, - "expected_water_liters": assignment.expected_water_liters, - "scheduled": 1, - } - ) - return pd.DataFrame(scheduled_rows).drop(columns=["priority_sort"], errors="ignore") - - -def schedule_jobs_naive( +def schedule_jobs( env_df: pd.DataFrame, jobs_df: pd.DataFrame, dc_df: pd.DataFrame, - prepared_state: PreparedSchedulerState | None = None, + alpha: float = 0.55, + beta: float = 0.35, + gamma: float = 0.10, ) -> pd.DataFrame: - """Naive baseline: prefer same-city placement, otherwise first feasible DC.""" - naive_dc = dc_df.copy() - return schedule_jobs( + """Run the normalized multi-objective greedy scheduler.""" + + def best_score( + _job: pd.Series, candidates: list[ScheduleCandidate] + ) -> ScheduleCandidate: + return candidates[0] + + return schedule_with_policy( env_df, jobs_df, - naive_dc, - alpha=0.0, - beta=0.0, - gamma=0.20, - prepared_state=prepared_state, + dc_df, + selector=best_score, + alpha=alpha, + beta=beta, + gamma=gamma, ) def summarize_schedule(schedule_df: pd.DataFrame) -> dict[str, float]: - scheduled_mask = schedule_df["scheduled"] == 1 + """Summarize coverage and physical impacts without hiding failed jobs.""" + if schedule_df.empty: + return { + "total_jobs": 0.0, + "scheduled_jobs": 0.0, + "coverage": 0.0, + "total_co2_kg": 0.0, + "total_water_liters": 0.0, + "total_scarcity_weighted_water_liters": 0.0, + "relocation_rate": 0.0, + } + scheduled = schedule_df["scheduled"].eq(1) return { - "scheduled_jobs": float(scheduled_mask.sum()), "total_jobs": float(len(schedule_df)), - "coverage": float(scheduled_mask.mean()) if len(schedule_df) else 0.0, - "total_expected_co2_kg": float( - schedule_df.loc[scheduled_mask, "expected_co2_kg"].sum() + "scheduled_jobs": float(scheduled.sum()), + "coverage": float(scheduled.mean()), + "total_co2_kg": float(schedule_df.loc[scheduled, "expected_co2_kg"].sum()), + "total_water_liters": float( + schedule_df.loc[scheduled, "expected_water_liters"].sum() ), - "total_expected_water_liters": float( - schedule_df.loc[scheduled_mask, "expected_water_liters"].sum() + "total_scarcity_weighted_water_liters": float( + schedule_df.loc[scheduled, "scarcity_weighted_water_liters"].sum() ), + "relocation_rate": float(schedule_df.loc[scheduled, "relocated"].mean()) + if scheduled.any() + else 0.0, } -def compare_summaries( - baseline: dict[str, float], optimized: dict[str, float] +def paired_comparison( + baseline_schedule: pd.DataFrame, + proposed_schedule: pd.DataFrame, ) -> dict[str, float]: - def reduction(old: float, new: float) -> float: - if old == 0: - return 0.0 - return ((old - new) / old) * 100.0 - - return { - "co2_reduction_pct": reduction( - baseline["total_expected_co2_kg"], optimized["total_expected_co2_kg"] - ), - "water_reduction_pct": reduction( - baseline["total_expected_water_liters"], - optimized["total_expected_water_liters"], - ), - "coverage_delta_pct": (optimized["coverage"] - baseline["coverage"]) * 100.0, - "scheduled_jobs_delta": optimized["scheduled_jobs"] - baseline["scheduled_jobs"], - } - - -def run_scheduling_pipeline( - data_path: str | Path, - jobs_path: str | Path = DEFAULT_JOBS_PATH, - dc_config_path: str | Path = DEFAULT_DC_CONFIG_PATH, - job_limit: int | None = None, - alpha: float = 1.0, - beta: float = 1.0, - gamma: float = 0.05, - priority_filter: str | None = None, -) -> dict[str, object]: - jobs_df = load_jobs_dataset(jobs_path) - if priority_filter is not None and priority_filter != "all" and "priority_label" in jobs_df.columns: - jobs_df = jobs_df[jobs_df["priority_label"].str.lower() == priority_filter.lower()].copy() - if job_limit is not None: - jobs_df = jobs_df.head(job_limit).copy() - - dc_df = load_dc_config(dc_config_path) - data_df = load_dataset(data_path) - env_df = prepare_environment_frame(data_df, city_subset=dc_df["city"].unique().tolist()) - prepared_state = prepare_scheduler_state(env_df, dc_df) - - optimized_schedule = schedule_jobs( - env_df, - jobs_df, - dc_df, - alpha=alpha, - beta=beta, - gamma=gamma, - prepared_state=prepared_state, - ) - baseline_schedule = schedule_jobs_naive(env_df, jobs_df, dc_df, prepared_state=prepared_state) - - optimized_summary = summarize_schedule(optimized_schedule) + """Compare impacts only for jobs completed by both policies.""" baseline_summary = summarize_schedule(baseline_schedule) - comparison = compare_summaries(baseline_summary, optimized_summary) + proposed_summary = summarize_schedule(proposed_schedule) + baseline = baseline_schedule[baseline_schedule["scheduled"].eq(1)][ + [ + "job_id", + "power_demand_kw", + "duration_hours", + "expected_co2_kg", + "expected_water_liters", + "scarcity_weighted_water_liters", + ] + ] + proposed = proposed_schedule[proposed_schedule["scheduled"].eq(1)][ + [ + "job_id", + "expected_co2_kg", + "expected_water_liters", + "scarcity_weighted_water_liters", + ] + ] + paired = baseline.merge(proposed, on="job_id", suffixes=("_baseline", "_proposed")) + + def reduction(column: str) -> float: + old = float(paired[f"{column}_baseline"].sum()) + new = float(paired[f"{column}_proposed"].sum()) + return 100.0 * (old - new) / old if old > 0 else 0.0 + common_energy = ( + paired["power_demand_kw"] * paired["duration_hours"] + ).sum() if not paired.empty else 0.0 return { - "environment": env_df, - "optimized_schedule": optimized_schedule, - "baseline_schedule": baseline_schedule, - "optimized_summary": optimized_summary, - "baseline_summary": baseline_summary, - "comparison": comparison, + "baseline_coverage": baseline_summary["coverage"], + "proposed_coverage": proposed_summary["coverage"], + "coverage_gap_percentage_points": 100.0 + * (proposed_summary["coverage"] - baseline_summary["coverage"]), + "common_jobs": float(len(paired)), + "common_it_energy_kwh": float(common_energy), + "co2_reduction_pct": reduction("expected_co2_kg"), + "water_reduction_pct": reduction("expected_water_liters"), + "scarcity_weighted_water_reduction_pct": reduction( + "scarcity_weighted_water_liters" + ), } - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Run the first scheduling pipeline using environmental metrics and workload traces." - ) - parser.add_argument( - "--data-path", - required=True, - help="Path to the master sustainability dataset (.zip, .parquet, .csv, or .xlsx).", - ) - parser.add_argument( - "--jobs-path", - default=str(DEFAULT_JOBS_PATH), - help="Path to the scheduler jobs CSV.", - ) - parser.add_argument( - "--dc-config-path", - default=str(DEFAULT_DC_CONFIG_PATH), - help="Path to the data-center configuration CSV.", - ) - parser.add_argument( - "--job-limit", - type=int, - default=100, - help="Optional number of jobs to schedule for a quick experiment.", - ) - parser.add_argument( - "--alpha", - type=float, - default=1.0, - help="Weight for carbon intensity in the scheduler objective.", - ) - parser.add_argument( - "--beta", - type=float, - default=1.0, - help="Weight for water intensity in the scheduler objective.", - ) - parser.add_argument( - "--gamma", - type=float, - default=0.05, - help="Latency penalty applied when routing away from the origin city.", - ) - parser.add_argument( - "--priority-filter", - default="all", - choices=["all", "high", "low"], - help="Optional job-priority subset to schedule.", - ) - parser.add_argument( - "--output-path", - default="data/processed/schedule_results.csv", - help="Where to write the optimized schedule CSV.", - ) - return parser.parse_args() - - -def main() -> None: - args = parse_args() - results = run_scheduling_pipeline( - data_path=args.data_path, - jobs_path=args.jobs_path, - dc_config_path=args.dc_config_path, - job_limit=args.job_limit, - alpha=args.alpha, - beta=args.beta, - gamma=args.gamma, - priority_filter=args.priority_filter, - ) - - output_path = _resolve_path(args.output_path) - output_path.parent.mkdir(parents=True, exist_ok=True) - results["optimized_schedule"].to_csv(output_path, index=False) - - print("Scheduling summary") - print( - f" Jobs scheduled: {int(results['optimized_summary']['scheduled_jobs'])}/" - f"{int(results['optimized_summary']['total_jobs'])}" - ) - print( - f" Coverage: {results['optimized_summary']['coverage']:.2%}" - ) - print( - f" Optimized CO2 (kg): {results['optimized_summary']['total_expected_co2_kg']:.2f}" - ) - print( - f" Optimized water (L): {results['optimized_summary']['total_expected_water_liters']:.2f}" - ) - print( - f" Baseline CO2 reduction: {results['comparison']['co2_reduction_pct']:.2f}%" - ) - print( - f" Baseline water reduction: {results['comparison']['water_reduction_pct']:.2f}%" - ) - print(f" Saved optimized schedule to: {output_path}") - - -if __name__ == "__main__": - main() diff --git a/src/sensitivity_analysis.py b/src/sensitivity_analysis.py deleted file mode 100644 index 1d1e2e2..0000000 --- a/src/sensitivity_analysis.py +++ /dev/null @@ -1,212 +0,0 @@ -""" -sensitivity_analysis.py ------------------------ -Grid search over scheduler weight parameters (α, β) to map the Pareto -frontier between carbon emissions and water usage. - -The scheduler optimizes: score = α×co2 + β×water + γ×distance -where α + β + γ = 1. - -This analysis answers: - "How does operator preference (carbon vs water vs latency) affect outcomes?" - -Methodology: - For each (α, β) combination on a grid: - 1. Schedule the same job batch. - 2. Record total CO2 and total water. - 3. Plot/report the Pareto frontier. - -Output: - - Console table - - data/results/sensitivity_analysis_{region}.csv - - Pareto-optimal points highlighted - -Usage: - python -m src.sensitivity_analysis - python -m src.sensitivity_analysis --region EU --jobs 50 -""" - -from __future__ import annotations - -import argparse -from pathlib import Path - -import numpy as np -import pandas as pd - -try: - from .scheduling import ( - load_dc_config, - prepare_environment_frame, - schedule_jobs, - summarize_schedule, - ) - from .data_loader import PROJECT_ROOT, load_dataset -except ImportError: - from scheduling import ( - load_dc_config, - prepare_environment_frame, - schedule_jobs, - summarize_schedule, - ) - from data_loader import PROJECT_ROOT, load_dataset - -EU_DATA_PATH = PROJECT_ROOT / "data" / "eu_data_with_metrics.csv" -EU_DC_PATH = PROJECT_ROOT / "data" / "templates" / "eu_dc_config.csv" - - -def _build_jobs(dc_df: pd.DataFrame, n: int, data_start: str, data_end: str, seed: int = 42) -> pd.DataFrame: - rng = np.random.default_rng(seed) - cities = dc_df["city"].tolist() - return pd.DataFrame({ - "job_id": [f"job_{i:04d}" for i in range(n)], - "power_demand": rng.choice([10.0, 20.0, 50.0, 100.0], n).tolist(), - "duration_hours": rng.integers(2, 9, n).tolist(), - "earliest_start": data_start, - "deadline": data_end, - "origin_city": rng.choice(cities, n).tolist(), - "priority": 1, - }) - - -def _is_pareto_optimal(df: pd.DataFrame) -> pd.Series: - """ - Mark rows that are Pareto-optimal on (co2, water) — minimizing both. - A point is dominated if another point has strictly lower CO2 AND water. - """ - dominated = np.zeros(len(df), dtype=bool) - co2 = df["total_co2_kg"].values - water = df["total_water_l"].values - for i in range(len(df)): - for j in range(len(df)): - if i == j: - continue - if co2[j] <= co2[i] and water[j] <= water[i] and (co2[j] < co2[i] or water[j] < water[i]): - dominated[i] = True - break - return pd.Series(~dominated, name="pareto_optimal") - - -def run_sensitivity_analysis( - data_df: pd.DataFrame, - dc_df: pd.DataFrame, - jobs_df: pd.DataFrame, - alpha_values: list[float] | None = None, - beta_values: list[float] | None = None, -) -> pd.DataFrame: - """ - Grid search over (α, β) weights. γ = 1 - α - β (distance penalty). - Points where α + β > 1 are skipped. - """ - if alpha_values is None: - alpha_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0] - if beta_values is None: - beta_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0] - - env = prepare_environment_frame(data_df) - rows = [] - - for alpha in alpha_values: - for beta in beta_values: - gamma = round(1.0 - alpha - beta, 4) - if gamma < -1e-6: - continue # invalid combination - - sched = schedule_jobs(env, jobs_df, dc_df, alpha=alpha, beta=beta, gamma=gamma) - summary = summarize_schedule(sched) - - rows.append({ - "alpha": alpha, - "beta": beta, - "gamma": round(gamma, 4), - "weight_label": f"α={alpha:.1f} β={beta:.1f} γ={gamma:.1f}", - "total_co2_kg": round(summary["total_expected_co2_kg"], 2), - "total_water_l": round(summary["total_expected_water_liters"], 2), - "jobs_scheduled": int(summary["scheduled_jobs"]), - "coverage": round(summary["coverage"], 3), - }) - - result = pd.DataFrame(rows) - result["pareto_optimal"] = _is_pareto_optimal(result) - return result - - -def _print_report(df: pd.DataFrame, region: str) -> None: - print("=" * 75) - print(f"SENSITIVITY ANALYSIS — {region} GRID") - print(f"{'α (carbon)':>10} {'β (water)':>10} {'γ (dist)':>9} {'CO2 (kg)':>12} {'Water (L)':>11} {'Pareto':>7}") - print("-" * 75) - - for _, row in df.sort_values(["alpha", "beta"]).iterrows(): - pareto = "★" if row["pareto_optimal"] else "" - print( - f"{row['alpha']:>10.1f} {row['beta']:>10.1f} {row['gamma']:>9.1f} " - f"{row['total_co2_kg']:>12,.1f} {row['total_water_l']:>11,.1f} {pareto:>7}" - ) - - pareto_df = df[df["pareto_optimal"]].sort_values("total_co2_kg") - print("=" * 75) - print(f"\nPareto-optimal configurations ({len(pareto_df)} points):") - print(f" {'Config':<22} {'CO2 (kg)':>12} {'Water (L)':>11}") - print(f" {'-'*47}") - for _, row in pareto_df.iterrows(): - print(f" {row['weight_label']:<22} {row['total_co2_kg']:>12,.1f} {row['total_water_l']:>11,.1f}") - - min_co2 = df.loc[df["total_co2_kg"].idxmin()] - min_water = df.loc[df["total_water_l"].idxmin()] - print(f"\nExtremes:") - print(f" Min CO2: {min_co2['total_co2_kg']:>10,.1f} kg at α={min_co2['alpha']:.1f}, β={min_co2['beta']:.1f}") - print(f" Min water: {min_water['total_water_l']:>10,.1f} L at α={min_water['alpha']:.1f}, β={min_water['beta']:.1f}") - - co2_range = df["total_co2_kg"].max() - df["total_co2_kg"].min() - water_range = df["total_water_l"].max() - df["total_water_l"].min() - print(f"\nSensitivity range:") - print(f" CO2: {co2_range:>10,.1f} kg ({100*co2_range/df['total_co2_kg'].max():.1f}% of max)") - print(f" Water: {water_range:>10,.1f} L ({100*water_range/df['total_water_l'].max():.1f}% of max)") - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Sensitivity analysis: α/β weight grid search" - ) - parser.add_argument("--region", choices=["US", "EU"], default="US") - parser.add_argument("--jobs", type=int, default=100) - parser.add_argument("--output", default=None) - args = parser.parse_args() - - if args.region == "EU": - if not EU_DATA_PATH.exists(): - print(f"[ERROR] EU dataset not found: {EU_DATA_PATH}") - return - data_df = pd.read_csv(EU_DATA_PATH) - dc_df = pd.read_csv(EU_DC_PATH) - data_df = data_df[ - (data_df["TIMESTAMP"] >= "2023-01-01") & - (data_df["TIMESTAMP"] < "2023-07-01") - ].copy() - jobs_df = _build_jobs(dc_df, args.jobs, "2023-01-01", "2023-06-30") - else: - data_df = load_dataset() - dc_df = load_dc_config() - data_df = data_df[ - (data_df["TIMESTAMP"] >= "2023-01-01") & - (data_df["TIMESTAMP"] < "2023-07-01") - ].copy() - jobs_df = _build_jobs(dc_df, args.jobs, "2023-01-01", "2023-06-30") - - print(f"Sensitivity Analysis — {args.region} | {args.jobs} jobs | 11×11 weight grid") - print(f" Dataset: {len(data_df):,} rows, {dc_df['city'].nunique()} cities\n") - - df = run_sensitivity_analysis(data_df, dc_df, jobs_df) - _print_report(df, args.region) - - out_path = Path(args.output) if args.output else ( - PROJECT_ROOT / "data" / "results" / f"sensitivity_analysis_{args.region.lower()}.csv" - ) - out_path.parent.mkdir(parents=True, exist_ok=True) - df.to_csv(out_path, index=False) - print(f"\nSaved → {out_path}") - - -if __name__ == "__main__": - main() diff --git a/src/synthetic_benchmark.py b/src/synthetic_benchmark.py new file mode 100644 index 0000000..e51ec5e --- /dev/null +++ b/src/synthetic_benchmark.py @@ -0,0 +1,152 @@ +"""Deterministic benchmark data for offline and CI experiments. + +The benchmark is synthetic by design. It encodes plausible hourly variation and +explicit carbon-water tradeoffs without representing a real operator or site. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +ZONE_PROFILES = { + "Zone North": { + "carbon_base": 0.18, + "carbon_amplitude": 0.05, + "carbon_phase": 4.0, + "water_base": 1.10, + "water_amplitude": 0.10, + "pue": 1.12, + "water_scarcity_index": 0.15, + }, + "Zone West": { + "carbon_base": 0.30, + "carbon_amplitude": 0.12, + "carbon_phase": 13.0, + "water_base": 0.35, + "water_amplitude": 0.05, + "pue": 1.18, + "water_scarcity_index": 0.80, + }, + "Zone South": { + "carbon_base": 0.48, + "carbon_amplitude": 0.07, + "carbon_phase": 18.0, + "water_base": 0.45, + "water_amplitude": 0.08, + "pue": 1.20, + "water_scarcity_index": 0.65, + }, + "Zone East": { + "carbon_base": 0.34, + "carbon_amplitude": 0.06, + "carbon_phase": 8.0, + "water_base": 0.70, + "water_amplitude": 0.07, + "pue": 1.15, + "water_scarcity_index": 0.30, + }, +} + + +def generate_environment(seed: int = 42, hours: int = 168) -> pd.DataFrame: + """Generate one week of complete hourly environmental signals.""" + if hours < 48: + raise ValueError("The benchmark requires at least 48 hours.") + rng = np.random.default_rng(seed) + timestamps = pd.date_range("2025-01-01", periods=hours, freq="h") + rows: list[dict[str, object]] = [] + hour_of_day = np.arange(hours) % 24 + slow_cycle = np.sin(2 * np.pi * np.arange(hours) / (24 * 7)) + + for city, profile in ZONE_PROFILES.items(): + phase = float(profile["carbon_phase"]) + carbon = ( + float(profile["carbon_base"]) + + float(profile["carbon_amplitude"]) + * np.sin(2 * np.pi * (hour_of_day - phase) / 24) + + 0.015 * slow_cycle + + rng.normal(0, 0.008, hours) + ) + water = ( + float(profile["water_base"]) + + float(profile["water_amplitude"]) + * np.sin(2 * np.pi * (hour_of_day - 14) / 24) + + rng.normal(0, 0.012, hours) + ) + for timestamp, carbon_value, water_value in zip( + timestamps, carbon, water, strict=True + ): + rows.append( + { + "timestamp": timestamp, + "city": city, + "carbon_intensity_kg_per_kwh": max(0.01, float(carbon_value)), + "water_intensity_l_per_kwh": max(0.01, float(water_value)), + } + ) + return pd.DataFrame(rows) + + +def generate_dc_config(max_power_kw: float = 160.0) -> pd.DataFrame: + """Generate four synthetic data center configurations.""" + return pd.DataFrame( + [ + { + "dc_id": f"dc_{index}", + "city": city, + "max_power_kw": float(max_power_kw), + "pue": float(profile["pue"]), + "water_usage_factor": 1.0, + "water_scarcity_index": float(profile["water_scarcity_index"]), + } + for index, (city, profile) in enumerate(ZONE_PROFILES.items(), start=1) + ] + ) + + +def generate_jobs( + seed: int = 42, + count: int = 180, + hours: int = 168, +) -> pd.DataFrame: + """Generate non-preemptive jobs with heterogeneous power, duration, and slack.""" + if count <= 0: + raise ValueError("count must be positive.") + rng = np.random.default_rng(seed) + cities = list(ZONE_PROFILES) + anchor = pd.Timestamp("2025-01-01") + latest_arrival_hour = max(1, hours - 30) + arrivals = rng.integers(0, latest_arrival_hour, count) + durations = rng.integers(1, 7, count) + slack = rng.integers(3, 25, count) + deadlines = np.minimum(arrivals + durations + slack, hours) + + return pd.DataFrame( + { + "job_id": [f"job_{seed:03d}_{index:04d}" for index in range(count)], + "origin_city": rng.choice(cities, count), + "power_demand_kw": rng.integers(15, 46, count).astype(float), + "duration_hours": durations.astype(int), + "earliest_start": [ + anchor + pd.Timedelta(hours=int(hour)) for hour in arrivals + ], + "deadline": [ + anchor + pd.Timedelta(hours=int(hour)) for hour in deadlines + ], + "priority": rng.choice([0, 1, 2], count, p=[0.20, 0.55, 0.25]), + } + ) + + +def generate_benchmark( + seed: int = 42, + jobs: int = 180, + hours: int = 168, +) -> tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame]: + """Return environment, jobs, and data center frames for one seed.""" + return ( + generate_environment(seed=seed, hours=hours), + generate_jobs(seed=seed, count=jobs, hours=hours), + generate_dc_config(), + ) diff --git a/src/train.py b/src/train.py deleted file mode 100644 index 8e486c5..0000000 --- a/src/train.py +++ /dev/null @@ -1,579 +0,0 @@ -from __future__ import annotations - -import argparse -from pathlib import Path -import re -from typing import Any - -import joblib -import numpy as np -import pandas as pd - -if __package__: - from .runtime import ensure_openmp_runtime -else: - from runtime import ensure_openmp_runtime - -ensure_openmp_runtime() - -try: - from prophet import Prophet -except ImportError: # pragma: no cover - handled at runtime. - Prophet = None - -try: - from xgboost import XGBRegressor -except ImportError: # pragma: no cover - handled at runtime. - XGBRegressor = None - -if __package__: - from .data_loader import PROJECT_ROOT, load_dataset - from .evaluate import compare_to_baseline, regression_metrics - from .preprocessing import ( - FutureFeatureProfiles, - build_feature_matrix, - build_future_feature_frame, - build_future_feature_profiles, - build_profile_feature_frame, - feature_columns_for_frame, - prepare_forecasting_frame, - select_recent_history, - split_time_train_test, - ) - from .tune import load_best_params - from .weather_loader import load_weather_dataset, merge_weather_features -else: - from data_loader import PROJECT_ROOT, load_dataset - from evaluate import compare_to_baseline, regression_metrics - from preprocessing import ( - FutureFeatureProfiles, - build_feature_matrix, - build_future_feature_frame, - build_future_feature_profiles, - build_profile_feature_frame, - feature_columns_for_frame, - prepare_forecasting_frame, - select_recent_history, - split_time_train_test, - ) - from tune import load_best_params - from weather_loader import load_weather_dataset, merge_weather_features - - -MODELS_DIR = PROJECT_ROOT / "models" -FORECAST_OUTPUT_DIR = PROJECT_ROOT / "data" / "processed" -PROPHET_MODELS_DIR = MODELS_DIR / "prophet_models" -TARGET_SPECS = { - "co2": { - "target_column": "carbon_intensity_kg_per_kwh", - "model_filename": "co2_model.pkl", - "label": "CO2 intensity", - }, - "wue": { - "target_column": "WUE_total", - "model_filename": "wue_model.pkl", - "label": "WUE", - }, -} - - -def _require_optional_dependencies() -> None: - missing: list[str] = [] - if Prophet is None: - missing.append("prophet") - if XGBRegressor is None: - missing.append("xgboost") - if missing: - raise ImportError( - "Missing optional training dependencies: " - + ", ".join(missing) - + ". Install them with `pip install -r requirements.txt`." - ) - - -def _slugify(value: str) -> str: - slug = re.sub(r"[^a-z0-9]+", "_", value.lower()).strip("_") - return slug or "unknown_city" - - -def build_xgboost_regressor( - random_state: int = 42, - target_key: str | None = None, -) -> XGBRegressor: - """ - Build an XGBRegressor, using tuned hyperparameters if available. - - If models/best_params.json exists (written by src/tune.py), the tuned - params for target_key are loaded and override the defaults below. - If not, the hand-tuned defaults are used — training still works. - """ - _require_optional_dependencies() - - # Default hand-tuned params - params: dict = { - "n_estimators": 300, - "learning_rate": 0.05, - "max_depth": 6, - "subsample": 0.85, - "colsample_bytree": 0.85, - "min_child_weight": 1, - "reg_lambda": 1.0, - "reg_alpha": 0.0, - "gamma": 0.0, - } - - # Override with tuned params if available for this target - if target_key is not None: - tuned = load_best_params(target_key) - if tuned: - params.update(tuned) - - return XGBRegressor( - objective="reg:squarederror", - tree_method="hist", - n_jobs=4, - verbosity=0, - random_state=random_state, - **params, - ) - - -def build_prophet_model(has_yearly_pattern: bool) -> Prophet: - _require_optional_dependencies() - model = Prophet( - daily_seasonality=True, - weekly_seasonality=True, - yearly_seasonality=has_yearly_pattern, - seasonality_mode="additive", - uncertainty_samples=0, - ) - model.add_seasonality(name="monthly", period=30.5, fourier_order=5) - return model - - -def enrich_with_weather_if_available( - df: pd.DataFrame, - weather_path: str | None = None, -) -> tuple[pd.DataFrame, dict[str, Any] | None]: - if not weather_path: - return df, None - - weather_df = load_weather_dataset(weather_path) - return merge_weather_features(df, weather_df) - - -def fit_prophet_models( - train_frame: pd.DataFrame, - target_column: str, -) -> tuple[dict[str, Prophet], dict[str, float], float]: - models: dict[str, Prophet] = {} - city_medians = ( - train_frame.groupby("city")[target_column].median().dropna().to_dict() - ) - global_median = float(train_frame[target_column].median()) - - for city, city_frame in train_frame.groupby("city"): - prophet_frame = ( - city_frame[["timestamp", target_column]] - .rename(columns={"timestamp": "ds", target_column: "y"}) - .dropna() - .sort_values("ds") - ) - if len(prophet_frame) < 24: - continue - - span_days = ( - (prophet_frame["ds"].max() - prophet_frame["ds"].min()).total_seconds() - / 86400.0 - ) - model = build_prophet_model(has_yearly_pattern=span_days >= 365.0) - try: - model.fit(prophet_frame) - models[str(city)] = model - except Exception: - continue - - return models, {str(key): float(value) for key, value in city_medians.items()}, global_median - - -def predict_with_prophet( - frame: pd.DataFrame, - prophet_models: dict[str, Prophet], - city_fallbacks: dict[str, float], - global_fallback: float, -) -> pd.Series: - predictions = pd.Series(index=frame.index, dtype=float) - for city, city_frame in frame.groupby("city"): - model = prophet_models.get(str(city)) - if model is None: - continue - ds_frame = pd.DataFrame({"ds": pd.to_datetime(city_frame["timestamp"], errors="coerce")}) - forecast = model.predict(ds_frame) - predictions.loc[city_frame.index] = pd.to_numeric(forecast["yhat"], errors="coerce").values - - for city, city_frame in frame.groupby("city"): - fallback = city_fallbacks.get(str(city), global_fallback) - predictions.loc[city_frame.index] = predictions.loc[city_frame.index].fillna(fallback) - return predictions.fillna(global_fallback) - - -def _fit_xgboost_target( - features: pd.DataFrame, - target: pd.Series, - random_state: int, - target_key: str | None = None, -) -> XGBRegressor: - model = build_xgboost_regressor(random_state=random_state, target_key=target_key) - model.fit(features, target) - return model - - -def _serialize_profiles(profiles: FutureFeatureProfiles) -> dict[str, Any]: - return { - "hourly_profiles": profiles.hourly_profiles, - "city_profiles": profiles.city_profiles, - "global_profile": profiles.global_profile, - "profile_columns": profiles.profile_columns, - "profile_history_days": profiles.profile_history_days, - } - - -def save_prophet_model_group( - prophet_models: dict[str, Prophet], - target_key: str, - base_dir: Path = PROPHET_MODELS_DIR, -) -> Path: - target_dir = base_dir / target_key - target_dir.mkdir(parents=True, exist_ok=True) - for city, model in prophet_models.items(): - joblib.dump(model, target_dir / f"{_slugify(city)}.pkl") - return target_dir - - -def _train_single_target( - train_frame: pd.DataFrame, - test_frame: pd.DataFrame, - evaluation_feature_frame: pd.DataFrame, - final_history_frame: pd.DataFrame, - future_frame: pd.DataFrame, - target_key: str, - random_state: int, - save_models: bool, -) -> dict[str, Any]: - target_column = TARGET_SPECS[target_key]["target_column"] - feature_columns = feature_columns_for_frame(train_frame) - - X_train, encoded_columns = build_feature_matrix(train_frame, feature_columns=feature_columns) - X_test, _ = build_feature_matrix( - evaluation_feature_frame, - feature_columns=feature_columns, - encoded_columns=encoded_columns, - ) - - y_train = pd.to_numeric(train_frame[target_column], errors="coerce") - y_test = pd.to_numeric(test_frame[target_column], errors="coerce") - - prophet_models, city_fallbacks, global_fallback = fit_prophet_models(train_frame, target_column) - prophet_train_pred = predict_with_prophet(train_frame, prophet_models, city_fallbacks, global_fallback) - prophet_test_pred = predict_with_prophet(test_frame, prophet_models, city_fallbacks, global_fallback) - - direct_model = _fit_xgboost_target(X_train, y_train, random_state=random_state, target_key=target_key) - direct_test_pred = pd.Series(direct_model.predict(X_test), index=test_frame.index, dtype=float) - - residual_target = y_train - prophet_train_pred - residual_model = _fit_xgboost_target(X_train, residual_target, random_state=random_state, target_key=target_key) - hybrid_test_pred = prophet_test_pred + pd.Series( - residual_model.predict(X_test), - index=test_frame.index, - dtype=float, - ) - - metrics = { - "prophet": regression_metrics(y_test, prophet_test_pred), - "xgboost": regression_metrics(y_test, direct_test_pred), - "hybrid": regression_metrics(y_test, hybrid_test_pred), - } - comparisons = { - "xgboost_vs_prophet": compare_to_baseline(metrics["xgboost"], metrics["prophet"]), - "hybrid_vs_prophet": compare_to_baseline(metrics["hybrid"], metrics["prophet"]), - } - - final_feature_columns = feature_columns_for_frame(final_history_frame) - X_final, final_encoded_columns = build_feature_matrix( - final_history_frame, - feature_columns=final_feature_columns, - ) - y_final = pd.to_numeric(final_history_frame[target_column], errors="coerce") - - final_prophet_models, final_city_fallbacks, final_global_fallback = fit_prophet_models( - final_history_frame, - target_column, - ) - final_prophet_history = predict_with_prophet( - final_history_frame, - final_prophet_models, - final_city_fallbacks, - final_global_fallback, - ) - final_direct_model = _fit_xgboost_target(X_final, y_final, random_state=random_state, target_key=target_key) - final_residual_model = _fit_xgboost_target( - X_final, - y_final - final_prophet_history, - random_state=random_state, - target_key=target_key, - ) - - X_future, _ = build_feature_matrix( - future_frame, - feature_columns=final_feature_columns, - encoded_columns=final_encoded_columns, - ) - future_prophet_pred = predict_with_prophet( - future_frame, - final_prophet_models, - final_city_fallbacks, - final_global_fallback, - ) - future_direct_pred = pd.Series(final_direct_model.predict(X_future), index=future_frame.index, dtype=float) - future_hybrid_pred = future_prophet_pred + pd.Series( - final_residual_model.predict(X_future), - index=future_frame.index, - dtype=float, - ) - - model_path = MODELS_DIR / TARGET_SPECS[target_key]["model_filename"] - prophet_dir = PROPHET_MODELS_DIR / target_key - saved_model_path: str | None = None - saved_prophet_dir: str | None = None - if save_models: - MODELS_DIR.mkdir(parents=True, exist_ok=True) - prophet_dir.mkdir(parents=True, exist_ok=True) - profiles = build_future_feature_profiles(final_history_frame) - bundle = { - "target_key": target_key, - "target_column": target_column, - "direct_xgboost_model": final_direct_model, - "hybrid_residual_model": final_residual_model, - "feature_columns": final_feature_columns, - "encoded_columns": final_encoded_columns, - "city_fallbacks": final_city_fallbacks, - "global_fallback": final_global_fallback, - "future_feature_profiles": _serialize_profiles(profiles), - "last_training_timestamp": pd.Timestamp(final_history_frame["timestamp"].max()).isoformat(), - } - joblib.dump(bundle, model_path) - save_prophet_model_group(final_prophet_models, target_key, base_dir=PROPHET_MODELS_DIR) - saved_model_path = str(model_path) - saved_prophet_dir = str(prophet_dir) - - forecast_frame = future_frame[["timestamp", "city"]].copy() - forecast_frame[f"prophet_{target_key}"] = future_prophet_pred.values - forecast_frame[f"xgboost_{target_key}"] = future_direct_pred.values - forecast_frame[f"hybrid_{target_key}"] = future_hybrid_pred.values - - return { - "metrics": metrics, - "comparisons": comparisons, - "forecast": forecast_frame, - "saved_model_path": saved_model_path, - "saved_prophet_dir": saved_prophet_dir, - } - - -def train_forecasting_models( - data_path: str | None = None, - weather_path: str | None = None, - history_days: int = 365, - test_horizon_hours: int = 48, - forecast_horizon_hours: int = 48, - random_state: int = 42, - save_models: bool = True, - city_limit: int | None = None, -) -> dict[str, Any]: - """Train XGBoost, Prophet, and hybrid forecasting models, then emit city forecasts.""" - _require_optional_dependencies() - - raw_df = load_dataset(data_path) - enriched_df, weather_info = enrich_with_weather_if_available(raw_df, weather_path=weather_path) - forecasting_frame = prepare_forecasting_frame(enriched_df, city_limit=city_limit) - - split = split_time_train_test( - forecasting_frame, - test_horizon_hours=test_horizon_hours, - history_days=history_days, - ) - final_history = select_recent_history( - forecasting_frame, - history_days=history_days, - end_time=pd.Timestamp(forecasting_frame["timestamp"].max()), - ) - future_profiles = build_future_feature_profiles(final_history) - evaluation_feature_frame = build_profile_feature_frame( - split.test[["timestamp", "city"]], - profiles=build_future_feature_profiles(split.train), - ) - future_frame = build_future_feature_frame( - final_history, - horizon_hours=forecast_horizon_hours, - profiles=future_profiles, - ) - - target_results = { - target_key: _train_single_target( - train_frame=split.train, - test_frame=split.test, - evaluation_feature_frame=evaluation_feature_frame, - final_history_frame=final_history, - future_frame=future_frame, - target_key=target_key, - random_state=random_state, - save_models=save_models, - ) - for target_key in TARGET_SPECS - } - - forecasts = ( - target_results["co2"]["forecast"] - .merge(target_results["wue"]["forecast"], on=["timestamp", "city"], how="inner") - .sort_values(["timestamp", "city"]) - .reset_index(drop=True) - ) - - FORECAST_OUTPUT_DIR.mkdir(parents=True, exist_ok=True) - forecast_output_path = FORECAST_OUTPUT_DIR / f"city_forecasts_{int(forecast_horizon_hours)}h.csv" - forecasts.to_csv(forecast_output_path, index=False) - - return { - "weather": weather_info, - "history_days": int(history_days), - "test_horizon_hours": int(test_horizon_hours), - "forecast_horizon_hours": int(forecast_horizon_hours), - "split": { - "train_start": pd.Timestamp(split.train["timestamp"].min()).isoformat(), - "train_end": pd.Timestamp(split.train["timestamp"].max()).isoformat(), - "test_start": pd.Timestamp(split.test_start).isoformat(), - "test_end": pd.Timestamp(split.test_end).isoformat(), - }, - "targets": { - target_key: { - "metrics": result["metrics"], - "comparisons": result["comparisons"], - "saved_model_path": result["saved_model_path"], - "saved_prophet_dir": result["saved_prophet_dir"], - } - for target_key, result in target_results.items() - }, - "forecast_path": str(forecast_output_path), - "forecast_preview": forecasts.head(10), - } - - -def train_models( - data_path: str | None = None, - test_size: float = 0.2, - random_state: int = 42, - save_models: bool = True, -) -> dict[str, Any]: - """Backward-compatible wrapper that now delegates to the forecasting pipeline.""" - test_horizon_hours = max(24, int(round(test_size * 24 * 10))) - return train_forecasting_models( - data_path=data_path, - history_days=365, - test_horizon_hours=test_horizon_hours, - forecast_horizon_hours=48, - random_state=random_state, - save_models=save_models, - ) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Train XGBoost, Prophet, and hybrid forecasting models for CO2 and WUE." - ) - parser.add_argument( - "--data-path", - default=None, - help="Path to the master dataset (.zip, .parquet, .csv, or .xlsx).", - ) - parser.add_argument( - "--weather-path", - default=None, - help="Optional path to external weather data for feature enrichment.", - ) - parser.add_argument( - "--history-days", - type=int, - default=365, - help="How many trailing historical days to use for training.", - ) - parser.add_argument( - "--test-horizon-hours", - type=int, - default=48, - help="How many final hours to reserve for evaluation.", - ) - parser.add_argument( - "--forecast-horizon-hours", - type=int, - default=48, - help="How many future hours to forecast per city.", - ) - parser.add_argument( - "--random-state", - type=int, - default=42, - help="Random seed for XGBoost reproducibility.", - ) - parser.add_argument( - "--city-limit", - type=int, - default=None, - help="Optional number of cities to train for faster smoke tests.", - ) - parser.add_argument( - "--no-save-models", - action="store_true", - help="Disable saving trained model bundles and Prophet artifacts.", - ) - return parser.parse_args() - - -def main() -> None: - args = parse_args() - results = train_forecasting_models( - data_path=args.data_path, - weather_path=args.weather_path, - history_days=args.history_days, - test_horizon_hours=args.test_horizon_hours, - forecast_horizon_hours=args.forecast_horizon_hours, - random_state=args.random_state, - save_models=not args.no_save_models, - city_limit=args.city_limit, - ) - - print("Forecast training complete") - print( - f" Train window: {results['split']['train_start']} to {results['split']['train_end']}" - ) - print( - f" Test window: {results['split']['test_start']} to {results['split']['test_end']}" - ) - for target_key, target_result in results["targets"].items(): - print(f"\n{TARGET_SPECS[target_key]['label']} models") - for model_name, metrics in target_result["metrics"].items(): - print( - f" {model_name:>7} -> RMSE {metrics['rmse']:.4f}, " - f"MAE {metrics['mae']:.4f}, R2 {metrics['r2']:.4f}" - ) - print( - " Hybrid vs Prophet RMSE improvement: " - f"{target_result['comparisons']['hybrid_vs_prophet']['rmse_improvement_pct']:.2f}%" - ) - if not args.no_save_models: - print(f" Saved model bundle: {target_result['saved_model_path']}") - print(f" Saved Prophet models: {target_result['saved_prophet_dir']}") - print(f"\nSaved forecasts to: {results['forecast_path']}") - - -if __name__ == "__main__": - main() diff --git a/src/tune.py b/src/tune.py deleted file mode 100644 index de005b4..0000000 --- a/src/tune.py +++ /dev/null @@ -1,433 +0,0 @@ -""" -Hyperparameter tuning for XGBoost CO2 and WUE models using Optuna. - -Why Optuna? - Optuna uses Tree-structured Parzen Estimators (TPE) — a Bayesian method - that learns which regions of the parameter space are promising from previous - trials, rather than sampling blindly like RandomizedSearchCV. - -Why not tune Prophet? - Prophet has very few meaningful knobs and its seasonality parameters are - already set by data characteristics (span_days >= 365 for yearly). - XGBoost is where the real gains are. - -How to run: - python3 -m src.tune # tune both targets, 40 trials each - python3 -m src.tune --target co2 --trials 60 - python3 -m src.tune --target wue --trials 20 --history-days 180 - -Best params are saved to models/best_params.json. -train.py automatically loads them if the file exists. -""" -from __future__ import annotations - -import argparse -import json -import logging -from pathlib import Path -from typing import Any - -import numpy as np -import pandas as pd - -# optuna and xgboost are imported lazily inside tune_target() so that -# load_best_params() — which only reads a JSON file — can be called from -# train.py without requiring optuna to be installed. -try: - from xgboost import XGBRegressor -except ImportError: - XGBRegressor = None # type: ignore[assignment,misc] - -from .data_loader import PROJECT_ROOT, load_dataset -from .preprocessing import ( - build_feature_matrix, - feature_columns_for_frame, - prepare_forecasting_frame, - select_recent_history, - split_time_train_test, -) - -MODELS_DIR = PROJECT_ROOT / "models" -BEST_PARAMS_PATH = MODELS_DIR / "best_params.json" - -TARGET_COLUMNS = { - "co2": "carbon_intensity_kg_per_kwh", - "wue": "WUE_total", -} - -logging.basicConfig(level=logging.INFO, format="%(message)s") -log = logging.getLogger(__name__) - - -# --------------------------------------------------------------------------- -# Parameter space -# --------------------------------------------------------------------------- - -def _suggest_params(trial: Any) -> dict[str, Any]: # trial: optuna.Trial - """ - Define the XGBoost hyperparameter search space. - - Each call to trial.suggest_* picks one value from the given range. - Optuna learns across trials which values tend to produce lower RMSE. - - Key parameters and what they control: - n_estimators — how many trees; more trees = more capacity but slower - learning_rate — how much each tree corrects the previous ones - (lower = needs more trees but generalises better) - max_depth — how complex each individual tree is - (higher = more detail but risks overfitting) - subsample — fraction of rows sampled per tree (reduces overfitting) - colsample_bytree — fraction of features sampled per tree - min_child_weight — minimum data points required to split a leaf - (higher = more conservative, less overfit) - reg_lambda — L2 regularisation on leaf weights - reg_alpha — L1 regularisation (sparse feature selection) - gamma — minimum loss reduction required to make a split - """ - return { - "n_estimators": trial.suggest_int("n_estimators", 100, 800), - "learning_rate": trial.suggest_float("learning_rate", 0.01, 0.3, log=True), - "max_depth": trial.suggest_int("max_depth", 3, 10), - "subsample": trial.suggest_float("subsample", 0.6, 1.0), - "colsample_bytree": trial.suggest_float("colsample_bytree", 0.5, 1.0), - "min_child_weight": trial.suggest_int("min_child_weight", 1, 10), - "reg_lambda": trial.suggest_float("reg_lambda", 1e-3, 10.0, log=True), - "reg_alpha": trial.suggest_float("reg_alpha", 1e-4, 1.0, log=True), - "gamma": trial.suggest_float("gamma", 0.0, 0.5), - } - - -# --------------------------------------------------------------------------- -# Time-series cross-validation -# --------------------------------------------------------------------------- - -def _time_series_cv_rmse( - X: pd.DataFrame, - y: pd.Series, - timestamps: pd.Series, - params: dict[str, Any], - n_splits: int = 3, - random_state: int = 42, -) -> float: - """ - Walk-forward cross-validation: always train on past, validate on future. - - Why not random k-fold? - Random k-fold shuffles data, letting the model see future observations - during training. For time series this is data leakage — the resulting - RMSE is optimistic and won't reflect real-world performance. - - Walk-forward splits the timeline into n_splits equal windows: - Split 1: train=[t0..t1] val=[t1..t2] - Split 2: train=[t0..t2] val=[t2..t3] (expanding window) - Split 3: train=[t0..t3] val=[t3..t4] - - Returns the mean RMSE across all validation windows. - """ - sorted_times = sorted(timestamps.unique()) - n = len(sorted_times) - fold_size = n // (n_splits + 1) - - rmse_scores: list[float] = [] - - for split_idx in range(1, n_splits + 1): - val_start = sorted_times[split_idx * fold_size] - val_end = ( - sorted_times[min((split_idx + 1) * fold_size, n) - 1] - if split_idx < n_splits - else sorted_times[-1] - ) - - train_mask = timestamps < val_start - val_mask = (timestamps >= val_start) & (timestamps <= val_end) - - if train_mask.sum() < 10 or val_mask.sum() < 5: - continue - - X_tr, y_tr = X[train_mask], y[train_mask] - X_val, y_val = X[val_mask], y[val_mask] - - model = XGBRegressor( - objective="reg:squarederror", - tree_method="hist", - n_jobs=4, - verbosity=0, - random_state=random_state, - early_stopping_rounds=30, - **params, - ) - model.fit( - X_tr, y_tr, - eval_set=[(X_val, y_val)], - verbose=False, - ) - - preds = model.predict(X_val) - rmse = float(np.sqrt(np.mean((y_val.values - preds) ** 2))) - rmse_scores.append(rmse) - - return float(np.mean(rmse_scores)) if rmse_scores else float("inf") - - -# --------------------------------------------------------------------------- -# Objective function -# --------------------------------------------------------------------------- - -def _make_objective( - X: pd.DataFrame, - y: pd.Series, - timestamps: pd.Series, - n_splits: int, - random_state: int, -): - """ - Returns a closure Optuna calls for each trial. - - A closure captures X, y, timestamps from the outer scope so Optuna - only needs to pass the trial object — keeping the interface clean. - """ - def objective(trial: Any) -> float: - params = _suggest_params(trial) - return _time_series_cv_rmse( - X, y, timestamps, params, - n_splits=n_splits, - random_state=random_state, - ) - return objective - - -# --------------------------------------------------------------------------- -# Main tuning entry point -# --------------------------------------------------------------------------- - -def tune_target( - target: str, - data_path: str | None = None, - n_trials: int = 40, - history_days: int = 365, - n_splits: int = 3, - random_state: int = 42, - timeout_seconds: int | None = None, -) -> dict[str, Any]: - """ - Run Optuna hyperparameter search for one target (co2 or wue). - - Parameters - ---------- - target : 'co2' or 'wue' - data_path : path to dataset (uses default if None) - n_trials : how many Optuna trials to run - history_days : how many days of history to use for tuning - n_splits : number of walk-forward CV folds - random_state : reproducibility seed - timeout_seconds: stop after this many seconds (None = no limit) - - Returns - ------- - dict with best_params, best_rmse, n_trials_completed - """ - if target not in TARGET_COLUMNS: - raise ValueError(f"target must be 'co2' or 'wue', got: {target!r}") - - # Lazy import — only needed when actually running tuning, not when loading best params - import optuna as _optuna - _optuna.logging.set_verbosity(_optuna.logging.WARNING) - - if XGBRegressor is None: - raise ImportError("xgboost is required for tuning.") - - target_column = TARGET_COLUMNS[target] - log.info(f"\nTuning {target} model ({n_trials} trials, {n_splits}-fold walk-forward CV)") - - # Load and prepare data - raw_df = load_dataset(data_path) - frame = prepare_forecasting_frame(raw_df) - history = select_recent_history( - frame, - history_days=history_days, - end_time=pd.Timestamp(frame["timestamp"].max()), - ) - - feature_columns = feature_columns_for_frame(history) - X, _ = build_feature_matrix(history, feature_columns=feature_columns) - y = pd.to_numeric(history[target_column], errors="coerce").fillna(0.0) - timestamps = history["timestamp"].reset_index(drop=True) - X = X.reset_index(drop=True) - y = y.reset_index(drop=True) - - # Run Optuna study (minimise RMSE) - study = _optuna.create_study( - direction="minimize", - sampler=_optuna.samplers.TPESampler(seed=random_state), - pruner=_optuna.pruners.MedianPruner(n_startup_trials=5, n_warmup_steps=1), - ) - study.optimize( - _make_objective(X, y, timestamps, n_splits=n_splits, random_state=random_state), - n_trials=n_trials, - timeout=timeout_seconds, - show_progress_bar=True, - ) - - best = study.best_trial - log.info(f" Best RMSE: {best.value:.6f}") - log.info(f" Best params: {best.params}") - - return { - "target": target, - "best_rmse": float(best.value), - "best_params": best.params, - "n_trials_completed": len(study.trials), - "baseline_params": { - "n_estimators": 300, - "learning_rate": 0.05, - "max_depth": 6, - "subsample": 0.85, - "colsample_bytree": 0.85, - "min_child_weight": 1, - "reg_lambda": 1.0, - "reg_alpha": 0.0, - "gamma": 0.0, - }, - } - - -def tune_all_targets( - data_path: str | None = None, - n_trials: int = 40, - history_days: int = 365, - n_splits: int = 3, - random_state: int = 42, - timeout_seconds: int | None = None, - save: bool = True, -) -> dict[str, Any]: - """ - Tune both CO2 and WUE models, save best params to models/best_params.json. - train.py automatically loads this file if it exists. - """ - results: dict[str, Any] = {} - - for target in ("co2", "wue"): - result = tune_target( - target=target, - data_path=data_path, - n_trials=n_trials, - history_days=history_days, - n_splits=n_splits, - random_state=random_state, - timeout_seconds=timeout_seconds, - ) - results[target] = result - - if save: - MODELS_DIR.mkdir(parents=True, exist_ok=True) - output = { - target: { - "best_params": res["best_params"], - "best_rmse": res["best_rmse"], - "n_trials": res["n_trials_completed"], - } - for target, res in results.items() - } - BEST_PARAMS_PATH.write_text(json.dumps(output, indent=2)) - log.info(f"\nSaved best params to: {BEST_PARAMS_PATH}") - - return results - - -def load_best_params(target: str) -> dict[str, Any] | None: - """ - Load saved best params for a target, or return None if not found. - Called by train.py to use tuned params when available. - """ - if not BEST_PARAMS_PATH.exists(): - return None - data = json.loads(BEST_PARAMS_PATH.read_text()) - return data.get(target, {}).get("best_params") - - -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser( - description="Tune XGBoost hyperparameters with Optuna." - ) - parser.add_argument( - "--target", - choices=["co2", "wue", "both"], - default="both", - help="Which model to tune (default: both)", - ) - parser.add_argument( - "--trials", type=int, default=40, - help="Number of Optuna trials per target (default: 40)", - ) - parser.add_argument( - "--history-days", type=int, default=365, - help="Days of history to use for tuning (default: 365)", - ) - parser.add_argument( - "--splits", type=int, default=3, - help="Number of walk-forward CV folds (default: 3)", - ) - parser.add_argument( - "--timeout", type=int, default=None, - help="Stop each target after this many seconds (default: no limit)", - ) - parser.add_argument( - "--data-path", default=None, - help="Path to dataset (uses project default if omitted)", - ) - parser.add_argument( - "--no-save", action="store_true", - help="Don't save best params to models/best_params.json", - ) - return parser.parse_args() - - -def main() -> None: - args = parse_args() - targets = ["co2", "wue"] if args.target == "both" else [args.target] - - all_results: dict[str, Any] = {} - for target in targets: - result = tune_target( - target=target, - data_path=args.data_path, - n_trials=args.trials, - history_days=args.history_days, - n_splits=args.splits, - random_state=42, - timeout_seconds=args.timeout, - ) - all_results[target] = result - - if not args.no_save: - MODELS_DIR.mkdir(parents=True, exist_ok=True) - output = { - t: { - "best_params": r["best_params"], - "best_rmse": r["best_rmse"], - "n_trials": r["n_trials_completed"], - } - for t, r in all_results.items() - } - BEST_PARAMS_PATH.write_text(json.dumps(output, indent=2)) - log.info(f"\nSaved: {BEST_PARAMS_PATH}") - - log.info("\n=== Tuning Summary ===") - for target, result in all_results.items(): - baseline_rmse_approx = "—" - log.info( - f" {target.upper():3s} best RMSE: {result['best_rmse']:.6f}" - f" ({result['n_trials_completed']} trials)" - ) - log.info(f" Best params: {json.dumps(result['best_params'], indent=4)}") - - log.info("\nRe-train with tuned params:") - log.info(" python3 -m src.train") - - -if __name__ == "__main__": - main() diff --git a/src/weather_loader.py b/src/weather_loader.py deleted file mode 100644 index fa2545b..0000000 --- a/src/weather_loader.py +++ /dev/null @@ -1,324 +0,0 @@ -from __future__ import annotations - -from pathlib import Path -from typing import Optional - -import pandas as pd - -try: - from .data_loader import PROJECT_ROOT -except ImportError: - from data_loader import PROJECT_ROOT - - -DEFAULT_WEATHER_FILENAMES = [ - "meteostat_weather_master_2019_2023.csv", - "meteostat_weather_master.csv", -] - -WEATHER_VALUE_COLUMNS = [ - "temp", - "dwpt", - "rhum", - "prcp", - "snow", - "wdir", - "wspd", - "wpgt", - "pres", - "tsun", - "coco", -] - -TIMESTAMP_COLUMN_CANDIDATES = ["TIMESTAMP", "Timestamp", "timestamp", "datetime"] -ZIP_COLUMN_CANDIDATES = ["ZIP", "zip", "Zip", "zip_code", "zipcode", "zip code"] -CITY_STATE_COLUMN_CANDIDATES = ["CityST", "city_state", "city_st", "CITYST"] -CITY_COLUMN_CANDIDATES = ["city", "City", "CITY"] - - -def _resolve_path(path: str | Path) -> Path: - candidate = Path(path) - if candidate.is_absolute(): - return candidate - return PROJECT_ROOT / candidate - - -def resolve_weather_path(path: str | Path) -> Path: - """Resolve a weather dataset path from a CSV file or directory.""" - candidate = _resolve_path(path) - if not candidate.exists(): - raise FileNotFoundError(f"Weather dataset path does not exist: {candidate}") - - if candidate.is_file(): - return candidate - - for filename in DEFAULT_WEATHER_FILENAMES: - nested = candidate / filename - if nested.exists(): - return nested - - raise FileNotFoundError( - "Weather dataset directory does not contain a supported master CSV. " - f"Expected one of: {', '.join(DEFAULT_WEATHER_FILENAMES)}" - ) - - -def _normalize_timestamp_hour(series: pd.Series) -> pd.Series: - """Convert timestamps to local wall-clock hourly resolution.""" - cleaned = ( - series.astype(str) - .str.strip() - .str.replace(r"([+-]\d{2}:\d{2})$", "", regex=True) - .str.replace("Z", "", regex=False) - ) - return pd.to_datetime(cleaned, errors="coerce").dt.floor("h") - - -def _normalize_zip(series: pd.Series) -> pd.Series: - values = series.astype(str).str.extract(r"(\d+)")[0] - return values.str.zfill(5) - - -def _normalize_city_state(series: pd.Series) -> pd.Series: - return series.astype(str).str.strip().str.lower() - - -def _normalize_city(series: pd.Series) -> pd.Series: - return ( - series.astype(str) - .str.split(",") - .str[0] - .str.strip() - .str.lower() - ) - - -def _find_first_column(df: pd.DataFrame, candidates: list[str]) -> str | None: - for column in candidates: - if column in df.columns: - return column - return None - - -def load_weather_dataset(path: str | Path) -> pd.DataFrame: - """Load the Meteostat weather dataset and compute prefixed features.""" - weather_path = resolve_weather_path(path) - usecols = ["Timestamp", "LocationID", "LocationType", "CityST"] + WEATHER_VALUE_COLUMNS - weather_df = pd.read_csv(weather_path, usecols=lambda col: col in usecols, low_memory=False) - - weather_df["weather_timestamp_hour"] = _normalize_timestamp_hour(weather_df["Timestamp"]) - weather_df["weather_zip"] = _normalize_zip(weather_df["LocationID"]) - weather_df["weather_city_state"] = _normalize_city_state(weather_df["CityST"]) - weather_df["weather_city"] = _normalize_city(weather_df["CityST"]) - - for column in WEATHER_VALUE_COLUMNS: - if column in weather_df.columns: - weather_df[f"weather_{column}"] = pd.to_numeric( - weather_df[column], errors="coerce" - ) - - weather_df["weather_temp_dewpoint_gap"] = ( - weather_df["weather_temp"] - weather_df["weather_dwpt"] - ) - weather_df["weather_rain_flag"] = ( - weather_df["weather_prcp"].fillna(0).gt(0).astype(int) - ) - weather_df["weather_humidity_temp_index"] = ( - weather_df["weather_temp"] * weather_df["weather_rhum"] / 100.0 - ) - - feature_columns = [ - "weather_timestamp_hour", - "weather_zip", - "weather_city_state", - "weather_city", - "weather_temp", - "weather_dwpt", - "weather_rhum", - "weather_prcp", - "weather_wdir", - "weather_wspd", - "weather_pres", - "weather_coco", - "weather_temp_dewpoint_gap", - "weather_rain_flag", - "weather_humidity_temp_index", - ] - return weather_df[feature_columns] - - -def _prepare_main_merge_keys(df: pd.DataFrame) -> tuple[pd.DataFrame, str]: - timestamp_column = _find_first_column(df, TIMESTAMP_COLUMN_CANDIDATES) - if timestamp_column is None: - raise ValueError( - "Main dataset must contain a timestamp column such as TIMESTAMP or Timestamp." - ) - - prepared = df.copy() - prepared["weather_timestamp_hour"] = _normalize_timestamp_hour(prepared[timestamp_column]) - - zip_column = _find_first_column(prepared, ZIP_COLUMN_CANDIDATES) - if zip_column is not None: - prepared["weather_join_location"] = _normalize_zip(prepared[zip_column]) - return prepared, "zip" - - city_state_column = _find_first_column(prepared, CITY_STATE_COLUMN_CANDIDATES) - if city_state_column is not None: - prepared["weather_join_location"] = _normalize_city_state( - prepared[city_state_column] - ) - return prepared, "city_state" - - city_column = _find_first_column(prepared, CITY_COLUMN_CANDIDATES) - if city_column is not None: - prepared["weather_join_location"] = _normalize_city(prepared[city_column]) - return prepared, "city" - - raise ValueError( - "Main dataset must contain a location column such as ZIP, CityST, or city to merge weather data." - ) - - -def merge_weather_features( - df: pd.DataFrame, weather_df: pd.DataFrame -) -> tuple[pd.DataFrame, dict[str, float | int | str]]: - """Merge weather features into the main dataset using timestamp and location.""" - prepared, join_strategy = _prepare_main_merge_keys(df) - - if join_strategy == "zip": - weather_key = "weather_zip" - elif join_strategy == "city_state": - weather_key = "weather_city_state" - else: - weather_key = "weather_city" - - merged = prepared.merge( - weather_df, - how="left", - left_on=["weather_timestamp_hour", "weather_join_location"], - right_on=["weather_timestamp_hour", weather_key], - ) - - matched_rows = int(merged["weather_temp"].notna().sum()) - total_rows = int(len(merged)) - coverage = float(matched_rows / total_rows) if total_rows else 0.0 - - merged.drop( - columns=["weather_join_location", "weather_zip", "weather_city_state", "weather_city"], - inplace=True, - errors="ignore", - ) - - return merged, { - "join_strategy": join_strategy, - "matched_rows": matched_rows, - "total_rows": total_rows, - "coverage": coverage, - } - - -# ── Live pipeline data source ───────────────────────────────────────────────── - -PIPELINE_DB_URL_DEFAULT = ( - "postgresql+psycopg2://weather_user:weather_password@localhost:15432/weather_data" -) - -# Maps pipeline weather_records columns -> capstone weather_* feature names. -# Pipeline uses metric units (temp in C, speed in m/s, pressure in hPa). -# Meteostat-based features use the same units, so no conversion needed. -_PIPELINE_COLUMN_MAP = { - "city": "weather_city", - "temperature": "weather_temp", - "humidity": "weather_rhum", - "pressure": "weather_pres", - "wind_speed": "weather_wspd", - "wind_direction": "weather_wdir", - "timestamp": "weather_timestamp_hour", -} - - -def load_from_pipeline_db( - hours: int = 48, - db_url: Optional[str] = None, -) -> pd.DataFrame: - """ - Load live weather data from the weather pipeline's PostgreSQL database. - - This is an alternative to load_weather_dataset() for when you want - real-time hourly data instead of the historical Meteostat CSV. - The pipeline collects weather every hour for the same 28 data center - locations used in the capstone, so the city names match directly. - - Args: - hours: How many hours of history to fetch (default 48). - db_url: SQLAlchemy connection string. Defaults to the pipeline's - local PostgreSQL on port 15432. - - Returns: - DataFrame with the same weather_* prefixed columns produced by - load_weather_dataset(), ready for merge_weather_features(). - """ - try: - from sqlalchemy import create_engine, text - except ImportError as exc: - raise ImportError("sqlalchemy is required: pip install sqlalchemy psycopg2-binary") from exc - - url = db_url or PIPELINE_DB_URL_DEFAULT - engine = create_engine(url) - - query = text( - f""" - SELECT - city, - timestamp, - temperature, - humidity, - pressure, - wind_speed, - wind_direction - FROM weather_records - WHERE timestamp >= NOW() - INTERVAL '{int(hours)} hours' - ORDER BY timestamp DESC - """ - ) - - with engine.connect() as conn: - df = pd.read_sql(query, conn) - - if df.empty: - raise ValueError( - f"No pipeline weather data found in the last {hours} hours. " - "Run the pipeline first: python -m src.main" - ) - - # Rename to capstone weather_* convention - df = df.rename(columns=_PIPELINE_COLUMN_MAP) - - # Floor to hour so timestamps align with capstone merge keys - df["weather_timestamp_hour"] = pd.to_datetime( - df["weather_timestamp_hour"] - ).dt.floor("h") - - # Lowercase city for case-insensitive join - df["weather_city"] = df["weather_city"].str.strip().str.lower() - - # Derived features that match what load_weather_dataset() produces - df["weather_temp_dewpoint_gap"] = None # dwpt not in pipeline - fill as needed - df["weather_rain_flag"] = 0 # no precip data in pipeline - df["weather_humidity_temp_index"] = ( - df["weather_temp"] * df["weather_rhum"] / 100.0 - ) - - feature_columns = [ - "weather_timestamp_hour", - "weather_city", - "weather_temp", - "weather_rhum", - "weather_pres", - "weather_wspd", - "weather_wdir", - "weather_temp_dewpoint_gap", - "weather_rain_flag", - "weather_humidity_temp_index", - ] - return df[[c for c in feature_columns if c in df.columns]] diff --git a/src/workload_loader.py b/src/workload_loader.py index 9f2deeb..87cf56d 100644 --- a/src/workload_loader.py +++ b/src/workload_loader.py @@ -13,7 +13,13 @@ from data_loader import PROJECT_ROOT -DEFAULT_AZURE_TRACE_PATH = PROJECT_ROOT / "data" / "external" / "azure_packing" / "packing_trace_zone_a_v1.sqlite" +DEFAULT_AZURE_TRACE_PATH = ( + PROJECT_ROOT + / "data" + / "external" + / "azure_packing" + / "packing_trace_zone_a_v1.sqlite" +) DEFAULT_CITY_POOL = [ "Ashburn", "Atlanta", @@ -126,7 +132,9 @@ def build_jobs_from_azure_trace( jobs["origin_city"] = _map_city_from_tenant(jobs["tenantId"], city_pool) # Power demand is not provided directly, so we proxy it with normalized CPU. - jobs["power_demand"] = (jobs["core"].fillna(0.05) * power_scale).round(2) + jobs["power_demand_kw"] = ( + jobs["core"].fillna(0.05) * power_scale + ).round(2) jobs["deadline"] = jobs["earliest_start"] + pd.to_timedelta( jobs["duration_hours"] + slack_hours, unit="h" @@ -136,7 +144,7 @@ def build_jobs_from_azure_trace( selected_columns = [ "job_id", "origin_city", - "power_demand", + "power_demand_kw", "duration_hours", "earliest_start", "deadline", @@ -194,7 +202,10 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--start-datetime", default="2019-01-01 00:00:00", - help="Anchor datetime for mapping relative Azure trace times into hourly scheduler timestamps.", + help=( + "Anchor datetime for mapping relative Azure trace times into hourly " + "scheduler timestamps." + ), ) return parser.parse_args() diff --git a/tests/conftest.py b/tests/conftest.py index 0ae7ef5..98a80bf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,119 +1,74 @@ -""" -Shared pytest fixtures. - -Why a conftest.py? - pytest auto-discovers this file and makes all fixtures here available - to every test in the directory — no import needed. -""" from __future__ import annotations import pandas as pd import pytest -# --------------------------------------------------------------------------- -# Minimal in-memory DataFrames (no file I/O, fast) -# --------------------------------------------------------------------------- - -@pytest.fixture -def sample_jobs_df() -> pd.DataFrame: - """Three jobs with the minimum columns the scheduler expects.""" - now = pd.Timestamp("2024-06-01 12:00") - return pd.DataFrame([ - { - "job_id": "job_a", - "origin_city": "Dallas", - "power_demand": 10.0, - "duration_hours": 2, - "earliest_start": now, - "deadline": now + pd.Timedelta(hours=6), - "priority": 0, - "priority_label": "high", - }, - { - "job_id": "job_b", - "origin_city": "Dallas", - "power_demand": 5.0, - "duration_hours": 1, - "earliest_start": now, - "deadline": now + pd.Timedelta(hours=4), - "priority": 1, - "priority_label": "medium", - }, - { - "job_id": "job_c", - "origin_city": "Atlanta", - "power_demand": 20.0, - "duration_hours": 3, - "earliest_start": now, - "deadline": now + pd.Timedelta(hours=8), - "priority": 2, - "priority_label": "low", - }, - ]) - - @pytest.fixture -def sample_dc_df() -> pd.DataFrame: - """Two data centers with the minimum columns the scheduler expects.""" - return pd.DataFrame([ - { - "dc_id": "dc_dallas_1", - "city": "Dallas", - "max_power_per_hour": 500.0, - "pue": 1.2, - "water_usage_factor": 1.0, - "region_water_scarcity": 0.5, - "is_new": 0, - }, - { - "dc_id": "dc_atlanta_1", - "city": "Atlanta", - "max_power_per_hour": 500.0, - "pue": 1.1, - "water_usage_factor": 0.9, - "region_water_scarcity": 0.3, - "is_new": 0, - }, - ]) +def environment() -> pd.DataFrame: + timestamps = pd.date_range("2025-01-01", periods=12, freq="h") + rows = [] + for city, carbon, water in ( + ("Alpha", 0.20, 1.00), + ("Beta", 0.50, 0.30), + ): + for index, timestamp in enumerate(timestamps): + rows.append( + { + "timestamp": timestamp, + "city": city, + "carbon_intensity_kg_per_kwh": carbon + index * 0.01, + "water_intensity_l_per_kwh": water, + } + ) + return pd.DataFrame(rows) @pytest.fixture -def sample_env_df() -> pd.DataFrame: - """Environment data (CO2, WUE, water scarcity) for two cities over 72 hours.""" - timestamps = pd.date_range("2024-06-01", periods=72, freq="h") - data = [] - for i, ts in enumerate(timestamps): - # Dallas: varies between 0.3 and 0.5 kg CO2/kWh - dallas_co2 = 0.3 + 0.1 * (i % 24) / 24 - # Atlanta: varies between 0.2 and 0.4 kg CO2/kWh (generally cleaner) - atlanta_co2 = 0.2 + 0.1 * (i % 24) / 24 - data.extend([ +def dc_config() -> pd.DataFrame: + return pd.DataFrame( + [ { - "TIMESTAMP": ts, - "CITY": "Dallas", - "co2_kg": dallas_co2 * 1000, # Assuming 1000 kWh for this hour - "total_gen_kwh": 1000.0, - "WUE_total": 1.5, # Water Usage Efficiency - "dsci_0_500": 300.0, # Water scarcity index + "dc_id": "dc_alpha", + "city": "Alpha", + "max_power_kw": 100.0, + "pue": 1.10, + "water_usage_factor": 1.0, + "water_scarcity_index": 0.10, }, { - "TIMESTAMP": ts, - "CITY": "Atlanta", - "co2_kg": atlanta_co2 * 1000, - "total_gen_kwh": 1000.0, - "WUE_total": 1.2, - "dsci_0_500": 150.0, + "dc_id": "dc_beta", + "city": "Beta", + "max_power_kw": 100.0, + "pue": 1.20, + "water_usage_factor": 1.0, + "water_scarcity_index": 0.80, }, - ]) - return pd.DataFrame(data) + ] + ) @pytest.fixture -def sample_schedule_df() -> pd.DataFrame: - """A small schedule result DataFrame mimicking scheduler output.""" - return pd.DataFrame([ - {"job_id": "job_a", "scheduled": 1, "expected_co2_kg": 2.0, "expected_water_liters": 10.0}, - {"job_id": "job_b", "scheduled": 1, "expected_co2_kg": 1.0, "expected_water_liters": 5.0}, - {"job_id": "job_c", "scheduled": 0, "expected_co2_kg": 0.0, "expected_water_liters": 0.0}, - ]) +def jobs() -> pd.DataFrame: + return pd.DataFrame( + [ + { + "job_id": "job_1", + "origin_city": "Alpha", + "power_demand_kw": 20.0, + "duration_hours": 2, + "earliest_start": "2025-01-01 00:00:00+00:00", + "deadline": "2025-01-01 05:00:00+00:00", + "priority": 1, + }, + { + "job_id": "job_2", + "origin_city": "Beta", + "power_demand_kw": 30.0, + "duration_hours": 3, + "earliest_start": "2025-01-01 01:00:00+00:00", + "deadline": "2025-01-01 08:00:00+00:00", + "priority": 1, + }, + ] + ) diff --git a/tests/test_api.py b/tests/test_api.py deleted file mode 100644 index e68905b..0000000 --- a/tests/test_api.py +++ /dev/null @@ -1,164 +0,0 @@ -""" -API-level tests using FastAPI's TestClient. - -TestClient wraps the ASGI app and makes real HTTP requests without starting -a server process. It's synchronous, fast, and great for CI. - -Why test the API layer separately from the scheduler? - The API converts HTTP request shapes into scheduler inputs and serializes - the output. Bugs can exist here even when the scheduler itself is correct - (e.g., missing fields, wrong status codes, validation rejects valid input). -""" -from __future__ import annotations - -import pytest -from fastapi.testclient import TestClient - -from src.api import app, get_backend - -# Clear lru_cache before each test so state doesn't leak between tests. -# lru_cache is module-level, so without clearing it the same backend -# instance is reused across the entire test session. -@pytest.fixture(autouse=True) -def clear_backend_cache(): - get_backend.cache_clear() - yield - get_backend.cache_clear() - - -@pytest.fixture -def client() -> TestClient: - return TestClient(app) - - -# --------------------------------------------------------------------------- -# /health -# --------------------------------------------------------------------------- - -@pytest.mark.api -class TestHealthEndpoint: - def test_returns_200(self, client): - response = client.get("/health") - assert response.status_code == 200 - - def test_returns_ok_status(self, client): - data = client.get("/health").json() - assert data == {"status": "ok"} - - -# --------------------------------------------------------------------------- -# /context -# --------------------------------------------------------------------------- - -@pytest.mark.api -class TestContextEndpoint: - def test_returns_200(self, client): - response = client.get("/context") - assert response.status_code == 200 - - def test_required_fields_present(self, client): - data = client.get("/context").json() - for field in [ - "origin_city", "time_min", "time_max", "default_time", - "default_workload_size", "default_batch_size", - "batch_size_min", "batch_size_max", - ]: - assert field in data, f"Missing field: {field}" - - def test_batch_size_bounds_valid(self, client): - data = client.get("/context").json() - assert data["batch_size_min"] <= data["default_batch_size"] <= data["batch_size_max"] - - def test_time_range_valid(self, client): - data = client.get("/context").json() - assert data["time_min"] < data["time_max"] - - -# --------------------------------------------------------------------------- -# /simulate — validation guard -# --------------------------------------------------------------------------- - -@pytest.mark.api -class TestSimulateValidation: - def test_invalid_priority_rejected(self, client): - context = client.get("/context").json() - response = client.post("/simulate", json={ - "priority": "urgent", # not in low/medium/high - "time": context["default_time"], - "alpha": 1.0, "beta": 1.0, "gamma": 1.0, - }) - assert response.status_code == 422 - - def test_alpha_out_of_range_rejected(self, client): - context = client.get("/context").json() - response = client.post("/simulate", json={ - "priority": "medium", - "time": context["default_time"], - "alpha": 99.0, # > 5.0 max - "beta": 1.0, "gamma": 1.0, - }) - assert response.status_code == 422 - - def test_valid_request_returns_200(self, client): - context = client.get("/context").json() - response = client.post("/simulate", json={ - "priority": "medium", - "time": context["default_time"], - "alpha": 1.0, "beta": 1.0, "gamma": 1.0, - }) - assert response.status_code == 200 - - def test_response_has_required_fields(self, client): - context = client.get("/context").json() - data = client.post("/simulate", json={ - "priority": "medium", - "time": context["default_time"], - "alpha": 1.0, "beta": 1.0, "gamma": 1.0, - }).json() - for field in ["time", "origin_city", "paths", "nodes", "metrics", "insight"]: - assert field in data, f"Missing field: {field}" - - -# --------------------------------------------------------------------------- -# /explain — validation -# --------------------------------------------------------------------------- - -@pytest.mark.api -class TestExplainEndpoint: - def test_invalid_target_rejected(self, client): - response = client.get("/explain", params={ - "city": "Dallas", - "target": "energy", # invalid - "time": "2024-06-01T12:00", - }) - assert response.status_code == 422 - - def test_valid_co2_request_structure(self, client): - """If models exist, response must have the right shape.""" - response = client.get("/explain", params={ - "city": "Dallas", - "target": "co2", - "time": "2024-06-01T12:00", - }) - # 200 = models loaded; 503 = model file missing (acceptable in CI) - assert response.status_code in (200, 503) - if response.status_code == 200: - data = response.json() - assert "prediction" in data - assert "base_value" in data - assert "features" in data - assert isinstance(data["features"], list) - - def test_explain_features_have_shap_values(self, client): - response = client.get("/explain", params={ - "city": "Dallas", - "target": "co2", - "time": "2024-06-01T12:00", - }) - if response.status_code == 200: - features = response.json()["features"] - assert len(features) > 0 - for f in features: - assert "name" in f - assert "shap_value" in f - assert isinstance(f["shap_value"], float) diff --git a/tests/test_baselines.py b/tests/test_baselines.py index 4a2df9c..47f64ab 100644 --- a/tests/test_baselines.py +++ b/tests/test_baselines.py @@ -1,135 +1,24 @@ -"""Tests for baseline scheduling strategies.""" +from __future__ import annotations import pandas as pd -import pytest -from src.scheduling import ( - PreparedSchedulerState, - PreparedCityMetrics, - build_job_record, - prepare_environment_frame, - prepare_scheduler_state, - load_dc_config, -) -from src.baselines import ( - baseline_random, - baseline_nearest_neighbor, - baseline_time_of_day_heuristic, -) +from src.baselines import run_baseline -@pytest.fixture -def sample_prepared_state(sample_env_df, sample_dc_df): - """Create a sample prepared state for testing baselines.""" - env_df = prepare_environment_frame(sample_env_df) - return prepare_scheduler_state(env_df, sample_dc_df) +def test_immediate_local_prefers_origin(environment, dc_config, jobs) -> None: + result = run_baseline("immediate_local", environment, jobs.iloc[[0]], dc_config) + assert result.loc[0, "assigned_city"] == "Alpha" -@pytest.fixture -def sample_job(): - """Create a sample job for testing.""" - return build_job_record( - origin_city="Dallas", - power_demand=500.0, - duration_hours=4, - earliest_start="2024-06-01T12:00", - deadline="2024-06-01T20:00", - priority=1, # 0=high, 1=medium, 2=low - job_id="test_job_001", - ) +def test_all_baselines_respect_deadlines(environment, dc_config, jobs) -> None: + for strategy in ("immediate_local", "carbon_greedy", "random"): + result = run_baseline(strategy, environment, jobs, dc_config, seed=7) + scheduled = result[result["scheduled"].eq(1)] + assert (scheduled["scheduled_end"] <= pd.to_datetime(scheduled["deadline"])).all() -class TestBaselineRandom: - """Tests for random baseline strategy.""" - - def test_returns_schedule_candidate(self, sample_prepared_state, sample_job): - """Random baseline returns a ScheduleCandidate or None.""" - result = baseline_random(sample_job, sample_prepared_state) - assert result is None or result.dc_id is not None - - def test_respects_deadline(self, sample_prepared_state, sample_job): - """Random baseline respects job deadline.""" - result = baseline_random(sample_job, sample_prepared_state) - if result: - assert result.end_time <= pd.Timestamp(sample_job["deadline"]) - - def test_respects_earliest_start(self, sample_prepared_state, sample_job): - """Random baseline respects job earliest start.""" - result = baseline_random(sample_job, sample_prepared_state) - if result: - assert result.start_time >= pd.Timestamp(sample_job["earliest_start"]) - - def test_has_positive_co2_expectation(self, sample_prepared_state, sample_job): - """Random baseline returns positive CO2 expectations.""" - result = baseline_random(sample_job, sample_prepared_state) - if result: - assert result.expected_co2_kg >= 0.0 - - def test_has_positive_water_expectation(self, sample_prepared_state, sample_job): - """Random baseline returns positive water expectations.""" - result = baseline_random(sample_job, sample_prepared_state) - if result: - assert result.expected_water_liters >= 0.0 - - -class TestBaselineNearestNeighbor: - """Tests for nearest neighbor baseline strategy.""" - - def test_returns_schedule_candidate(self, sample_prepared_state, sample_job): - """Nearest neighbor baseline returns a ScheduleCandidate or None.""" - result = baseline_nearest_neighbor(sample_job, sample_prepared_state) - assert result is None or result.dc_id is not None - - def test_prefers_same_city(self, sample_prepared_state, sample_job): - """Nearest neighbor prefers same-city DC if available.""" - result = baseline_nearest_neighbor(sample_job, sample_prepared_state) - if result: - # If origin city is in DC list, result should be in same city - if sample_job["origin_city"] in [dc["city"] for dc in sample_prepared_state.dc_records]: - assert result.city == sample_job["origin_city"] - - def test_respects_deadline(self, sample_prepared_state, sample_job): - """Nearest neighbor respects job deadline.""" - result = baseline_nearest_neighbor(sample_job, sample_prepared_state) - if result: - assert result.end_time <= pd.Timestamp(sample_job["deadline"]) - - def test_respects_earliest_start(self, sample_prepared_state, sample_job): - """Nearest neighbor respects job earliest start.""" - result = baseline_nearest_neighbor(sample_job, sample_prepared_state) - if result: - assert result.start_time >= pd.Timestamp(sample_job["earliest_start"]) - - -class TestBaselineTimeOfDay: - """Tests for time-of-day heuristic baseline strategy.""" - - def test_returns_schedule_candidate(self, sample_prepared_state, sample_job): - """Time-of-day heuristic returns a ScheduleCandidate or None.""" - result = baseline_time_of_day_heuristic(sample_job, sample_prepared_state) - assert result is None or result.dc_id is not None - - def test_picks_low_carbon_city(self, sample_prepared_state, sample_job): - """Time-of-day heuristic picks city with lowest CO2.""" - result = baseline_time_of_day_heuristic(sample_job, sample_prepared_state) - if result: - # Result should have a city in prepared state - assert result.city in sample_prepared_state.city_metrics - - def test_respects_deadline(self, sample_prepared_state, sample_job): - """Time-of-day heuristic respects job deadline.""" - result = baseline_time_of_day_heuristic(sample_job, sample_prepared_state) - if result: - assert result.end_time <= pd.Timestamp(sample_job["deadline"]) - - def test_respects_earliest_start(self, sample_prepared_state, sample_job): - """Time-of-day heuristic respects job earliest start.""" - result = baseline_time_of_day_heuristic(sample_job, sample_prepared_state) - if result: - assert result.start_time >= pd.Timestamp(sample_job["earliest_start"]) - - def test_has_positive_co2_expectation(self, sample_prepared_state, sample_job): - """Time-of-day heuristic returns positive CO2 expectations.""" - result = baseline_time_of_day_heuristic(sample_job, sample_prepared_state) - if result: - assert result.expected_co2_kg >= 0.0 +def test_random_baseline_is_reproducible(environment, dc_config, jobs) -> None: + first = run_baseline("random", environment, jobs, dc_config, seed=12) + second = run_baseline("random", environment, jobs, dc_config, seed=12) + assert first["assigned_dc_id"].tolist() == second["assigned_dc_id"].tolist() + assert first["scheduled_start"].tolist() == second["scheduled_start"].tolist() diff --git a/tests/test_cross_region.py b/tests/test_cross_region.py deleted file mode 100644 index 5bc7ceb..0000000 --- a/tests/test_cross_region.py +++ /dev/null @@ -1,140 +0,0 @@ -""" -Tests for cross_region_comparison.py and sensitivity_analysis.py -""" -from __future__ import annotations - -import numpy as np -import pandas as pd -import pytest - -from src.cross_region_comparison import _build_jobs, _run_region -from src.sensitivity_analysis import ( - _build_jobs as sa_build_jobs, - _is_pareto_optimal, - run_sensitivity_analysis, -) - - -# ── Shared fixtures ──────────────────────────────────────────────────────────── - -@pytest.fixture() -def tiny_dc() -> pd.DataFrame: - return pd.DataFrame({ - "dc_id": ["dc_low", "dc_mid", "dc_high"], - "city": ["LowCarbon", "MidCarbon", "HighCarbon"], - "max_power_per_hour": [500.0, 500.0, 500.0], - "pue": [1.1, 1.2, 1.3], - "water_usage_factor": [0.05, 0.30, 0.60], - "region_water_scarcity":[0.05, 0.30, 0.60], - "is_new": [0, 0, 0], - "category": ["hyperscaler"] * 3, - }) - - -@pytest.fixture() -def tiny_data() -> pd.DataFrame: - """24 hours × 3 cities with clearly different CO2 intensities.""" - timestamps = pd.date_range("2023-06-01", periods=24, freq="h") - rows = [] - for city, co2, wue, dsci in [ - ("LowCarbon", 0.02, 1.1, 25.0), - ("MidCarbon", 0.30, 1.2, 150.0), - ("HighCarbon", 0.60, 1.3, 300.0), - ]: - for ts in timestamps: - rows.append({ - "CITY": city, - "TIMESTAMP": ts, - "co2_kg": co2 * 1000, - "total_gen_kwh": 1000.0, - "WUE_total": wue, - "dsci_0_500": dsci, - }) - return pd.DataFrame(rows) - - -# ── _build_jobs ──────────────────────────────────────────────────────────────── - -def test_build_jobs_count(tiny_dc: pd.DataFrame) -> None: - jobs = _build_jobs(tiny_dc, 30, "2023-06-01", "2023-06-30", seed=0) - assert len(jobs) == 30 - - -def test_build_jobs_required_columns(tiny_dc: pd.DataFrame) -> None: - jobs = _build_jobs(tiny_dc, 10, "2023-06-01", "2023-06-30", seed=0) - for col in ["job_id", "power_demand", "duration_hours", "origin_city", "priority"]: - assert col in jobs.columns - - -def test_build_jobs_origin_cities_valid(tiny_dc: pd.DataFrame) -> None: - jobs = _build_jobs(tiny_dc, 20, "2023-06-01", "2023-06-30", seed=0) - valid_cities = set(tiny_dc["city"].tolist()) - assert set(jobs["origin_city"]).issubset(valid_cities) - - -# ── _run_region ──────────────────────────────────────────────────────────────── - -def test_run_region_returns_expected_keys(tiny_data: pd.DataFrame, tiny_dc: pd.DataFrame) -> None: - jobs = _build_jobs(tiny_dc, 10, "2023-06-01", "2023-06-01 23:00:00", seed=0) - result = _run_region("TEST", tiny_data, tiny_dc, jobs, alpha=0.7, beta=0.2, gamma=0.1) - for key in ["region", "co2_reduction_pct", "optimal_co2_kg", "naive_co2_kg", "co2_spread_x"]: - assert key in result - - -def test_run_region_optimal_le_naive(tiny_data: pd.DataFrame, tiny_dc: pd.DataFrame) -> None: - """Optimal scheduler should never be worse than naive on CO2.""" - jobs = _build_jobs(tiny_dc, 15, "2023-06-01", "2023-06-01 23:00:00", seed=1) - result = _run_region("TEST", tiny_data, tiny_dc, jobs, alpha=0.7, beta=0.2, gamma=0.1) - assert result["optimal_co2_kg"] <= result["naive_co2_kg"] + 1e-6 - - -def test_run_region_co2_spread_positive(tiny_data: pd.DataFrame, tiny_dc: pd.DataFrame) -> None: - jobs = _build_jobs(tiny_dc, 5, "2023-06-01", "2023-06-01 23:00:00", seed=2) - result = _run_region("TEST", tiny_data, tiny_dc, jobs, alpha=0.7, beta=0.2, gamma=0.1) - assert result["co2_spread_x"] >= 1.0 - - -# ── _is_pareto_optimal ───────────────────────────────────────────────────────── - -def test_pareto_marks_dominated_correctly() -> None: - df = pd.DataFrame({ - "total_co2_kg": [100, 50, 200], - "total_water_l": [200, 150, 100], - }) - result = _is_pareto_optimal(df) - # Row 0 (100, 200) is dominated by row 1 (50, 150) → not Pareto - assert result.iloc[0] == False # noqa: E712 - # Row 1 (50, 150) is Pareto — nothing strictly better on both - assert result.iloc[1] == True # noqa: E712 - # Row 2 (200, 100) is Pareto — best water - assert result.iloc[2] == True # noqa: E712 - - -def test_pareto_single_row_is_optimal() -> None: - df = pd.DataFrame({"total_co2_kg": [50.0], "total_water_l": [100.0]}) - result = _is_pareto_optimal(df) - assert result.iloc[0] == True # noqa: E712 - - -# ── run_sensitivity_analysis ─────────────────────────────────────────────────── - -def test_sensitivity_output_shape(tiny_data: pd.DataFrame, tiny_dc: pd.DataFrame) -> None: - jobs = sa_build_jobs(tiny_dc, 10, "2023-06-01", "2023-06-01 23:00:00") - df = run_sensitivity_analysis( - tiny_data, tiny_dc, jobs, - alpha_values=[0.0, 0.5, 1.0], - beta_values=[0.0, 0.5], - ) - # Valid combinations: (0,0),(0,0.5),(0.5,0),(0.5,0.5),(1,0) = 5 rows - assert len(df) == 5 - assert "pareto_optimal" in df.columns - - -def test_sensitivity_at_least_one_pareto(tiny_data: pd.DataFrame, tiny_dc: pd.DataFrame) -> None: - jobs = sa_build_jobs(tiny_dc, 10, "2023-06-01", "2023-06-01 23:00:00") - df = run_sensitivity_analysis( - tiny_data, tiny_dc, jobs, - alpha_values=[0.0, 0.5, 1.0], - beta_values=[0.0, 0.5], - ) - assert df["pareto_optimal"].any() diff --git a/tests/test_data_adapters.py b/tests/test_data_adapters.py new file mode 100644 index 0000000..de6a75c --- /dev/null +++ b/tests/test_data_adapters.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import pandas as pd +import pytest + +from src.fetch_eia import process_region +from src.fetch_entso_e import _parse_generation_xml + + +def test_eia_fuel_codes_are_not_collapsed_to_other() -> None: + raw = pd.DataFrame( + [ + {"period": "2025-01-01T00", "type-name": "COL", "value": "60"}, + {"period": "2025-01-01T00", "type-name": "NG", "value": "40"}, + ] + ) + result = process_region(raw, "PJM") + assert result.loc[0, "coal_share"] == pytest.approx(0.6) + assert result.loc[0, "natural_gas_share"] == pytest.approx(0.4) + assert result.loc[0, "other_share"] == pytest.approx(0.0) + + +def test_entsoe_quarter_hour_solar_is_aggregated_to_mwh() -> None: + points = "".join( + f"{index}100" + for index in range(1, 5) + ) + xml = f""" + + + B17 + + 2025-01-01T00:00Z + PT15M + {points} + + + + """ + rows = _parse_generation_xml(xml, "TEST") + assert len(rows) == 1 + assert rows[0]["fuel_type"] == "solar" + assert rows[0]["generation_mwh"] == pytest.approx(100.0) diff --git a/tests/test_experiments.py b/tests/test_experiments.py new file mode 100644 index 0000000..b5f60a9 --- /dev/null +++ b/tests/test_experiments.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import hashlib +import json + +import pandas as pd +import pytest + +from src.experiments import run_experiments + + +@pytest.mark.integration +def test_end_to_end_artifacts_are_created(tmp_path) -> None: + paths = run_experiments( + results_dir=tmp_path / "results", + figures_dir=tmp_path / "figures", + seeds=2, + jobs=24, + hours=72, + ) + assert all(path.is_file() and path.stat().st_size > 0 for path in paths.values()) + summary = pd.read_csv(paths["paired_summary"]) + assert set(summary["baseline"]) == {"immediate_local", "carbon_greedy", "random"} + metadata = json.loads(paths["metadata"].read_text(encoding="utf-8")) + assert metadata["claim_scope"] == "simulation only" + + checksum_dir = paths["checksums"].parent + for line in paths["checksums"].read_text(encoding="utf-8").splitlines(): + expected, relative_path = line.split(" ", maxsplit=1) + artifact = checksum_dir / relative_path + assert artifact.is_file() + assert hashlib.sha256(artifact.read_bytes()).hexdigest() == expected diff --git a/tests/test_explain.py b/tests/test_explain.py deleted file mode 100644 index e94930e..0000000 --- a/tests/test_explain.py +++ /dev/null @@ -1,93 +0,0 @@ -""" -Unit tests for src/explain.py (SHAP feature importance). - -These tests verify the explain module's output contract: - - Shape and types of the returned dict - - Features are sorted by |shap_value| - - Base features only (no city_ one-hot columns) - - prediction ≈ base_value + sum(shap_values) within tolerance - -Why test SHAP output shape? - The frontend reads specific keys. If the keys change or are missing, - the bar chart silently breaks. These tests catch that regression. -""" -from __future__ import annotations - -import pytest - -# Skip the whole module if model bundles don't exist (e.g., clean CI clone) -pytest.importorskip("shap") - -try: - from src.explain import compute_shap, _load_bundle - MODELS_AVAILABLE = (_load_bundle.__wrapped__ if hasattr(_load_bundle, "__wrapped__") - else _load_bundle) # just check if importable - try: - _load_bundle("co2") - MODELS_AVAILABLE = True - except FileNotFoundError: - MODELS_AVAILABLE = False -except Exception: - MODELS_AVAILABLE = False - -needs_models = pytest.mark.skipif( - not MODELS_AVAILABLE, - reason="Model bundle files not present — run train.py first", -) - - -# --------------------------------------------------------------------------- -# compute_shap — output contract -# --------------------------------------------------------------------------- - -@pytest.mark.unit -@needs_models -class TestComputeShapContract: - @pytest.mark.parametrize("target", ["co2", "wue"]) - def test_returns_dict_with_required_keys(self, target): - result = compute_shap(target=target, city="Dallas", time="2024-06-01T12:00") - for key in ["city", "target", "prediction", "base_value", "features"]: - assert key in result, f"Missing key: {key}" - - @pytest.mark.parametrize("target", ["co2", "wue"]) - def test_features_is_list(self, target): - result = compute_shap(target=target, city="Dallas", time="2024-06-01T12:00") - assert isinstance(result["features"], list) - assert len(result["features"]) > 0 - - @pytest.mark.parametrize("target", ["co2", "wue"]) - def test_feature_schema(self, target): - result = compute_shap(target=target, city="Dallas", time="2024-06-01T12:00") - for f in result["features"]: - assert "name" in f - assert "raw_name" in f - assert "value" in f - assert "shap_value" in f - assert isinstance(f["shap_value"], float) - - @pytest.mark.parametrize("target", ["co2", "wue"]) - def test_no_city_columns_in_features(self, target): - """City one-hot columns (city_Dallas etc.) should be excluded.""" - result = compute_shap(target=target, city="Dallas", time="2024-06-01T12:00") - for f in result["features"]: - assert not f["raw_name"].startswith("city_"), ( - f"City column leaked into features: {f['raw_name']}" - ) - - @pytest.mark.parametrize("target", ["co2", "wue"]) - def test_features_sorted_by_magnitude(self, target): - """Features must be sorted descending by |shap_value|.""" - result = compute_shap(target=target, city="Dallas", time="2024-06-01T12:00") - shap_abs = [abs(f["shap_value"]) for f in result["features"]] - assert shap_abs == sorted(shap_abs, reverse=True), ( - "Features are not sorted by |shap_value| descending" - ) - - def test_invalid_target_raises(self): - with pytest.raises(ValueError, match="target must be"): - compute_shap(target="energy", city="Dallas", time="2024-06-01T12:00") - - @pytest.mark.parametrize("target", ["co2", "wue"]) - def test_returns_at_most_8_features(self, target): - result = compute_shap(target=target, city="Dallas", time="2024-06-01T12:00", top_n=8) - assert len(result["features"]) <= 8 diff --git a/tests/test_forecast_robustness.py b/tests/test_forecast_robustness.py index e1e2a69..69945dc 100644 --- a/tests/test_forecast_robustness.py +++ b/tests/test_forecast_robustness.py @@ -1,12 +1,3 @@ -""" -Tests for forecast_robustness.py - -Validates that: -- Noise injection preserves non-negative intensities -- Zero noise produces zero regret (oracle == actual) -- Regret increases monotonically with noise up to moderate levels -- Both US and EU datasets produce valid output tables -""" from __future__ import annotations import numpy as np @@ -14,154 +5,50 @@ import pytest from src.forecast_robustness import ( - _add_forecast_noise, - _evaluate_schedule_against_truth, + add_forecast_noise, + evaluate_schedule_against_truth, run_forecast_robustness, ) -# ── Fixtures ────────────────────────────────────────────────────────────────── - -@pytest.fixture() -def tiny_env() -> pd.DataFrame: - """Minimal environment frame: 2 cities × 24 hours.""" - timestamps = pd.date_range("2023-06-01", periods=24, freq="h") - rows = [] - for city, intensity in [("LowCarbon", 0.02), ("HighCarbon", 0.60)]: - for ts in timestamps: - rows.append({ - "city": city, - "timestamp": ts, - "predicted_co2_intensity": intensity, - "predicted_water_intensity": 1.2, - "scarcity_index": 0.1, - }) - return pd.DataFrame(rows) - - -@pytest.fixture() -def tiny_dc() -> pd.DataFrame: - return pd.DataFrame({ - "dc_id": ["dc_low", "dc_high"], - "city": ["LowCarbon", "HighCarbon"], - "max_power_per_hour": [500.0, 500.0], - "pue": [1.2, 1.3], - "water_usage_factor": [0.05, 0.60], - "region_water_scarcity":[0.1, 0.6], - "is_new": [0, 0], - "category": ["hyperscaler", "hyperscaler"], - }) - - -@pytest.fixture() -def tiny_jobs(tiny_dc: pd.DataFrame) -> pd.DataFrame: - cities = tiny_dc["city"].tolist() - rng = np.random.default_rng(0) - n = 10 - return pd.DataFrame({ - "job_id": [f"j{i}" for i in range(n)], - "power_demand": [20.0] * n, - "duration_hours": [2] * n, - "earliest_start": "2023-06-01 00:00:00", - "deadline": "2023-06-01 23:00:00", - "origin_city": rng.choice(cities, n).tolist(), - "priority": 1, - }) +def test_noise_is_nonnegative(environment) -> None: + noisy = add_forecast_noise(environment, 4.0, np.random.default_rng(2)) + assert (noisy["carbon_intensity_kg_per_kwh"] >= 0).all() -# ── Noise injection ──────────────────────────────────────────────────────────── - -def test_zero_noise_is_identity(tiny_env: pd.DataFrame) -> None: - rng = np.random.default_rng(0) - noisy = _add_forecast_noise(tiny_env, noise_level=0.0, rng=rng) - pd.testing.assert_series_equal( - noisy["predicted_co2_intensity"], - tiny_env["predicted_co2_intensity"], - check_names=False, +def test_truth_evaluation_uses_full_window_and_pue(dc_config) -> None: + truth = pd.DataFrame( + { + "timestamp": pd.date_range("2025-01-01", periods=2, freq="h"), + "city": ["Alpha", "Alpha"], + "carbon_intensity_kg_per_kwh": [0.1, 0.3], + "water_intensity_l_per_kwh": [1.0, 1.0], + } ) - - -def test_noise_stays_non_negative(tiny_env: pd.DataFrame) -> None: - rng = np.random.default_rng(42) - noisy = _add_forecast_noise(tiny_env, noise_level=5.0, rng=rng) # extreme noise - assert (noisy["predicted_co2_intensity"] >= 0).all() - - -def test_noise_changes_values(tiny_env: pd.DataFrame) -> None: - rng = np.random.default_rng(1) - noisy = _add_forecast_noise(tiny_env, noise_level=0.20, rng=rng) - assert not noisy["predicted_co2_intensity"].equals(tiny_env["predicted_co2_intensity"]) - - -# ── Evaluation ──────────────────────────────────────────────────────────────── - -def test_evaluate_empty_schedule_returns_zero(tiny_env: pd.DataFrame) -> None: - empty_schedule = pd.DataFrame(columns=[ - "scheduled", "assigned_city", "scheduled_start", "power_demand", "duration_hours" - ]) - result = _evaluate_schedule_against_truth(empty_schedule, tiny_env) - assert result == 0.0 - - -def test_evaluate_known_assignment(tiny_env: pd.DataFrame) -> None: - """LowCarbon city at 0.02 kg/kWh × 20 kW × 2 h = 0.80 kg CO2.""" - schedule = pd.DataFrame([{ - "scheduled": 1, - "assigned_city": "LowCarbon", - "scheduled_start": "2023-06-01 00:00:00", - "power_demand": 20.0, - "duration_hours": 2, - }]) - actual = _evaluate_schedule_against_truth(schedule, tiny_env) - assert pytest.approx(actual, rel=0.01) == 0.02 * 20.0 * 2 - - -# ── Full experiment ──────────────────────────────────────────────────────────── - -def test_run_returns_correct_columns( - tiny_env: pd.DataFrame, tiny_dc: pd.DataFrame, tiny_jobs: pd.DataFrame -) -> None: - # Build a minimal data_df that prepare_environment_frame can process - data_df = tiny_env.rename(columns={ - "city": "CITY", - "timestamp": "TIMESTAMP", - "predicted_co2_intensity": "co2_intensity_kg_per_kwh", - }) - data_df["co2_kg"] = data_df["co2_intensity_kg_per_kwh"] * 1000 - data_df["total_gen_kwh"] = 1000.0 - data_df["WUE_total"] = 1.2 - data_df["dsci_0_500"] = 50.0 - - results = run_forecast_robustness( - data_df=data_df, - dc_df=tiny_dc, - jobs_df=tiny_jobs, - noise_levels=[0.0, 0.10, 0.20], + schedule = pd.DataFrame( + [ + { + "job_id": "job", + "scheduled": 1, + "assigned_dc_id": "dc_alpha", + "assigned_city": "Alpha", + "scheduled_start": "2025-01-01 00:00:00", + "power_demand_kw": 10.0, + "duration_hours": 2, + } + ] ) - required = {"noise_level", "oracle_co2_kg", "actual_co2_kg", "regret_kg", "regret_pct"} - assert required.issubset(set(results.columns)) - assert len(results) == 3 - + actual = evaluate_schedule_against_truth(schedule, truth, dc_config) + assert actual == pytest.approx(10.0 * 1.1 * (0.1 + 0.3)) -def test_zero_noise_produces_zero_regret( - tiny_env: pd.DataFrame, tiny_dc: pd.DataFrame, tiny_jobs: pd.DataFrame -) -> None: - data_df = tiny_env.rename(columns={ - "city": "CITY", - "timestamp": "TIMESTAMP", - "predicted_co2_intensity": "co2_intensity_kg_per_kwh", - }) - data_df["co2_kg"] = data_df["co2_intensity_kg_per_kwh"] * 1000 - data_df["total_gen_kwh"] = 1000.0 - data_df["WUE_total"] = 1.2 - data_df["dsci_0_500"] = 50.0 - results = run_forecast_robustness( - data_df=data_df, - dc_df=tiny_dc, - jobs_df=tiny_jobs, +def test_zero_noise_matches_perfect_forecast(environment, dc_config, jobs) -> None: + result = run_forecast_robustness( + environment, + dc_config, + jobs, noise_levels=[0.0], + repetitions=2, ) - row = results[results["noise_level"] == 0.0].iloc[0] - assert row["regret_kg"] == pytest.approx(0.0, abs=1e-6) - assert row["regret_pct"] == pytest.approx(0.0, abs=1e-6) + assert result.loc[0, "carbon_delta_pct_mean"] == pytest.approx(0.0) + assert result.loc[0, "forecast_rmse_mean"] == pytest.approx(0.0) diff --git a/tests/test_scheduling.py b/tests/test_scheduling.py index 6d84c2e..505addb 100644 --- a/tests/test_scheduling.py +++ b/tests/test_scheduling.py @@ -1,219 +1,132 @@ -""" -Unit tests for src/scheduling.py - -These tests use only in-memory DataFrames — no file I/O, no models. -They verify the scheduling algorithm's core logic: - - Job record construction - - Candidate scoring (alpha/beta/gamma weights) - - Schedule summarization - - Baseline vs. optimized comparison -""" from __future__ import annotations import pandas as pd import pytest from src.scheduling import ( - _candidate_score, build_job_record, - compare_summaries, - summarize_schedule, + paired_comparison, + prepare_scheduler_state, + schedule_jobs, + score_candidates, ) -# --------------------------------------------------------------------------- -# build_job_record -# --------------------------------------------------------------------------- - -@pytest.mark.unit -class TestBuildJobRecord: - def test_returns_series(self): - record = build_job_record( - origin_city="Dallas", - power_demand=10.0, - duration_hours=2, - earliest_start="2024-06-01 12:00", - deadline="2024-06-01 18:00", - ) - assert isinstance(record, pd.Series) - - def test_required_fields_present(self): - record = build_job_record("Dallas", 10.0, 2, "2024-06-01", "2024-06-02") - for field in ["job_id", "origin_city", "power_demand", "duration_hours", - "earliest_start", "deadline", "priority", "priority_label"]: - assert field in record.index, f"Missing field: {field}" - - def test_timestamps_parsed(self): - record = build_job_record("Dallas", 10.0, 2, "2024-06-01", "2024-06-02") - assert isinstance(record["earliest_start"], pd.Timestamp) - assert isinstance(record["deadline"], pd.Timestamp) - - def test_priority_label_mapping(self): - high = build_job_record("Dallas", 5.0, 1, "2024-06-01", "2024-06-02", priority=0) - med = build_job_record("Dallas", 5.0, 1, "2024-06-01", "2024-06-02", priority=1) - low = build_job_record("Dallas", 5.0, 1, "2024-06-01", "2024-06-02", priority=2) - assert high["priority_label"] == "high" - assert med["priority_label"] == "medium" - assert low["priority_label"] == "low" - - def test_custom_job_id(self): - record = build_job_record("Dallas", 5.0, 1, "2024-06-01", "2024-06-02", - job_id="test_123") - assert record["job_id"] == "test_123" - - -# --------------------------------------------------------------------------- -# _candidate_score — the heart of the scheduler -# -# score = alpha * (co2_mean * pue) -# + beta * (water_mean * water_usage_factor * region_water_scarcity) -# + gamma * (0 if same city, else 1) -# -# TODO: Fill in the parametrize cases below. -# Each tuple is: (alpha, beta, gamma, origin_city, dc_city, description) -# Think about edge cases where a weight = 0, or same vs. different city. -# --------------------------------------------------------------------------- - -@pytest.mark.unit -@pytest.mark.parametrize("alpha, beta, gamma, origin_city, dc_city, check", [ - # Case 1: alpha=0 → CO2 component is computed but adds nothing to score - (0.0, 1.0, 1.0, "Dallas", "Atlanta", "zero_alpha"), - # Case 2: beta=0 → water component is computed but adds nothing to score - (1.0, 0.0, 1.0, "Dallas", "Atlanta", "zero_beta"), - # Case 3: gamma=0 → no latency penalty regardless of city distance - (1.0, 1.0, 0.0, "Dallas", "Atlanta", "zero_gamma"), - # Case 4: same city → latency penalty is 0 even with a high gamma - (1.0, 1.0, 5.0, "Dallas", "Dallas", "same_city_no_latency"), -]) -def test_candidate_score_weights(alpha, beta, gamma, origin_city, dc_city, check): - """ - Verifies the alpha/beta/gamma knobs behave correctly. - Each case isolates one dimension of the scoring formula. - - score = alpha * co2_component - + beta * water_component - + gamma * (0 if same city else 1) - """ - dc_record = { - "city": dc_city, - "pue": 1.2, - "water_usage_factor": 1.0, - "region_water_scarcity": 0.5, - } - co2_mean = 0.4 # co2_component = 0.4 * 1.2 = 0.48 - water_mean = 2.0 # water_component = 2.0 * 1.0 * 0.5 = 1.0 - - score, co2_component, water_component = _candidate_score( - co2_mean=co2_mean, - water_mean=water_mean, - dc_record=dc_record, - origin_city=origin_city, - alpha=alpha, - beta=beta, - gamma=gamma, +def test_build_job_record_normalizes_timezone() -> None: + record = build_job_record( + "Alpha", + 10, + 2, + "2025-01-01 01:00:00+01:00", + "2025-01-01 05:00:00+01:00", ) - - latency_penalty = gamma if origin_city != dc_city else 0.0 - - if check == "zero_alpha": - # CO2 component is still computed (0.48), just not weighted in the score - assert co2_component == pytest.approx(co2_mean * 1.2) - assert score == pytest.approx(0.0 * co2_component + beta * water_component + latency_penalty) - - elif check == "zero_beta": - # Water component is still computed (1.0), just not weighted in the score - assert water_component == pytest.approx(water_mean * 1.0 * 0.5) - assert score == pytest.approx(alpha * co2_component + 0.0 * water_component + latency_penalty) - - elif check == "zero_gamma": - # No latency penalty — score is purely environmental even across cities - assert score == pytest.approx(alpha * co2_component + beta * water_component) - - elif check == "same_city_no_latency": - # Same city: latency term is structurally 0, even though gamma=5.0 - expected_without_latency = alpha * co2_component + beta * water_component - assert score == pytest.approx(expected_without_latency) - - -# --------------------------------------------------------------------------- -# summarize_schedule -# --------------------------------------------------------------------------- - -@pytest.mark.unit -class TestSummarizeSchedule: - def test_coverage_two_of_three(self, sample_schedule_df): - result = summarize_schedule(sample_schedule_df) - assert result["scheduled_jobs"] == pytest.approx(2.0) - assert result["total_jobs"] == pytest.approx(3.0) - assert result["coverage"] == pytest.approx(2 / 3) - - def test_total_co2_only_scheduled(self, sample_schedule_df): - result = summarize_schedule(sample_schedule_df) - # job_c is not scheduled → its 0.0 co2 doesn't contribute, sum is 3.0 - assert result["total_expected_co2_kg"] == pytest.approx(3.0) - - def test_total_water_only_scheduled(self, sample_schedule_df): - result = summarize_schedule(sample_schedule_df) - assert result["total_expected_water_liters"] == pytest.approx(15.0) - - def test_empty_schedule(self): - empty = pd.DataFrame(columns=["scheduled", "expected_co2_kg", "expected_water_liters"]) - result = summarize_schedule(empty) - assert result["coverage"] == pytest.approx(0.0) - assert result["scheduled_jobs"] == pytest.approx(0.0) - - def test_all_unscheduled(self): - df = pd.DataFrame([ - {"scheduled": 0, "expected_co2_kg": 0.0, "expected_water_liters": 0.0}, - {"scheduled": 0, "expected_co2_kg": 0.0, "expected_water_liters": 0.0}, - ]) - result = summarize_schedule(df) - assert result["coverage"] == pytest.approx(0.0) - assert result["scheduled_jobs"] == pytest.approx(0.0) - - -# --------------------------------------------------------------------------- -# compare_summaries -# --------------------------------------------------------------------------- - -@pytest.mark.unit -class TestCompareSummaries: - def _make_summary(self, co2: float, water: float, coverage: float, jobs: float): - return { - "total_expected_co2_kg": co2, - "total_expected_water_liters": water, - "coverage": coverage, - "scheduled_jobs": jobs, - "total_jobs": 10.0, - } - - def test_co2_reduction_pct(self): - baseline = self._make_summary(co2=100.0, water=50.0, coverage=0.8, jobs=8) - optimized = self._make_summary(co2=80.0, water=50.0, coverage=0.8, jobs=8) - result = compare_summaries(baseline, optimized) - assert result["co2_reduction_pct"] == pytest.approx(20.0) - - def test_water_reduction_pct(self): - baseline = self._make_summary(co2=100.0, water=200.0, coverage=0.8, jobs=8) - optimized = self._make_summary(co2=100.0, water=150.0, coverage=0.8, jobs=8) - result = compare_summaries(baseline, optimized) - assert result["water_reduction_pct"] == pytest.approx(25.0) - - def test_zero_baseline_co2_no_division_error(self): - """If baseline CO2 is 0, reduction should be 0.0 (not ZeroDivisionError).""" - baseline = self._make_summary(co2=0.0, water=50.0, coverage=0.8, jobs=8) - optimized = self._make_summary(co2=0.0, water=40.0, coverage=0.9, jobs=9) - result = compare_summaries(baseline, optimized) - assert result["co2_reduction_pct"] == pytest.approx(0.0) - - def test_coverage_delta(self): - baseline = self._make_summary(co2=100.0, water=50.0, coverage=0.6, jobs=6) - optimized = self._make_summary(co2=80.0, water=40.0, coverage=0.9, jobs=9) - result = compare_summaries(baseline, optimized) - assert result["coverage_delta_pct"] == pytest.approx(30.0) - - def test_scheduled_jobs_delta(self): - baseline = self._make_summary(co2=100.0, water=50.0, coverage=0.6, jobs=6) - optimized = self._make_summary(co2=80.0, water=40.0, coverage=0.9, jobs=9) - result = compare_summaries(baseline, optimized) - assert result["scheduled_jobs_delta"] == pytest.approx(3.0) + assert record["earliest_start"] == pd.Timestamp("2025-01-01 00:00:00") + + +def test_scheduler_respects_deadline(environment, dc_config) -> None: + jobs = pd.DataFrame( + [ + { + "job_id": "exact", + "origin_city": "Alpha", + "power_demand_kw": 10.0, + "duration_hours": 2, + "earliest_start": "2025-01-01 00:00:00", + "deadline": "2025-01-01 02:00:00", + "priority": 1, + } + ] + ) + result = schedule_jobs(environment, jobs, dc_config) + assert result.loc[0, "scheduled"] == 1 + assert result.loc[0, "scheduled_end"] <= pd.Timestamp(jobs.loc[0, "deadline"]) + + +def test_impossible_deadline_is_not_scheduled(environment, dc_config) -> None: + jobs = pd.DataFrame( + [ + { + "job_id": "late", + "origin_city": "Alpha", + "power_demand_kw": 10.0, + "duration_hours": 3, + "earliest_start": "2025-01-01 00:00:00", + "deadline": "2025-01-01 02:00:00", + "priority": 1, + } + ] + ) + result = schedule_jobs(environment, jobs, dc_config) + assert result.loc[0, "scheduled"] == 0 + assert result.loc[0, "failure_reason"] == "no_feasible_slot" + + +def test_capacity_is_enforced(environment, dc_config) -> None: + constrained = dc_config.copy() + constrained["max_power_kw"] = 10.0 + jobs = pd.DataFrame( + [ + { + "job_id": "too_large", + "origin_city": "Alpha", + "power_demand_kw": 11.0, + "duration_hours": 1, + "earliest_start": "2025-01-01", + "deadline": "2025-01-01 03:00:00", + "priority": 1, + } + ] + ) + assert schedule_jobs(environment, jobs, constrained).loc[0, "scheduled"] == 0 + + +def test_invalid_weights_are_rejected() -> None: + with pytest.raises(ValueError, match="sum to 1"): + score_candidates([], alpha=0.5, beta=0.5, gamma=0.5) + + +def test_paired_comparison_does_not_reward_missing_jobs() -> None: + baseline = pd.DataFrame( + [ + { + "job_id": "shared", + "power_demand_kw": 10.0, + "duration_hours": 1, + "scheduled": 1, + "expected_co2_kg": 10.0, + "expected_water_liters": 10.0, + "scarcity_weighted_water_liters": 12.0, + "relocated": 0, + }, + { + "job_id": "baseline_only", + "power_demand_kw": 10.0, + "duration_hours": 1, + "scheduled": 1, + "expected_co2_kg": 90.0, + "expected_water_liters": 90.0, + "scarcity_weighted_water_liters": 100.0, + "relocated": 0, + }, + ] + ) + proposed = baseline.copy() + proposed.loc[proposed["job_id"].eq("baseline_only"), "scheduled"] = 0 + proposed.loc[proposed["job_id"].eq("baseline_only"), "expected_co2_kg"] = float("nan") + proposed.loc[proposed["job_id"].eq("baseline_only"), "expected_water_liters"] = float( + "nan" + ) + proposed.loc[ + proposed["job_id"].eq("baseline_only"), "scarcity_weighted_water_liters" + ] = float("nan") + result = paired_comparison(baseline, proposed) + assert result["common_jobs"] == 1 + assert result["co2_reduction_pct"] == pytest.approx(0.0) + assert result["proposed_coverage"] == pytest.approx(0.5) + + +def test_missing_city_environment_is_rejected(environment, dc_config) -> None: + invalid = dc_config.copy() + invalid.loc[0, "city"] = "Missing" + with pytest.raises(ValueError, match="No environmental data"): + prepare_scheduler_state(environment, invalid)