Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 52 additions & 23 deletions .github/workflows/build-models.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,34 +2,45 @@ name: Build Example Model Pickles

on:
workflow_dispatch:
pull_request:
paths:
- ".github/workflows/build-models.yml"
- "numerai/**"
push:
# paths:
# - example_model.ipynb
# - hello_numerai.ipynb
# - feature_neutralization.ipynb
# - target_ensemble.ipynb
# - signals/example_model.ipynb
# - crypto/example_model.ipynb
branches:
- master

concurrency: build-example-models
concurrency:
group: build-example-models-${{ github.ref }}
cancel-in-progress: true

jobs:

build_and_test:
name: "Build Example Model Pickles"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install jupyter
- name: install-notebook-dependencies
run: |
python -m pip install --upgrade pip
pip install jupyter
pip install -r https://raw.githubusercontent.com/numerai/numerai-predict/refs/heads/master/py3.12/requirements.txt
pip install \
cloudpickle==3.1.1 \
jupyter \
lightgbm==4.5.0 \
matplotlib==3.10.3 \
numerapi==2.20.7 \
pandas==2.3.1 \
pyarrow==18.1.0 \
scikit-learn==1.6.1 \
scipy==1.16.0 \
seaborn==0.13.2
pip install --no-deps numerai-tools==0.4.0
- name: test-agent-helpers
run: PYTHONPATH=numerai python -m unittest numerai.agents.tests.test_target_transforms -v
- name: build-example-model
run: |
jupyter nbconvert \
Expand All @@ -55,37 +66,55 @@ jobs:
--ExecutePreprocessor.timeout=-1 \
--to html
- name: build-signals-example-model
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
run: |
jupyter nbconvert \
--execute signals/example_model.ipynb \
--ExecutePreprocessor.timeout=-1 \
--to html
- name: build-crypto-example-model
if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
run: |
jupyter nbconvert \
--execute crypto/example_model.ipynb \
--ExecutePreprocessor.timeout=-1 \
--to html
- name: move-numerai-pickles-to-cached-pickles-dir
run: |
mkdir -p cached-pickles/
mv -f numerai/example_model_v53_ender60.pkl cached-pickles/
mv -f numerai/hello_numerai_v53_ender60.pkl cached-pickles/
mv -f numerai/feature_neutralization_v53_ender60.pkl cached-pickles/
mv -f numerai/target_ensemble_v53_ender60.pkl cached-pickles/
- name: generate-publication-candidates-and-provenance
run: |
python numerai/generate_example_predictions.py \
--model cached-pickles/example_model_v53_ender60.pkl \
--output-dir generated-example-predictions
- name: upload-ender60-validation-evidence
uses: actions/upload-artifact@v4
with:
name: v53-ender60-example-model-${{ github.sha }}
if-no-files-found: error
path: |
cached-pickles/*_v53_ender60.pkl
generated-example-predictions/*
numerai/*.html
- name: cache-provenance-for-master
if: github.event_name == 'push'
run: |
cp generated-example-predictions/v53_ender60_provenance.json cached-pickles/
- name: delete-generated-files
run: |
rm -f numerai/example_model.html
rm -f numerai/hello_numerai.html
rm -f numerai/feature_neutralization.html
rm -f numerai/target_ensemble.html
rm -f numerai/*.html
rm -f signals/example_model.html
rm -f signals/signals_example_preds.csv
rm -f signals_example_preds.csv
rm -f crypto/example_model.html
rm -f crypto/crypto_example_preds.csv
rm -f crypto_example_preds.csv
- name: move-numerai-pickles-to-cached-pickles-dir
run: |
mkdir -p cached-pickles/
mv -f numerai/example_model.pkl cached-pickles/
mv -f numerai/hello_numerai.pkl cached-pickles/
mv -f numerai/feature_neutralization.pkl cached-pickles/
mv -f numerai/target_ensemble.pkl cached-pickles/
- name: commit-to-master
if: github.event_name == 'push'
uses: EndBug/add-and-commit@v9
with:
add: "cached-pickles/*"
3 changes: 2 additions & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -301,5 +301,6 @@ Build datasets with `python -m agents.code.data.build_full_datasets`:
- **Register repo skills**: `ln -s $PWD/numerai/agents/skills/* ~/.codex/skills/`
- **Network access required** for MCP operations (Codex CLI may need `--yolo` flag)
- **Always query Python version** before creating pkl files
- **BMC (Benchmark Model Contribution)** is the key experiment metric (proxy for MMC), computed vs official `v53_lgbm_ender20` benchmark predictions in `*_benchmark_models.parquet`
- **BMC (Benchmark Model Contribution)** is the key experiment metric (proxy for MMC), computed by default vs official `v53_lgbm_ender60` benchmark predictions in `*_benchmark_models.parquet`.
- The v5.3 default research target is explicit `target_ender_60` with a 16-era embargo. Ender-20 work must explicitly select `target_ender_20`; never let an Ender-20-named config read generic `target`.
- **Only Classic tournament (8)** supports pickle uploads
8 changes: 7 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,15 @@ Numerai is quickly developing open-source agent skills for you to use in the tou
```
git clone [email protected]:numerai/example-scripts
cd example-scripts && curl -sL http://numer.ai/install-mcp.sh | bash
codex exec --yolo "find the best neural network architecture to predict target ender"
codex exec --yolo "find the best neural network architecture to predict target_ender_60"
```

The maintained v5.3 examples pin `target_ender_60` explicitly. After the
in-place default-target cutover, generic `target` aliases Ender-60, while
`target_ender_20` remains available for intentionally reproducing the older
horizon. Explicit target names and the target-versioned cached model filenames
prevent an old Ender-20 model from being mistaken for the current example.

## Notebooks
We highly recommend getting started with Agents using the above section. But, if you're looking to kill some time on artisan data science, you can check out our tutorial notebooks here as well:

Expand Down
Binary file not shown.
Binary file removed cached-pickles/feature_neutralization.pkl
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
39 changes: 39 additions & 0 deletions cached-pickles/v53_ender60_provenance.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
{
"generated_at": "2026-08-06T19:22:31.088355+00:00",
"source_commit": "085420354bcd88caa29dfb68964bd14fbbc7522a",
"source_notebook": "numerai/example_model.ipynb",
"dataset_version": "v5.3",
"target_col": "target_ender_60",
"model_artifact": "example_model_v53_ender60.pkl",
"model_sha256": "a84448a2ff635631d111802398fc65fcba8250f943a729d64bbdf255a7b0cd34",
"model_artifacts": {
"example_model_v53_ender60.pkl": {
"source_notebook": "numerai/example_model.ipynb",
"sha256": "a84448a2ff635631d111802398fc65fcba8250f943a729d64bbdf255a7b0cd34"
},
"feature_neutralization_v53_ender60.pkl": {
"source_notebook": "numerai/feature_neutralization.ipynb",
"sha256": "8c45f0f1062bb337ebc33b1f1a9497a7e71ccc380ea054bb238b147c85d26230"
},
"hello_numerai_v53_ender60.pkl": {
"source_notebook": "numerai/hello_numerai.ipynb",
"sha256": "f94465d7431f6c53130fb7361b4132184539181e74e43e077db220d7ac3ca1a7"
},
"target_ensemble_v53_ender60.pkl": {
"source_notebook": "numerai/target_ensemble.ipynb",
"sha256": "fabba10e6701984022e28845f0748252e65ac1ffbdc4d4dcefc141a4061d5775"
}
},
"predictions": {
"validation": {
"file": "validation_example_preds_v53_ender60.parquet",
"rows": 4107040,
"sha256": "9587b14138c572fd041c1a7c05f0b36a53a3991f911aca162eae8bae9f82a2b3"
},
"live": {
"file": "live_example_preds_v53_ender60.parquet",
"rows": 7034,
"sha256": "89b4702157b68bee4bfedd0e21e0b7368274c4f7fc861ccbe0af2853a8d7cb45"
}
}
}
2 changes: 2 additions & 0 deletions numerai/AGENTS.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
# Numerai Tournament
This folder contains examples on how to participate in the Numerai Tournament.

For v5.3, select `target_ender_60` explicitly in maintained training examples. The generic `target` column demonstrates the current default alias only; do not use it when the horizon must remain reproducible.

## Directory Guide:
- `agents/`: agentic research framework + training/analysis pipeline (`python -m agents.code.modeling`)
- `v5.3/` (and other `v*/`): Numerai dataset files (often gitignored locally)
Expand Down
2 changes: 2 additions & 0 deletions numerai/agents/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ Always check the agents/skills/ folder for skills that match the user request.
Run commands from `numerai/` (so `agents` is importable), or from repo root with `PYTHONPATH=numerai`.
Data is expected to live under `numerai/<data_version>/` (e.g. `numerai/v5.3/`), which is often gitignored locally.

The default v5.3 workflow uses explicit `target_ender_60`, `v53_lgbm_ender60`, and a 16-era embargo. Generic `target` is a mutable dataset alias and must not be used where target identity matters. Keep Ender-20-named configs pinned to `target_ender_20`.

To make these repo skills available to Codex CLI, symlink them into `~/.codex/skills/`:
`ln -s $PWD/numerai/agents/skills/* ~/.codex/skills/`

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
'embargo_eras': 13,
'era_col': 'era',
'feature_set': 'all',
'target_col': 'target'},
'target_col': 'target_ender_20'},
'model': {'x_groups': ['features', 'era', 'benchmark_models'], 'params': {'colsample_bytree': 0.1,
'learning_rate': 0.001,
'max_depth': 10,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"embargo_eras": 13,
"era_col": "era",
"feature_set": "all",
"target_col": "target",
"target_col": "target_ender_20",
"id_col": "id",
"full_data_path": "v5.3/downsampled_full.parquet",
"benchmark_data_path": "v5.3/downsampled_full_benchmark_models.parquet",
Expand Down
22 changes: 22 additions & 0 deletions numerai/agents/baselines/configs/deep_lgbm_ender60_baseline.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
CONFIG = {'data': {'data_version': 'v5.3',
'embargo_eras': 16,
'era_col': 'era',
'feature_set': 'all',
'target_col': 'target_ender_60'},
'model': {'x_groups': ['features', 'era', 'benchmark_models'], 'params': {'colsample_bytree': 0.1,
'learning_rate': 0.001,
'max_depth': 10,
'min_data_in_leaf': 10000,
'n_estimators': 30000,
'n_jobs': 30,
'num_leaves': 1024,
'random_state': 1337},
'type': 'LGBMRegressor'},
'output': {'output_dir': 'baselines',
'results_name': 'deep_lgbm_ender60_baseline'},
'preprocessing': {'missing_value': 2.0, 'nan_missing_all_twos': False},
'training': {'cv': {'embargo': 16,
'enabled': True,
'min_train_size': 0,
'mode': 'expanding',
'n_splits': 5}}}
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
CONFIG = {
"data": {
"data_version": "v5.3",
"embargo_eras": 16,
"era_col": "era",
"feature_set": "all",
"target_col": "target_ender_60",
"id_col": "id",
"full_data_path": "v5.3/downsampled_full.parquet",
"benchmark_data_path": "v5.3/downsampled_full_benchmark_models.parquet",
},
"model": {
"x_groups": ["features", "era", "benchmark_models"],
"params": {
"colsample_bytree": 0.1,
"learning_rate": 0.001,
"max_depth": 10,
"min_data_in_leaf": 10000,
"n_estimators": 30000,
"n_jobs": 30,
"num_leaves": 1024,
"random_state": 1337,
},
"type": "LGBMRegressor",
},
"output": {
"output_dir": "baselines",
"results_name": "deep_lgbm_ender60_baseline_downsampled",
},
"preprocessing": {"missing_value": 2.0, "nan_missing_all_twos": False},
"training": {
"cv": {
"embargo": 16,
"enabled": True,
"min_train_size": 0,
"mode": "expanding",
"n_splits": 5,
}
},
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
'embargo_eras': 13,
'era_col': 'era',
'feature_set': 'medium',
'target_col': 'target',
'target_col': 'target_ender_20',
'full_data_path': 'v5.3/downsampled_full.parquet',
'benchmark_data_path': 'v5.3/downsampled_full_benchmark_models.parquet'},
'model': {'x_groups': ['features', 'era', 'benchmark_models'], 'params': {'colsample_bytree': 0.1,
Expand Down
25 changes: 25 additions & 0 deletions numerai/agents/baselines/configs/small_lgbm_ender60_baseline.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
CONFIG = {'data': {'data_version': 'v5.3',
'embargo_eras': 16,
'era_col': 'era',
'feature_set': 'medium',
'target_col': 'target_ender_60',
'full_data_path': 'v5.3/downsampled_full.parquet',
'benchmark_data_path': 'v5.3/downsampled_full_benchmark_models.parquet'},
'model': {'x_groups': ['features', 'era', 'benchmark_models'], 'params': {'colsample_bytree': 0.1,
'device_type': 'gpu',
'learning_rate': 0.01,
'max_depth': 5,
'min_data_in_leaf': 10000,
'n_estimators': 2000,
'n_jobs': 30,
'num_leaves': 31,
'random_state': 1337},
'type': 'LGBMRegressor'},
'output': {'output_dir': 'baselines',
'results_name': 'small_lgbm_ender60_baseline'},
'preprocessing': {'missing_value': 2.0, 'nan_missing_all_twos': False},
'training': {'cv': {'embargo': 16,
'enabled': True,
'min_train_size': 0,
'mode': 'expanding',
'n_splits': 5}}}
14 changes: 9 additions & 5 deletions numerai/agents/code/analysis/plot_benchmark_corrs.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,23 +28,27 @@
DEFAULT_DATA_VERSION_V52 = "v5.2"
DEFAULT_DATA_VERSION_V51 = "v5.1"
DEFAULT_DATA_VERSION_V5 = "v5.0"
DEFAULT_MODELS = ["v52_teager20", "v51_teager20", "v5_teager20"]
DEFAULT_MODELS = ["v53_ender60", "v52_teager20", "v51_teager20", "v5_teager20"]

ALIASES: dict[str, Sequence[str]] = {
"v52_teager20": ("v52_lgbm_teager2b20", "v52_lgbm_teager20"),
"v51_teager20": ("v51_lgbm_teager2b20", "v51_lgbm_teager20", "v51_teager20"),
"v5_teager20": ("v5_lgbm_teager2b20", "v5_lgbm_teager20", "v5_teager20"),
"v53_ender20": ("v53_lgbm_ender20",),
"v53_ender60": ("v53_lgbm_ender60",),
"v52_ender20": ("v52_lgbm_ender20",),
"ender20": ("v53_lgbm_ender20",),
"ender60": ("v53_lgbm_ender60",),
"v52_cyrus": ("v52_lgbm_cyrusd20",),
"cyrus": ("v52_lgbm_cyrusd20",),
"cyrusd20": ("v52_lgbm_cyrusd20",),
}

TARGET_ALIASES: dict[str, Sequence[str]] = {
"ender20": ("target_ender_20", "target_ender20"),
"ender60": ("target_ender_60", "target_ender60"),
"v53_ender20": ("target_ender_20",),
"v53_ender60": ("target_ender_60",),
"v52_ender20": ("target_ender_20",),
"cyrus": ("target_cyrusd_20", "target_cyrus_20", "target_cyrus20"),
"cyrusd20": ("target_cyrusd_20",),
Expand All @@ -55,7 +59,7 @@
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description=(
"Plot cumulative per-era numerai_corr for benchmark models vs ender/cyrus."
"Plot cumulative per-era numerai_corr for benchmark models vs Ender-60/Cyrus."
)
)
parser.add_argument(
Expand Down Expand Up @@ -139,8 +143,8 @@ def parse_args() -> argparse.Namespace:
parser.add_argument(
"--ender-col",
type=str,
default="ender20",
help="Target column (or alias) for ender20 correlations.",
default="ender60",
help="Target column (or alias) for Ender-60 correlations.",
)
parser.add_argument(
"--cyrus-col",
Expand Down Expand Up @@ -223,7 +227,7 @@ def _resolve_target_column(name: str, columns: Iterable[str]) -> str:

def _infer_version(name: str, default_version: str, args: argparse.Namespace) -> str:
lowered = name.lower()
if lowered in {"ender20"}:
if lowered in {"ender20", "ender60"}:
return args.v53_version
if lowered.startswith("v53_"):
return args.v53_version
Expand Down
Loading