discovery catalog import

.env.example

@@ -1,26 +1,41 @@
-# LLM Provider
-OPENAI_API_KEY=your-openai-key-here
-# Or use local LLM via Ollama
-OLLAMA_BASE_URL=http://localhost:11434
-OLLAMA_MODEL=llama3.1
+# =============================================================================
+# TaarYa — Environment Configuration Template
+# =============================================================================
+# Copy this to .env and fill in your values:
+#   cp .env.example .env
+#
+# All settings can also be set via environment variables (see below).
+# =============================================================================
 
-# Database Connections
+# ── Database URLs ─────────────────────────────────────────────────────────────
+DATABASE_URL=postgresql+psycopg2://taarya:taarya123@localhost:5432/astronomy
+
+# ── Qdrant (Vector DB) ───────────────────────────────────────────────────────
+QDRANT_HOST=localhost
+QDRANT_PORT=6333
+# QDRANT_API_KEY=your_api_key_here        # for Qdrant Cloud
+
+# ── Neo4j (Graph DB) ──────────────────────────────────────────────────────────
 NEO4J_URI=bolt://localhost:7687
 NEO4J_USER=neo4j
 NEO4J_PASSWORD=taarya123
 
-POSTGRES_HOST=localhost
-POSTGRES_PORT=5432
-POSTGRES_DB=astronomy
-POSTGRES_USER=taarya
-POSTGRES_PASSWORD=taarya123
+# ── Embedding Model ──────────────────────────────────────────────────────────
+EMBEDDING_MODEL=all-MiniLM-L6-v2
 
-QDRANT_HOST=localhost
-QDRANT_PORT=6333
-
-# Embedding Model
-EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
+# ── LLM (optional — for agentic chat interface) ──────────────────────────────
+OPENAI_API_KEY=your_openai_key_here
+# ANTHROPIC_API_KEY=your_anthropic_key_here
+# OLLAMA_BASE_URL=http://localhost:11434
 
-# Application
+# ── Server ───────────────────────────────────────────────────────────────────
+HOST=0.0.0.0
+PORT=8000
+WORKERS=1
 LOG_LEVEL=INFO
-ENVIRONMENT=development
+
+# ── SIMBAD / Astroquery ──────────────────────────────────────────────────────
+SIMBAD_TIMEOUT=10
+
+# ── Development ───────────────────────────────────────────────────────────────
+DEBUG=false

.github/workflows/regression.yml

@@ -0,0 +1,135 @@
+###############################################################################
+# TaarYa Regression Suite
+#
+# Runs on every push / PR to main.
+# Uses two job tiers:
+#   1. unit-tests      — Pure-Python logic (metrics, NER, streaming, exports)
+#   2. eval-offline    — Offline ablation + discovery validation (no backends)
+#
+# Live integration tests (Postgres/Qdrant/Neo4j) are skipped by default
+# and guarded by `pytest.mark.skipif` probes in the test files.
+###############################################################################
+
+name: TaarYa Regression
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  # -------------------------------------------------------------------
+  # Job 1: Unit & integration tests (no external services)
+  # -------------------------------------------------------------------
+  unit-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install pytest pytest-cov
+
+      - name: Run unit tests
+        run: |
+          python -m pytest tests/ \
+            -v \
+            --tb=short \
+            --strict-markers \
+            -x \
+            --ignore=tests/test_backends.py \
+            2>&1
+        env:
+          PYTHONPATH: .
+          # Prevent GPU / CUDA initialization in CI
+          CUDA_VISIBLE_DEVICES: ""
+          TF_CPP_MIN_LOG_LEVEL: "3"
+
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results
+          path: |
+            .pytest_cache/
+            htmlcov/
+          retention-days: 7
+
+  # -------------------------------------------------------------------
+  # Job 2: Offline evaluation (ablation + discovery precision)
+  # -------------------------------------------------------------------
+  eval-offline:
+    runs-on: ubuntu-latest
+    needs: unit-tests
+    timeout-minutes: 10
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Run offline ablation study
+        run: |
+          python eval/ablation_formal.py --offline --publish \
+            --output eval/ablation_results.json
+        env:
+          PYTHONPATH: .
+
+      - name: Run offline discovery validation
+        run: |
+          python eval/validate_discovery.py --offline \
+            --output eval/discovery_validation.json
+        env:
+          PYTHONPATH: .
+
+      - name: Upload evaluation artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: evaluation-results
+          path: |
+            eval/ablation_results.json
+            eval/ablation_table.tex
+            eval/discovery_validation.json
+          retention-days: 30
+
+      - name: Verify ablation output
+        run: |
+          python -c "
+          import json, sys
+          with open('eval/ablation_results.json') as f:
+              data = json.load(f)
+          configs = data.get('configurations', [])
+          assert len(configs) == 4, f'Expected 4 configs, got {len(configs)}'
+          print(f'✅ Ablation: {len(configs)} configurations × {len(data.get(\"regions\", []))} regions')
+
+          with open('eval/discovery_validation.json') as f:
+              dv = json.load(f)
+          regions = dv.get('regions', [])
+          assert len(regions) >= 5, f'Expected >=5 regions, got {len(regions)}'
+          print(f'✅ Discovery: {len(regions)} regions validated')
+          "

.gitignore

@@ -48,6 +48,27 @@ docker-compose.override.yml
 # Project documents and reports
 stitch_research_command_center/
 CorrectedReportContent.pdf
+
+# Distribution artifacts
+dist/
+build/
+*.egg-info/
+*.whl
+
+# Pytest / Coverage
+.pytest_cache/
+.coverage
+htmlcov/
+.tox/
+
+# Generated eval output
+eval/ablation_results.json
+eval/ablation_table.tex
+eval/*_output/
+provenance_*.zip
+
+# Docker data volumes
+taarya-*/
 project ppt sample.ppt
 RAG_Astronomy_Review.pptx
 TaarYa_Major_Project_Report_ES452.docx

CHANGELOG.md

@@ -0,0 +1,55 @@
+# Changelog
+
+All notable changes to TaarYa are documented in this file.
+
+Format based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [1.0.0] — 2026-04-21
+
+### Added
+
+- **Hybrid retrieval engine** combining PostgreSQL/Q3C (spatial), Qdrant (semantic), and Neo4j (graph) backends behind a unified orchestration layer.
+- **Discovery ranking** with 3 configurable profiles (`strict`, `balanced`, `aggressive`), 7 signal families, and uncertainty-aware scoring (SNR-based parallax penalties, confidence bands).
+- **Formal ablation study** (`eval/ablation_formal.py`): 4 configurations × 5 sky regions with IR metrics (P@k, R@k, MRR, nDCG, F1). Live result: **+300.5% F1@10** and **+575.2% MRR** vs spatial-only baseline.
+- **50 benchmark queries** with ground truth labels across 12 task categories (`eval/benchmark_queries.json`).
+- **Discovery validation** (`eval/validate_discovery.py`): 6 anomaly types × 5 regions.
+- **ArXiv corpus expansion pipeline** (`src/ingestion/arxiv_ingest.py`): 50 queries × 10 thematic categories, batch upserts, resumable ingestion. Current corpus: **6,464 papers**.
+- **Time-domain alert ingestion** (`src/ingestion/alert_ingest.py`): TNS and ALeRCE broker support with spatial cross-matching to Gaia stars.
+- **Extended sky coverage** (`scripts/expand_sky_coverage.py`): 30 additional regions across 6 types (GC, OB, SFR, MG, OC, CAL) with optional Gaia TAP+ fetching.
+- **Publication-grade HR diagrams** with evolutionary track overlays (ZAMS, RGB, WD, HB) citing Pecaut & Mamajek (2013), Bressan+ (2012), Bergeron+ (2011), Catelan+ (2009).
+- **Cross-catalog overlap analysis** endpoint (`GET /api/catalog/overlap`).
+- **VO interoperability**: DS9 region export, MESA inlists, SAMP broadcast, Aladin deep links, VOTable/CSV/JSON export.
+- **Provenance logging**: `X-TaarYa-Session-Id` headers, session manifests, `.zip` bundle export.
+- **CI/CD pipeline**: GitHub Actions 2-tier workflow (unit-tests → eval-offline), local regression harness (`scripts/run_regression.py`).
+- **CLI entry points**: `taarya-server`, `taarya-ingest`, `taarya-readiness`, `taarya-ablation`, `taarya-corpus`.
+- **Graph entity linking**: Multi-strategy NER (Gaia DR3 prefix matching, catalog ID resolution, common star name lookup, SIMBAD fallback).
+- **Uncertainty propagation**: First-order δ-method for derived absolute magnitudes with formal SNR gating.
+
+### Fixed
+
+- `SpatialSearch.nearby_stars()` API signature mismatch (`radius_deg` → `radius`).
+- `datetime.utcnow()` deprecation (Python 3.12) replaced with `datetime.now(timezone.utc)` across 7 files.
+- `ALIASED_CLUSTERS` moved from function scope to module level in `graph_ingest.py`.
+- Corrupt import ordering and duplicate logger line in `graph_ingest.py`.
+- `transformers` dependency pinned to `>=4.40,<5` to prevent circular import crashes.
+- Embedding model loading probe in test skip guards to prevent CI-time import failures.
+
+### Changed
+
+- Benchmark queries expanded from 35 to 50 with full ground truth labels.
+- NER extractor expanded to 200+ common star names and 40+ local coordinate entries.
+- Paper corpus expanded from 1,475 to 6,464 papers.
+- Sky coverage expanded from 8 to 38 regions.
+- Test suite: **38 passed, 0 failed**.
+
+## [0.1.0] — 2026-04-01
+
+### Added
+
+- Initial TaarYa implementation with FastAPI backend.
+- PostgreSQL/Q3C spatial search, Qdrant vector search, Neo4j graph search.
+- LangChain-based agentic query routing.
+- Glassmorphism web dashboard (Chat, Explore, Settings tabs).
+- Basic NER extractor and SIMBAD validation.
+- Gaia DR3 catalog ingestion pipeline.

Dockerfile

@@ -0,0 +1,87 @@
+# =============================================================================
+# TaarYa — Multi-stage production Dockerfile
+# =============================================================================
+# Stage 1 (builder): compile heavy dependencies (astropy, sentence-transformers)
+# Stage 2 (runtime): slim production image
+# =============================================================================
+
+# ── Stage 1: Builder ──────────────────────────────────────────────────────────
+FROM python:3.12-slim AS builder
+
+WORKDIR /build
+
+# Install build dependencies (no root package manager during pip install)
+RUN apt-get update && apt-get install --no-install-recommends -y \
+        gcc \
+        g++ \
+        libpq-dev \
+        cmake \
+        pkg-config \
+        libfreetype6-dev \
+        libjpeg-dev \
+        zlib1g-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install uv for fast dependency management
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+
+# Pre-install heavy astronomy dependencies (pre-built wheels = faster rebuilds)
+RUN uv pip install --system --no-cache \
+        "numpy>=1.26.3" \
+        "scipy>=1.11.0"
+
+# Copy and install the project (editable for entrypoints to resolve)
+COPY pyproject.toml ./
+RUN uv pip install --system --no-cache --editable ".[all]"
+
+# ── Stage 2: Runtime ──────────────────────────────────────────────────────────
+FROM python:3.12-slim AS runtime
+
+LABEL maintainer="Amit Kumar <[email protected]>"
+LABEL description="TaarYa — Hybrid Retrieval Extension for Cross-Catalog Astronomical Discovery Support"
+LABEL homepage="https://github.com/HmbleCreator/TaarYa"
+LABEL version="1.0.0"
+
+# Non-root user for security
+RUN groupadd --gid 1000 taarya \
+    && useradd --uid 1000 --gid taarya --shell /bin/bash --create-home taarya
+
+WORKDIR /app
+
+# Install runtime dependencies only (no compiler needed)
+RUN apt-get update && apt-get install --no-install-recommends -y \
+        libpq5 \
+        libgomp1 \
+        libfreetype6 \
+        libjpeg-turbo \
+        fonts-dejavu-core \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy pre-built site-packages from builder
+COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages
+COPY --from=builder /usr/local/bin /usr/local/bin
+
+# Copy static files (required for web UI)
+COPY static/ /app/static/
+
+# Copy config templates
+COPY .env.example /app/.env.example
+
+# Pre-create data directory
+RUN mkdir -p /app/data && chown taarya:taarya /app/data
+
+# Switch to non-root user
+USER taarya
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \
+    CMD python -c "import httpx; r=httpx.get('http://localhost:8000/health', timeout=5); assert r.status_code==200" \
+    || exit 1
+
+# Default: start the API server
+EXPOSE 8000
+
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
+
+CMD ["python", "-m", "uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8000"]