feat: FHIR Bulk Data $import and CLI backup/restore

README.md

@@ -67,8 +67,8 @@ Early-stage, under active development.
 | | |
 |---|---|
 | **Done** | `types`, `proto`, `store`, `sqlite`, `postgres`, `core`, `validate`, `fhirpath`, `registry`, `sync`, `modules`, `view`, `ai`, `auth`, `jobs`, `audit`, `smart`, `subscriptions`, `http`, `runtime`, `client` — CRUD, history, transaction bundles, atomic writes, structural validation, FHIRPath, FHIR definition catalog, device-to-hub push/pull, manifest-driven module installer, ViewDefinition execution, policy-governed AI tool harness, shared identity and policy library, shared job runtime, shared audit event library, optional SMART on FHIR (scopes, tokens, auth adapters), change-triggered workflows with webhook/local delivery, FHIR REST HTTP adapter, runtime composition, and Go client SDK |
-| **Partial** | `cli` (operator surface usable; backup/restore and conflict drill-down remain) |
-| **Next (Stage 1)** | `testkit`, CLI backup/restore and conflict drill-down |
+| **Partial** | `cli` (operator surface usable; conflict drill-down remains) |
+| **Next (Stage 1)** | `testkit`, CLI conflict drill-down |
 
 Durable job and audit persistence is available via `pkg/postgres` and `pkg/sqlite`. Shared runtime helpers live in `pkg/jobs` and `pkg/audit`. See [Roadmap](#roadmap) for the full plan.
 

cmd/haistack/README.md

@@ -19,7 +19,7 @@ It does **not**:
 - Replace `pkg/runtime` composition logic (commands call the runtime builder)
 - Implement FHIR business rules or persistence (those live in `pkg/*`)
 - Call remote sync hubs for health checks (`sync status` is local-store only)
-- Provide backup/restore or conflict drill-down (planned later)
+- Provide conflict drill-down (planned later)
 
 ## When to use it
 
@@ -71,6 +71,8 @@ haistack serve
 | `haistack serve` | Build `pkg/runtime`, start HTTP, print bound address. |
 | `haistack validate <file>` | Structural validation via `validate.Engine`. Exits non-zero when invalid. |
 | `haistack import <file>` | Import one JSON resource; use `--create-only` or `--update-only` to control conflicts. |
+| `haistack backup [dir]` | Write NDJSON files plus `manifest.json` for stored resources (default dir `backup`). |
+| `haistack restore [dir]` | Reload an NDJSON backup directory with create-or-update. |
 | `haistack read <ResourceType/id>` | Read one stored resource. |
 | `haistack delete <ResourceType/id> --force` | Delete one stored resource. |
 | `haistack export <ResourceType[/id]>` | Export one resource or a resource collection as JSON. |

cmd/haistack/command/backup.go

@@ -0,0 +1,74 @@
+package command
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/degoke/health-ai-stack/cmd/haistack/internal/app"
+	"github.com/spf13/cobra"
+)
+
+func newBackupCommand(opts *Options, printer *app.Printer) *cobra.Command {
+	return &cobra.Command{
+		Use:   "backup [dir]",
+		Short: "Write an NDJSON backup of stored FHIR resources",
+		Args:  cobra.MaximumNArgs(1),
+		Example: `  haistack backup
+  haistack backup ./snapshots/today --output json`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			dir := "backup"
+			if len(args) == 1 {
+				dir = args[0]
+			}
+			ctx := context.Background()
+			session, err := openSession(opts, printer, ctx)
+			if err != nil {
+				return err
+			}
+			defer func() { _ = session.Close(ctx) }()
+			report, err := session.Backup(ctx, dir)
+			if err != nil {
+				return exitErr(printer, err)
+			}
+			if printer.Format == app.OutputJSON {
+				return printer.Print(report)
+			}
+			writeStdout(printer, fmt.Sprintf("backed up %d resource type(s) to %s", len(report.Files), report.Directory))
+			for _, file := range report.Files {
+				writeStdout(printer, fmt.Sprintf("  %s %d", file.File, file.Count))
+			}
+			return nil
+		},
+	}
+}
+
+func newRestoreCommand(opts *Options, printer *app.Printer) *cobra.Command {
+	return &cobra.Command{
+		Use:   "restore [dir]",
+		Short: "Restore FHIR resources from an NDJSON backup directory",
+		Args:  cobra.MaximumNArgs(1),
+		Example: `  haistack restore
+  haistack restore ./snapshots/today`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			dir := "backup"
+			if len(args) == 1 {
+				dir = args[0]
+			}
+			ctx := context.Background()
+			session, err := openSession(opts, printer, ctx)
+			if err != nil {
+				return err
+			}
+			defer func() { _ = session.Close(ctx) }()
+			report, err := session.Restore(ctx, dir)
+			if err != nil {
+				return exitErr(printer, err)
+			}
+			if printer.Format == app.OutputJSON {
+				return printer.Print(report)
+			}
+			writeStdout(printer, fmt.Sprintf("restored %s: created %d, updated %d", report.Directory, report.Created, report.Updated))
+			return nil
+		},
+	}
+}

cmd/haistack/command/command_test.go

@@ -235,6 +235,50 @@ func TestImportCreatesThenUpdates(t *testing.T) {
 	}
 }
 
+func TestBackupRestoreRoundTrip(t *testing.T) {
+	dir := t.TempDir()
+	wd, _ := os.Getwd()
+	defer func() { _ = os.Chdir(wd) }()
+
+	coreModule := repoCoreModule(t)
+	writeConfig(t, dir, filepath.Join(dir, "test.db"), coreModule)
+
+	patientPath := filepath.Join(dir, "patient.json")
+	if err := os.WriteFile(patientPath, []byte(`{"resourceType":"Patient","id":"bak-1","name":[{"family":"Backup"}]}`), 0o644); err != nil {
+		t.Fatalf("write patient: %v", err)
+	}
+	if _, _, err := runCLI(t, dir, "import", patientPath); err != nil {
+		t.Fatalf("import: %v", err)
+	}
+
+	backupDir := filepath.Join(dir, "snap")
+	stdout, _, err := runCLI(t, dir, "backup", backupDir, "--output", "json")
+	if err != nil {
+		t.Fatalf("backup: %v", err)
+	}
+	if !strings.Contains(stdout, "Patient.ndjson") {
+		t.Fatalf("backup stdout = %q", stdout)
+	}
+	if _, err := os.Stat(filepath.Join(backupDir, "manifest.json")); err != nil {
+		t.Fatalf("manifest: %v", err)
+	}
+
+	if _, _, err := runCLI(t, dir, "delete", "Patient/bak-1", "--force"); err != nil {
+		t.Fatalf("delete: %v", err)
+	}
+	restoreOut, _, err := runCLI(t, dir, "restore", backupDir, "--output", "json")
+	if err != nil {
+		t.Fatalf("restore: %v", err)
+	}
+	if !strings.Contains(restoreOut, `"created"`) {
+		t.Fatalf("restore stdout = %q", restoreOut)
+	}
+	readOut, _, err := runCLI(t, dir, "read", "Patient/bak-1", "--output", "json")
+	if err != nil || !strings.Contains(readOut, "Backup") {
+		t.Fatalf("read after restore: output=%q err=%v", readOut, err)
+	}
+}
+
 func TestSearchParsesKeyValueArgs(t *testing.T) {
 	dir := t.TempDir()
 	wd, _ := os.Getwd()

cmd/haistack/command/doc.go

@@ -10,6 +10,7 @@
 //   - serve — start managed HTTP runtime
 //   - validate — structural FHIR validation
 //   - import — create or update one JSON resource file with an explicit conflict policy
+//   - backup, restore — NDJSON dump and reload of stored resources
 //   - read, delete, export — resource inspection and file-safe data operations
 //   - search — FHIR search with key=value parameters
 //   - fhirpath eval — evaluate a FHIRPath expression

cmd/haistack/command/root.go

@@ -90,6 +90,8 @@ against a Health AI Stack runtime. Configure storage and capabilities in haistac
 		newReadCommand(opts, printer),
 		newDeleteCommand(opts, printer),
 		newExportCommand(opts, printer),
+		newBackupCommand(opts, printer),
+		newRestoreCommand(opts, printer),
 		newSearchCommand(opts, printer),
 		newFHIRPathCommand(opts, printer),
 		newSyncCommand(opts, printer),

cmd/haistack/internal/app/operator.go

@@ -1,9 +1,15 @@
 package app
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
+	"os"
+	"path/filepath"
 	"sort"
+	"strings"
+	"time"
 
 	"github.com/degoke/health-ai-stack/pkg/store"
 	"github.com/degoke/health-ai-stack/pkg/types"
@@ -72,6 +78,191 @@ func (s *Session) ExportResources(ctx context.Context, resourceType string, limi
 	return out, nil
 }
 
+// BackupManifest describes a directory of NDJSON resource dumps.
+type BackupManifest struct {
+	Format    string               `json:"format"`
+	CreatedAt time.Time            `json:"createdAt"`
+	Files     []BackupManifestFile `json:"files"`
+}
+
+// BackupManifestFile is one resource-type NDJSON file in a backup.
+type BackupManifestFile struct {
+	Type  string `json:"type"`
+	Count int    `json:"count"`
+	File  string `json:"file"`
+}
+
+// BackupReport is the CLI result of writing a backup directory.
+type BackupReport struct {
+	Directory string               `json:"directory"`
+	Files     []BackupManifestFile `json:"files"`
+}
+
+const backupFormatV1 = "haistack-backup/v1"
+
+// Backup writes one NDJSON file per resource type plus manifest.json.
+func (s *Session) Backup(ctx context.Context, dir string) (*BackupReport, error) {
+	if dir == "" {
+		dir = "backup"
+	}
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return nil, fmt.Errorf("create backup directory: %w", err)
+	}
+	manifest := BackupManifest{Format: backupFormatV1, CreatedAt: time.Now().UTC()}
+	for _, resourceType := range s.backupResourceTypes() {
+		resources, err := s.ExportResources(ctx, resourceType, 0)
+		if err != nil {
+			return nil, err
+		}
+		if len(resources) == 0 {
+			continue
+		}
+		filename := resourceType + ".ndjson"
+		var buf bytes.Buffer
+		for i, resource := range resources {
+			if i > 0 {
+				_ = buf.WriteByte('\n')
+			}
+			_, _ = buf.Write(bytes.TrimSpace(resource.JSON))
+		}
+		if err := os.WriteFile(filepath.Join(dir, filename), buf.Bytes(), 0o644); err != nil {
+			return nil, fmt.Errorf("write %s: %w", filename, err)
+		}
+		manifest.Files = append(manifest.Files, BackupManifestFile{
+			Type:  resourceType,
+			Count: len(resources),
+			File:  filename,
+		})
+	}
+	data, err := json.MarshalIndent(manifest, "", "  ")
+	if err != nil {
+		return nil, err
+	}
+	if err := os.WriteFile(filepath.Join(dir, "manifest.json"), append(data, '\n'), 0o644); err != nil {
+		return nil, fmt.Errorf("write manifest: %w", err)
+	}
+	return &BackupReport{Directory: dir, Files: manifest.Files}, nil
+}
+
+// RestoreReport is the CLI result of loading a backup directory.
+type RestoreReport struct {
+	Directory string `json:"directory"`
+	Created   int    `json:"created"`
+	Updated   int    `json:"updated"`
+	Files     int    `json:"files"`
+}
+
+// Restore loads NDJSON files from a backup directory using create-or-update.
+func (s *Session) Restore(ctx context.Context, dir string) (*RestoreReport, error) {
+	if dir == "" {
+		dir = "backup"
+	}
+	svc := s.Runtime.Services().ResourceService
+	if svc == nil {
+		return nil, fmt.Errorf("resource service is not available")
+	}
+	files, err := s.restoreFiles(dir)
+	if err != nil {
+		return nil, err
+	}
+	report := &RestoreReport{Directory: dir, Files: len(files)}
+	for _, file := range files {
+		data, err := os.ReadFile(filepath.Join(dir, file.File))
+		if err != nil {
+			return nil, fmt.Errorf("read %s: %w", file.File, err)
+		}
+		for i, line := range bytes.Split(data, []byte("\n")) {
+			line = bytes.TrimSpace(line)
+			if len(line) == 0 {
+				continue
+			}
+			env, err := types.NewJSONCodec().ParseJSON(file.Type, line)
+			if err != nil {
+				return nil, fmt.Errorf("%s line %d: %w", file.File, i+1, err)
+			}
+			cleaned, err := types.SetMeta(env.JSON, types.Meta{})
+			if err != nil {
+				return nil, fmt.Errorf("%s line %d: clear meta: %w", file.File, i+1, err)
+			}
+			env.JSON = cleaned
+			env.VersionID = ""
+			action, _, err := ImportResource(ctx, svc, env, false, false)
+			if err != nil {
+				return nil, fmt.Errorf("%s %s/%s: %w", file.File, env.ResourceType, env.ID, err)
+			}
+			switch action {
+			case "create":
+				report.Created++
+			case "update":
+				report.Updated++
+			}
+		}
+	}
+	return report, nil
+}
+
+func (s *Session) backupResourceTypes() []string {
+	if s != nil && s.Runtime != nil && s.Runtime.Services() != nil && s.Runtime.Services().RegistrySnapshot != nil {
+		types := s.Runtime.Services().RegistrySnapshot.EnabledResourceTypes()
+		if len(types) > 0 {
+			return types
+		}
+	}
+	return []string{"Patient", "Observation", "Encounter", "Practitioner", "Organization", "Appointment"}
+}
+
+func (s *Session) restoreFiles(dir string) ([]BackupManifestFile, error) {
+	manifestPath := filepath.Join(dir, "manifest.json")
+	if data, err := os.ReadFile(manifestPath); err == nil {
+		var manifest BackupManifest
+		if err := json.Unmarshal(data, &manifest); err != nil {
+			return nil, fmt.Errorf("parse manifest: %w", err)
+		}
+		return orderRestoreFiles(manifest.Files), nil
+	}
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, fmt.Errorf("read backup directory: %w", err)
+	}
+	var files []BackupManifestFile
+	for _, entry := range entries {
+		name := entry.Name()
+		if entry.IsDir() || !strings.HasSuffix(name, ".ndjson") {
+			continue
+		}
+		files = append(files, BackupManifestFile{
+			Type: strings.TrimSuffix(name, ".ndjson"),
+			File: name,
+		})
+	}
+	return orderRestoreFiles(files), nil
+}
+
+func orderRestoreFiles(files []BackupManifestFile) []BackupManifestFile {
+	priority := map[string]int{
+		"Organization": 0,
+		"Practitioner": 1,
+		"Patient":      2,
+		"Encounter":    3,
+	}
+	out := append([]BackupManifestFile(nil), files...)
+	sort.SliceStable(out, func(i, j int) bool {
+		pi, okI := priority[out[i].Type]
+		if !okI {
+			pi = 50
+		}
+		pj, okJ := priority[out[j].Type]
+		if !okJ {
+			pj = 50
+		}
+		if pi != pj {
+			return pi < pj
+		}
+		return out[i].Type < out[j].Type
+	})
+	return out
+}
+
 // ReindexPlan counts current resources for the selected enabled type(s) without
 // modifying search indexes.
 func (s *Session) ReindexPlan(ctx context.Context, resourceType string) ([]ReindexPlanItem, error) {

pkg/bulkimport/doc.go

@@ -0,0 +1,19 @@
+// Package bulkimport implements FHIR Bulk Data $import orchestration for HAIStack.
+//
+// The import service accepts a Parameters kickoff (Prefer: respond-async),
+// reads application/fhir+ndjson inputs, and creates or updates resources
+// through a ResourceWriter. Background execution integrates with pkg/jobs
+// via TypeImportBulk.
+//
+// Remote input.url values are fetched only when a URLLoader is configured.
+// The runtime wires HTTPLoader, which GETs http and https URLs with a 60s
+// timeout and a 64MiB size limit. file:// and other non-http(s) schemes are
+// rejected; redirects to those schemes are also refused. Tests may omit the
+// loader so URL-only kickoffs fail closed.
+//
+// HTTP adapters in pkg/http expose:
+//   - POST /fhir/$import
+//   - GET /fhir/$import/status/{jobId}
+//   - DELETE /fhir/$import/status/{jobId}
+//   - GET /fhir/$import/files/{jobId}/{filename} — download import error NDJSON
+package bulkimport