From 14f1f523f05a9698fe4546541f5a96801ab9de4d Mon Sep 17 00:00:00 2001 From: Eduardo Flores Date: Fri, 16 Oct 2026 19:38:52 -0600 Subject: [PATCH 1/2] style(docs): apply mdformat to plugin lifecycle docs Formatting-only change (mdformat --wrap 100) so the following behavior change carries a content-only docs diff. --- .../ADR/001-dynamic-job-loading-system.md | 494 ++++++++++++------ .../docs/job-plugins/developer-guide.md | 204 ++++---- fr-batch-service/docs/production-checklist.md | 227 +++++--- fr-batch-service/docs/runbook.md | 88 ++-- 4 files changed, 659 insertions(+), 354 deletions(-) diff --git a/fr-batch-service/docs/ADR/001-dynamic-job-loading-system.md b/fr-batch-service/docs/ADR/001-dynamic-job-loading-system.md index c8d68074..5be72462 100644 --- a/fr-batch-service/docs/ADR/001-dynamic-job-loading-system.md +++ b/fr-batch-service/docs/ADR/001-dynamic-job-loading-system.md @@ -1,29 +1,47 @@ # ADR 001: Dynamic Job Loading System with Database-Driven Registry ## Status + **Implemented (Phase 1-6)** — 2026-05-31. Original proposal: 2026-03-02. -Phase 1 (Foundation), Phase 2 (Core Integration), Phase 3 (Dynamic Loading API Layer), Phase 4 (Dynamic Job Loading), Phase 5 (Security and Hardening), and Phase 6 (Example, Documentation, Health & Metrics) are complete. Phase 3 built the REST API, persistence, and JAR upload infrastructure for runtime job management — REST endpoints, Flyway migration, JPA entities, JAR validation, and global error handling. Phase 4 delivers the `DynamicJobClassLoader`, `DynamicJobLoaderService`, and load/unload/reload lifecycle. Phase 5 closes critical security gaps with Spring Security HTTP Basic auth, checksum re-verification at load, JAR signature verification, approval workflow, and audit logging. Phase 6 adds observability (health indicator, Micrometer metrics), a standalone example plugin project (`example-payment-job/`), updated developer docs, and a production readiness checklist. Phase 7 remains as future work. +Phase 1 (Foundation), Phase 2 (Core Integration), Phase 3 (Dynamic Loading API Layer), Phase 4 +(Dynamic Job Loading), Phase 5 (Security and Hardening), and Phase 6 (Example, Documentation, Health +& Metrics) are complete. Phase 3 built the REST API, persistence, and JAR upload infrastructure for +runtime job management — REST endpoints, Flyway migration, JPA entities, JAR validation, and global +error handling. Phase 4 delivers the `DynamicJobClassLoader`, `DynamicJobLoaderService`, and +load/unload/reload lifecycle. Phase 5 closes critical security gaps with Spring Security HTTP Basic +auth, checksum re-verification at load, JAR signature verification, approval workflow, and audit +logging. Phase 6 adds observability (health indicator, Micrometer metrics), a standalone example +plugin project (`example-payment-job/`), updated developer docs, and a production readiness +checklist. Phase 7 remains as future work. See [Revised Approach](#revised-approach) below for the actual architecture. ## Context and Problem Statement -The current fr-batch-service implementation has several limitations that hinder scalability and operational flexibility: +The current fr-batch-service implementation has several limitations that hinder scalability and +operational flexibility: ### Current Limitations -1. **Hardcoded Job Registry**: Jobs are manually registered in `JobsManagerService.manualDefinedJobs` HashMap, requiring code changes and redeployment for new jobs -2. **Monolithic Architecture**: All job logic resides in the same codebase, making it difficult for teams to work independently -3. **No Version Management**: Cannot run multiple versions of the same job or roll back to previous versions -4. **Tight Coupling**: Job implementations are tightly coupled to the core batch service -5. **Limited Reusability**: Jobs cannot be shared across different batch service instances or environments -6. **Manual Deployment**: Adding new jobs requires full application rebuild and deployment +1. **Hardcoded Job Registry**: Jobs are manually registered in + `JobsManagerService.manualDefinedJobs` HashMap, requiring code changes and redeployment for new + jobs +1. **Monolithic Architecture**: All job logic resides in the same codebase, making it difficult for + teams to work independently +1. **No Version Management**: Cannot run multiple versions of the same job or roll back to previous + versions +1. **Tight Coupling**: Job implementations are tightly coupled to the core batch service +1. **Limited Reusability**: Jobs cannot be shared across different batch service instances or + environments +1. **Manual Deployment**: Adding new jobs requires full application rebuild and deployment ### Business Drivers -- **Faster Time-to-Market**: Business teams need to deploy new batch jobs without waiting for core platform releases -- **Team Autonomy**: Multiple teams should be able to develop and deploy their own batch jobs independently +- **Faster Time-to-Market**: Business teams need to deploy new batch jobs without waiting for core + platform releases +- **Team Autonomy**: Multiple teams should be able to develop and deploy their own batch jobs + independently - **Flexibility**: Ability to enable/disable jobs dynamically based on business needs - **Operational Safety**: Ability to test jobs in isolation and roll back problematic versions - **Multi-tenancy**: Support for different job sets per environment or customer @@ -33,10 +51,10 @@ The current fr-batch-service implementation has several limitations that hinder We will implement a **Plugin-Based Dynamic Job Loading System** that allows batch jobs to be: 1. Developed as separate, independent Maven/Gradle projects -2. Packaged as executable JAR files -3. Uploaded and registered dynamically at runtime -4. Stored and managed through a database registry -5. Loaded, unloaded, and reloaded without service restart +1. Packaged as executable JAR files +1. Uploaded and registered dynamically at runtime +1. Stored and managed through a database registry +1. Loaded, unloaded, and reloaded without service restart ### Key Components @@ -111,9 +129,13 @@ We will implement a **Plugin-Based Dynamic Job Loading System** that allows batc ## Revised Approach (Implemented) -The original proposal envisioned **dynamic JAR loading with classloader isolation** — jobs packaged as external JARs, uploaded at runtime, loaded by a custom `DynamicJobClassLoader`, and registered into a database-driven registry. That approach remains the long-term vision (Phases 3-7). +The original proposal envisioned **dynamic JAR loading with classloader isolation** — jobs packaged +as external JARs, uploaded at runtime, loaded by a custom `DynamicJobClassLoader`, and registered +into a database-driven registry. That approach remains the long-term vision (Phases 3-7). -**What was actually built** is a pragmatic first step: a **classpath-based plugin registry** that solves the immediate problem — eliminating the hardcoded job map in `JobsManagerService` — without introducing classloader complexity. +**What was actually built** is a pragmatic first step: a **classpath-based plugin registry** that +solves the immediate problem — eliminating the hardcoded job map in `JobsManagerService` — without +introducing classloader complexity. ### Actual Architecture (Phase 1-2) @@ -160,49 +182,70 @@ The original proposal envisioned **dynamic JAR loading with classloader isolatio ### Key Differences from Original Proposal -| Aspect | Original ADR | Actual Implementation | -|---|---|---| -| Plugin discovery | Dynamic JAR scan + SPI | Spring bean injection (`List`) | -| Class loading | Custom `DynamicJobClassLoader` per JAR | Shared classpath (no isolation) | -| Registration | Database-driven + REST upload API | In-memory `MapJobRegistry` at startup | -| Hot reload | Unload → reload cycle | Requires app restart | -| Plugin location | External JAR in `/var/batch-jobs/` | Same codebase (`@Component` in `fr-batch-service`) | -| Version management | Multiple versions coexist | One version per deployment | -| Complexity | Very high (classloader, memory mgmt, SPI) | Low (plain Spring beans) | +| Aspect | Original ADR | Actual Implementation | +| ------------------ | ----------------------------------------- | -------------------------------------------------- | +| Plugin discovery | Dynamic JAR scan + SPI | Spring bean injection (`List`) | +| Class loading | Custom `DynamicJobClassLoader` per JAR | Shared classpath (no isolation) | +| Registration | Database-driven + REST upload API | In-memory `MapJobRegistry` at startup | +| Hot reload | Unload → reload cycle | Requires app restart | +| Plugin location | External JAR in `/var/batch-jobs/` | Same codebase (`@Component` in `fr-batch-service`) | +| Version management | Multiple versions coexist | One version per deployment | +| Complexity | Very high (classloader, memory mgmt, SPI) | Low (plain Spring beans) | ### Why This Approach First -1. **Solves the immediate problem**: `JobsManagerService` no longer has a hardcoded job map. Adding a new job means creating a `@Component` that implements `BatchJobPlugin` — no other code changes. -2. **Zero operational risk**: No classloader leaks, no JAR validation, no upload security surface. Plain Spring beans behave exactly like the rest of the application. -3. **Progressive complexity**: The `PluginRegistryService` contract (`getPlugins()`, `getPlugin(name)`) is the same abstraction a future dynamic loader would populate. Swap the backend, keep the API. -4. **Testable today**: Integration tests can `@Autowired List` and verify the full pipeline end-to-end with `ExitStatus.COMPLETED`. +1. **Solves the immediate problem**: `JobsManagerService` no longer has a hardcoded job map. Adding + a new job means creating a `@Component` that implements `BatchJobPlugin` — no other code changes. +1. **Zero operational risk**: No classloader leaks, no JAR validation, no upload security surface. + Plain Spring beans behave exactly like the rest of the application. +1. **Progressive complexity**: The `PluginRegistryService` contract (`getPlugins()`, + `getPlugin(name)`) is the same abstraction a future dynamic loader would populate. Swap the + backend, keep the API. +1. **Testable today**: Integration tests can `@Autowired List` and verify the full + pipeline end-to-end with `ExitStatus.COMPLETED`. ### ADR Decisions Made During Implementation **ADR-I1: Classpath-based discovery over dynamic JAR loading for Phase 2** -- **Decision**: Use Spring's existing bean injection (`List`) instead of custom classloaders. -- **Rationale**: The business need is removing the hardcoded job map, not runtime JAR upload. Dynamic classloading adds 10x complexity for a feature (hot reload) that has no production demand yet. The plugin contract is the same either way — when dynamic loading is needed, the registry service API stays stable. -- **Trade-off**: Plugins live in the same codebase and classpath. True team isolation (separate repos, independent deploy) is deferred. -- **Rejected**: Building the full `DynamicJobClassLoader` infrastructure now, which would have delayed the plugin migration by weeks and introduced memory-leak risk before the basic contract was validated. +- **Decision**: Use Spring's existing bean injection (`List`) instead of custom + classloaders. +- **Rationale**: The business need is removing the hardcoded job map, not runtime JAR upload. + Dynamic classloading adds 10x complexity for a feature (hot reload) that has no production demand + yet. The plugin contract is the same either way — when dynamic loading is needed, the registry + service API stays stable. +- **Trade-off**: Plugins live in the same codebase and classpath. True team isolation (separate + repos, independent deploy) is deferred. +- **Rejected**: Building the full `DynamicJobClassLoader` infrastructure now, which would have + delayed the plugin migration by weeks and introduced memory-leak risk before the basic contract + was validated. **ADR-I2: @MockitoBean over @MockBean for Spring Boot 4.0.0** -- **Decision**: Use `org.springframework.test.context.bean.override.mockito.MockitoBean` for test mocking. -- **Rationale**: Spring Boot 4.0.0 / Spring Framework 7.x relocated the mock annotation. `@MockBean` still exists but `@MockitoBean` is the forward-looking choice for the version this project targets. +- **Decision**: Use `org.springframework.test.context.bean.override.mockito.MockitoBean` for test + mocking. +- **Rationale**: Spring Boot 4.0.0 / Spring Framework 7.x relocated the mock annotation. `@MockBean` + still exists but `@MockitoBean` is the forward-looking choice for the version this project + targets. - **Rejected**: `@MockBean` (deprecated path). **ADR-I3: Assert on Map.get("result") instead of unwrapping JobExecution** -- **Decision**: The integration test assertion reads `result.get("result")` from `JobsManagerService.syncRunJobWithParams()` return map, not a direct `JobExecution` reference. -- **Rationale**: The map IS the public contract of the service. Adding a `JobExecutionListener` or `JobRepository` query to the test expands surface area for no extra signal. -- **Rejected**: Injecting `JobRepository` and querying execution history (brittle ordering), or adding test-only scaffolding. +- **Decision**: The integration test assertion reads `result.get("result")` from + `JobsManagerService.syncRunJobWithParams()` return map, not a direct `JobExecution` reference. +- **Rationale**: The map IS the public contract of the service. Adding a `JobExecutionListener` or + `JobRepository` query to the test expands surface area for no extra signal. +- **Rejected**: Injecting `JobRepository` and querying execution history (brittle ordering), or + adding test-only scaffolding. **ADR-I4: 3-row CSV fixture over 1000 rows** - **Decision**: The test CSV `sample-persons-1k.csv` contains 3 rows despite the filename. -- **Rationale**: The filename is a hardcoded artifact in `CsvReader`. Changing it would require a production code change (out of scope). 3 rows exercise chunk processing without bloating CI runtime. The name does not need to match the size. -- **Rejected**: Generating 1000 rows (slower CI, no extra coverage) or refactoring `CsvReader` to accept a configurable path (production change out of scope). +- **Rationale**: The filename is a hardcoded artifact in `CsvReader`. Changing it would require a + production code change (out of scope). 3 rows exercise chunk processing without bloating CI + runtime. The name does not need to match the size. +- **Rejected**: Generating 1000 rows (slower CI, no extra coverage) or refactoring `CsvReader` to + accept a configurable path (production change out of scope). ## Technical Architecture @@ -322,6 +365,7 @@ Job JAR Classes ``` **Key Features:** + - Each job gets its own `URLClassLoader` instance - Jobs can use different versions of libraries - Spring core classes loaded from parent to share infrastructure @@ -704,6 +748,7 @@ public class JobManagementController { ### 6. Example Job Plugin Project **Project Structure:** + ``` example-payment-job/ ├── pom.xml @@ -724,6 +769,7 @@ example-payment-job/ ``` **pom.xml:** + ```xml @@ -777,6 +823,7 @@ example-payment-job/ ``` **PaymentJobPlugin.java:** + ```java package com.example.payment; @@ -879,6 +926,7 @@ public class PaymentJobPlugin implements BatchJobPlugin { ``` **META-INF/services/com.fronzec.api.BatchJobPlugin:** + ``` com.example.payment.PaymentJobPlugin ``` @@ -886,6 +934,7 @@ com.example.payment.PaymentJobPlugin ### 7. Integration with Existing JobsManagerService **Refactor approach:** + ```java @Service public class JobsManagerService { @@ -922,51 +971,51 @@ public class JobsManagerService { ### Positive 1. **Agility**: Teams can develop and deploy jobs independently -2. **Isolation**: Job failures don't affect other jobs or core platform -3. **Versioning**: Multiple versions can coexist, easy rollback -4. **Scalability**: Core platform doesn't grow with every new job -5. **Flexibility**: Enable/disable jobs without deployment -6. **Testability**: Jobs can be tested in isolation -7. **Reusability**: Jobs can be shared across environments -8. **Auditability**: Complete history of job deployments in database +1. **Isolation**: Job failures don't affect other jobs or core platform +1. **Versioning**: Multiple versions can coexist, easy rollback +1. **Scalability**: Core platform doesn't grow with every new job +1. **Flexibility**: Enable/disable jobs without deployment +1. **Testability**: Jobs can be tested in isolation +1. **Reusability**: Jobs can be shared across environments +1. **Auditability**: Complete history of job deployments in database ### Negative 1. **Complexity**: Significant increase in system complexity -2. **Memory Management**: Risk of memory leaks from classloaders -3. **Debugging Difficulty**: Harder to debug across classloader boundaries -4. **Transaction Boundaries**: Care needed with transaction management across classloaders -5. **Classpath Conflicts**: Potential for subtle bugs due to library version conflicts -6. **Performance Overhead**: Classloader delegation adds minor overhead -7. **Operational Burden**: Need to manage JAR storage, versioning, cleanup -8. **Security Risks**: Uploaded JARs could contain malicious code +1. **Memory Management**: Risk of memory leaks from classloaders +1. **Debugging Difficulty**: Harder to debug across classloader boundaries +1. **Transaction Boundaries**: Care needed with transaction management across classloaders +1. **Classpath Conflicts**: Potential for subtle bugs due to library version conflicts +1. **Performance Overhead**: Classloader delegation adds minor overhead +1. **Operational Burden**: Need to manage JAR storage, versioning, cleanup +1. **Security Risks**: Uploaded JARs could contain malicious code ### Risks and Mitigations -| Risk | Likelihood | Impact | Mitigation | -|------|-----------|--------|------------| -| Memory leaks from classloaders | High | High | Implement proper cleanup, monitoring, automated tests | -| Malicious JAR uploads | Medium | Critical | JAR signing, checksum validation, sandboxing, approval workflow | -| Version conflicts | Medium | Medium | Isolated classloaders, dependency documentation | -| Performance degradation | Low | Medium | Benchmarking, lazy loading, caching | -| Job incompatibility after platform upgrade | Medium | High | Versioned API, compatibility testing, deprecation policy | -| Database corruption | Low | Critical | Regular backups, transaction management, validation | -| Storage exhaustion | Medium | Medium | Quota limits, cleanup policies, monitoring | +| Risk | Likelihood | Impact | Mitigation | +| ------------------------------------------ | ---------- | -------- | --------------------------------------------------------------- | +| Memory leaks from classloaders | High | High | Implement proper cleanup, monitoring, automated tests | +| Malicious JAR uploads | Medium | Critical | JAR signing, checksum validation, sandboxing, approval workflow | +| Version conflicts | Medium | Medium | Isolated classloaders, dependency documentation | +| Performance degradation | Low | Medium | Benchmarking, lazy loading, caching | +| Job incompatibility after platform upgrade | Medium | High | Versioned API, compatibility testing, deprecation policy | +| Database corruption | Low | Critical | Regular backups, transaction management, validation | +| Storage exhaustion | Medium | Medium | Quota limits, cleanup policies, monitoring | ### Trade-offs **Chosen Approach vs. Alternatives:** -| Aspect | Dynamic Loading | Microservices | Script-Based | -|--------|----------------|---------------|--------------| -| Complexity | Medium | High | Low | -| Isolation | Medium | High | Low | -| Performance | Good | Medium (network) | Good | -| Resource Usage | Medium | High | Low | -| Development Effort | High | Very High | Low | -| Operational Overhead | Medium | High | Low | -| Security | Medium | High | Low | -| Spring Integration | Excellent | Medium | Medium | +| Aspect | Dynamic Loading | Microservices | Script-Based | +| -------------------- | --------------- | ---------------- | ------------ | +| Complexity | Medium | High | Low | +| Isolation | Medium | High | Low | +| Performance | Good | Medium (network) | Good | +| Resource Usage | Medium | High | Low | +| Development Effort | High | Very High | Low | +| Operational Overhead | Medium | High | Low | +| Security | Medium | High | Low | +| Spring Integration | Excellent | Medium | Medium | ## Alternatives Considered @@ -975,12 +1024,14 @@ public class JobsManagerService { **Description:** Each job as a separate Spring Boot microservice with REST API **Pros:** + - Complete isolation (separate processes) - Independent scaling - Technology heterogeneity - Standard deployment patterns **Cons:** + - Much higher resource usage (JVM per job) - Network latency for orchestration - Complex service discovery and coordination @@ -994,12 +1045,14 @@ public class JobsManagerService { **Description:** Store job logic as scripts in database, execute via scripting engine **Pros:** + - Maximum flexibility - No JAR packaging needed - Easy to modify jobs - Low complexity **Cons:** + - Security nightmare (arbitrary code execution) - Poor performance compared to compiled code - Difficult to test and debug @@ -1013,12 +1066,14 @@ public class JobsManagerService { **Description:** Compile each job to native binary, execute as separate process **Pros:** + - Best isolation - Fast startup - Low memory footprint - No classloader issues **Cons:** + - Limited Spring Batch support in native mode - Long compilation times - Reflection configuration complexity @@ -1032,12 +1087,14 @@ public class JobsManagerService { **Description:** Continue with monolithic jobs in single codebase **Pros:** + - Simple to understand - No additional complexity - Easy debugging - Good IDE support **Cons:** + - Cannot add jobs without redeployment - Tight coupling - No version management @@ -1048,94 +1105,198 @@ public class JobsManagerService { ## Implementation Plan ### Phase 1: Foundation ✅ COMPLETE (May 2026) + - [x] Create `batch-job-api` module with plugin interfaces -- [x] Design database schema (Flyway removed; schema lives in test resources and external migrations) +- [x] Design database schema (Flyway removed; schema lives in test resources and external + migrations) - [x] Set up artifact repository for API module (GitHub Packages) - [x] Create documentation for job plugin development (`docs/job-plugins/developer-guide.md`) -**Implemented:** `batch-job-api` published to GitHub Packages as `com.fronzec:batch-job-api:0.0.1-SNAPSHOT`. Developer guide written. Flyway migrations exist in `migrations/` but are externalized from the main build — see commit `e675bb9`. +**Implemented:** `batch-job-api` published to GitHub Packages as +`com.fronzec:batch-job-api:0.0.1-SNAPSHOT`. Developer guide written. Flyway migrations exist in +`migrations/` but are externalized from the main build — see commit `e675bb9`. ### Phase 2: Core Integration ✅ COMPLETE (May 2026) — REVISED SCOPE -- [x] Implement `PluginRegistryService` (classpath-based bean discovery, MapJobRegistry registration) + +- [x] Implement `PluginRegistryService` (classpath-based bean discovery, MapJobRegistry + registration) - [x] Migrate `job1` from hardcoded `@Bean Job` to `Job1Plugin` implementing `BatchJobPlugin` -- [x] Refactor `JobsManagerService` to delegate discovery to plugin registry (remove `manualDefinedJobs`) +- [x] Refactor `JobsManagerService` to delegate discovery to plugin registry (remove + `manualDefinedJobs`) - [x] Add `PluginRegistrationException` for fail-fast duplicate/null/name-mismatch detection -- [x] Expose `GET /jobs/plugins` endpoint with plugin metadata (`PluginController`, `PluginInfoResponse`) +- [x] Expose `GET /jobs/plugins` endpoint with plugin metadata (`PluginController`, + `PluginInfoResponse`) - [x] Add unit tests (`PluginRegistryServiceTest`, updated `JobsManagerServiceTest`) - [x] Add integration test (`PluginArchitectureIntegrationTest`) with full ETL pipeline assertion - [x] Fix H2 2.4.x dialect bug in `Step3Reader` (parameterized `IS ?` → literal `IS NULL`) **Deferred from original scope:** + - [ ] `DynamicJobClassLoader` (deferred — no production demand for runtime JAR loading yet) - [ ] `ClassLoaderManager` (deferred) - [ ] `DynamicJobLoaderService` with JAR upload (deferred — replaced by `PluginRegistryService`) - [ ] JPA entities and repositories for job registry (deferred) -**Deliverables:** Plugin registry functional with 5/5 integration tests passing. Any `@Component` implementing `BatchJobPlugin` is auto-discovered at startup. `Job1Plugin` runs 3-step ETL pipeline to `ExitStatus.COMPLETED` in integration tests. See PRs #27, #28, #29. +**Deliverables:** Plugin registry functional with 5/5 integration tests passing. Any `@Component` +implementing `BatchJobPlugin` is auto-discovered at startup. `Job1Plugin` runs 3-step ETL pipeline +to `ExitStatus.COMPLETED` in integration tests. See PRs #27, #28, #29. ### Phase 3: Dynamic Loading API Layer ✅ COMPLETE (May 2026) -- [x] Flyway enabled + `V1__job_registry_tables.sql` migration (3 tables: `job_definitions`, `job_parameters_template`, `job_executions_audit`) -- [x] 3 JPA entities + 3 Spring Data JPA repositories (getter/setter style matching `PersonsEntity` conventions) -- [x] `JarUploadService` with JAR extension validation + ZIP magic bytes (`PK\x03\x04`), SHA-256 checksum, path traversal protection, duplicate job-name detection via `JobDefinitionRepository.findByJobName()` -- [x] `JobManagementController` with 6 endpoints: `POST /jobs/upload`, `GET /jobs/definitions`, `GET /jobs/definitions/{id}`, `PUT /jobs/definitions/{id}/enable`, `PUT /jobs/definitions/{id}/disable`, `DELETE /jobs/definitions/{id}` -- [x] `PluginRegistryService` extended with `registerDynamicPlugin()` / `unregisterDynamicPlugin()` — thread-safe via `synchronized(pluginsByJobName)` block, stores `classLoaderRef` for Phase 4 -- [x] `MergedPluginInfoResponse` with `PluginSource` (`CLASSPATH`|`UPLOADED`) and `PluginStatus` (`ACTIVE`|`LOADED`|`ENABLED`|`DISABLED`) enums — merged plugin listing from both classpath + DB sources -- [x] `GlobalExceptionHandler` (`@ControllerAdvice`) with 6 handlers: `InvalidJarException`→400, `DuplicateJobDefinitionException`→409, `NoSuchElementException`→404, `MethodArgumentNotValidException`→400, `MaxUploadSizeExceededException`→413, `Exception`→500 -- [x] `PluginManagementIntegrationTest` with 11 tests covering upload, CRUD, error handling, enable/disable, merged plugin listing, max-size validation + +- [x] Flyway enabled + `V1__job_registry_tables.sql` migration (3 tables: `job_definitions`, + `job_parameters_template`, `job_executions_audit`) +- [x] 3 JPA entities + 3 Spring Data JPA repositories (getter/setter style matching `PersonsEntity` + conventions) +- [x] `JarUploadService` with JAR extension validation + ZIP magic bytes (`PK\x03\x04`), SHA-256 + checksum, path traversal protection, duplicate job-name detection via + `JobDefinitionRepository.findByJobName()` +- [x] `JobManagementController` with 6 endpoints: `POST /jobs/upload`, `GET /jobs/definitions`, + `GET /jobs/definitions/{id}`, `PUT /jobs/definitions/{id}/enable`, + `PUT /jobs/definitions/{id}/disable`, `DELETE /jobs/definitions/{id}` +- [x] `PluginRegistryService` extended with `registerDynamicPlugin()` / `unregisterDynamicPlugin()` + — thread-safe via `synchronized(pluginsByJobName)` block, stores `classLoaderRef` for Phase 4 +- [x] `MergedPluginInfoResponse` with `PluginSource` (`CLASSPATH`|`UPLOADED`) and `PluginStatus` + (`ACTIVE`|`LOADED`|`ENABLED`|`DISABLED`) enums — merged plugin listing from both classpath + DB + sources +- [x] `GlobalExceptionHandler` (`@ControllerAdvice`) with 6 handlers: `InvalidJarException`→400, + `DuplicateJobDefinitionException`→409, `NoSuchElementException`→404, + `MethodArgumentNotValidException`→400, `MaxUploadSizeExceededException`→413, `Exception`→500 +- [x] `PluginManagementIntegrationTest` with 11 tests covering upload, CRUD, error handling, + enable/disable, merged plugin listing, max-size validation - [x] All 42 tests passing (31 existing + 11 new), `mvn test` BUILD SUCCESS **Deferred from Phase 3 (resolved):** + - [x] `DynamicJobClassLoader` — implemented in Phase 4 (PR #35) - [x] `POST /jobs/definitions/{id}/reload` — implemented in Phase 4 (PR #37) - [x] JAR signature verification — implemented in Phase 5 (PR #42) -**Deliverables:** REST API layer for runtime job management fully operational. JAR upload (SHA-256 checksum, validation), CRUD operations, enable/disable toggle, and consistent error handling all production-ready. The `PluginController.getPlugins()` endpoint now merges classpath plugins with DB-registered definitions (`MergedPluginInfoResponse`), providing visibility into both worlds. 4 PRs merged via stacked-to-main chain (PR #30 → #32 → #33 → #34), 35/35 tasks complete, 42/42 tests passing. +**Deliverables:** REST API layer for runtime job management fully operational. JAR upload (SHA-256 +checksum, validation), CRUD operations, enable/disable toggle, and consistent error handling all +production-ready. The `PluginController.getPlugins()` endpoint now merges classpath plugins with +DB-registered definitions (`MergedPluginInfoResponse`), providing visibility into both worlds. 4 PRs +merged via stacked-to-main chain (PR #30 → #32 → #33 → #34), 35/35 tasks complete, 42/42 tests +passing. ### Phase 4: Dynamic Job Loading ✅ COMPLETE (May 2026) -- [x] `DynamicJobClassLoader` — parent-last `URLClassLoader` with shared-package delegation (`org.springframework.*`, `jakarta.*`, `org.slf4j.*`, `com.fronzec.api.*`); `cleanup()` for resource release -- [x] `DynamicJobLoaderService` — orchestrates load/unload/reload lifecycle; validates JAR checksum, creates classloader, loads `BatchJobPlugin` via `mainClassName`, registers via `PluginRegistryService.registerDynamicPlugin()`, tracks `loadStatus` transitions in DB + +- [x] `DynamicJobClassLoader` — parent-last `URLClassLoader` with shared-package delegation + (`org.springframework.*`, `jakarta.*`, `org.slf4j.*`, `com.fronzec.api.*`); `cleanup()` for + resource release +- [x] `DynamicJobLoaderService` — orchestrates load/unload/reload lifecycle; validates JAR checksum, + creates classloader, loads `BatchJobPlugin` via `mainClassName`, registers via + `PluginRegistryService.registerDynamicPlugin()`, tracks `loadStatus` transitions in DB - [x] `POST /jobs/definitions/{id}/load` — load plugin from uploaded JAR; returns 200/404/409/400 -- [x] `POST /jobs/definitions/{id}/unload` — unload and close classloader; `?force=true` stops running executions first; returns 200/404/409 +- [x] `POST /jobs/definitions/{id}/unload` — unload and close classloader; `?force=true` stops + running executions first; returns 200/404/409 - [x] `POST /jobs/definitions/{id}/reload` — atomic unload+load cycle - [x] `POST /jobs/load-all` — bulk load all enabled/unloaded definitions -- [x] `PluginAutoLoader` (ApplicationRunner) — auto-load `enabled=true` definitions at startup; failures logged, do not block startup -- [x] 11 E2E integration tests (`DynamicJobLoadingIntegrationTest`) covering full lifecycle: upload→enable→load→unload→reload→execute→loadAll -- [x] Backward compatibility verified — all 69 pre-Phase-4 tests pass unchanged; classpath `Job1Plugin` continues working +- [x] `PluginAutoLoader` (ApplicationRunner) — auto-load `enabled=true` definitions at startup; + failures logged, do not block startup +- [x] 11 E2E integration tests (`DynamicJobLoadingIntegrationTest`) covering full lifecycle: + upload→enable→load→unload→reload→execute→loadAll +- [x] Backward compatibility verified — all 69 pre-Phase-4 tests pass unchanged; classpath + `Job1Plugin` continues working **Deferred from Phase 4:** + - [x] Health checks for loaded jobs — ✅ COMPLETE (Phase 6, PR #44) - [x] Metrics and monitoring (Micrometer instrumentation) — ✅ COMPLETE (Phase 6, PR #44) - [ ] Performance testing — Phase 7 -**Deliverables:** Dynamic job loading fully operational. Uploaded JARs can be loaded, executed, and unloaded without service restart. `DynamicJobClassLoader` provides classloader isolation with shared-package delegation. `DynamicJobLoaderService` owns the lifecycle with thread-safe `ConcurrentHashMap` tracking. `PluginAutoLoader` restores loaded plugins after restart. 4 PRs merged via stacked-to-main chain (PR #35 → #36 → #37 → #38), 26/26 tasks complete, 80/80 tests passing. +**Deliverables:** Dynamic job loading fully operational. Uploaded JARs can be loaded, executed, and +unloaded without service restart. `DynamicJobClassLoader` provides classloader isolation with +shared-package delegation. `DynamicJobLoaderService` owns the lifecycle with thread-safe +`ConcurrentHashMap` tracking. `PluginAutoLoader` restores loaded plugins after restart. 4 PRs merged +via stacked-to-main chain (PR #35 → #36 → #37 → #38), 26/26 tasks complete, 80/80 tests passing. -**Bug discovered:** `JdbcJobExecutionDao.findRunningJobExecutions()` in Spring Batch 6.0.0 throws `EmptyResultDataAccessException` when `JOB_EXECUTION` records exist but none have running status — the method uses `queryForObject` with a single-row `ResultSetExtractor`, which throws on empty results. Fixed in `DynamicJobLoaderService.unloadJob()` with try-catch guard. See commit `6120f87`. +**Bug discovered:** `JdbcJobExecutionDao.findRunningJobExecutions()` in Spring Batch 6.0.0 throws +`EmptyResultDataAccessException` when `JOB_EXECUTION` records exist but none have running status — +the method uses `queryForObject` with a single-row `ResultSetExtractor`, which throws on empty +results. Fixed in `DynamicJobLoaderService.unloadJob()` with try-catch guard. See commit `6120f87`. ### Phase 5: Security and Hardening ✅ COMPLETE (May 2026) -- [x] **ChecksumUtil** — Extracted SHA-256 computation from `JarUploadService` into reusable static utility with `File`, `Path`, and `MultipartFile` overloads; added checksum re-verification at load time in `DynamicJobLoaderService` (mismatch → `loadStatus=FAILED`, `loadError="Checksum mismatch"`) -- [x] **Spring Security baseline** — Added `spring-boot-starter-security`; `SecurityConfig` with HTTP Basic auth, role-based access (`PLUGIN_ADMIN` = all endpoints, `PLUGIN_VIEWER` = GET only), CSRF disabled; in-memory users via `SecurityProperties`; `GlobalExceptionHandler` for 401/403; security disabled in `test` profile via `@Profile("!test")`; all 88+ existing tests pass unchanged -- [x] **Approval workflow** — `V2__add_approval_fields.sql` Flyway migration adding `approval_status`, `approved_by`, `approved_at` columns; `PUT /jobs/definitions/{id}/approve` and `/reject` endpoints; approval guard in `DynamicJobLoaderService.loadJob()` (`APPROVED` check → 409 if not approved); `AutoApproveConfig` (dev profile); integration test coverage -- [x] **JAR signature verification** — `JarSignatureVerifier` using `java.util.jar.JarFile` with strict/permissive modes (`app.plugins.signature.mode`); strict rejects unsigned JARs, permissive logs WARN and continues; `SignatureResult` record; wired into `JarUploadService.uploadJar()`; 10 unit tests -- [x] **Audit service + JobExecutionListener** — `AuditEventType` enum (8 lifecycle events), `AuditEvent` record, `AuditService` with SLF4J structured JSON logging + MDC correlation IDs; `JobExecutionAuditListener` populates `job_executions_audit` on job start/end; `createdBy` populated via `SecurityContext` resolver with `"system"` fallback; wired into `JarUploadService`, `DynamicJobLoaderService`, `JobManagementController` -- [x] **8 Snyk CVEs resolved** — `spring-boot-starter-security` fixed `CVE-2024-38807` (DoS in Webflux token validation), `CVE-2024-38811` (OIDC session fixation), `CVE-2024-38813` (H2 Console XSS), `CVE-2024-38814` (OAuth2 client state parameter), `CVE-2024-38816` (Spring Web resource directory traversal), `CVE-2024-38819` (Reactive `WebClient` memory leak), `CVE-2024-38821` (Spring WebFlux `@RequestBody` validation bypass), and `CVE-2024-38825` (SSRF via `RestTemplate`/`WebClient`) + +- [x] **ChecksumUtil** — Extracted SHA-256 computation from `JarUploadService` into reusable static + utility with `File`, `Path`, and `MultipartFile` overloads; added checksum re-verification at load + time in `DynamicJobLoaderService` (mismatch → `loadStatus=FAILED`, + `loadError="Checksum mismatch"`) +- [x] **Spring Security baseline** — Added `spring-boot-starter-security`; `SecurityConfig` with + HTTP Basic auth, role-based access (`PLUGIN_ADMIN` = all endpoints, `PLUGIN_VIEWER` = GET only), + CSRF disabled; in-memory users via `SecurityProperties`; `GlobalExceptionHandler` for 401/403; + security disabled in `test` profile via `@Profile("!test")`; all 88+ existing tests pass unchanged +- [x] **Approval workflow** — `V2__add_approval_fields.sql` Flyway migration adding + `approval_status`, `approved_by`, `approved_at` columns; `PUT /jobs/definitions/{id}/approve` and + `/reject` endpoints; approval guard in `DynamicJobLoaderService.loadJob()` (`APPROVED` check → 409 + if not approved); `AutoApproveConfig` (dev profile); integration test coverage +- [x] **JAR signature verification** — `JarSignatureVerifier` using `java.util.jar.JarFile` with + strict/permissive modes (`app.plugins.signature.mode`); strict rejects unsigned JARs, permissive + logs WARN and continues; `SignatureResult` record; wired into `JarUploadService.uploadJar()`; 10 + unit tests +- [x] **Audit service + JobExecutionListener** — `AuditEventType` enum (8 lifecycle events), + `AuditEvent` record, `AuditService` with SLF4J structured JSON logging + MDC correlation IDs; + `JobExecutionAuditListener` populates `job_executions_audit` on job start/end; `createdBy` + populated via `SecurityContext` resolver with `"system"` fallback; wired into `JarUploadService`, + `DynamicJobLoaderService`, `JobManagementController` +- [x] **8 Snyk CVEs resolved** — `spring-boot-starter-security` fixed `CVE-2024-38807` (DoS in + Webflux token validation), `CVE-2024-38811` (OIDC session fixation), `CVE-2024-38813` (H2 Console + XSS), `CVE-2024-38814` (OAuth2 client state parameter), `CVE-2024-38816` (Spring Web resource + directory traversal), `CVE-2024-38819` (Reactive `WebClient` memory leak), `CVE-2024-38821` + (Spring WebFlux `@RequestBody` validation bypass), and `CVE-2024-38825` (SSRF via + `RestTemplate`/`WebClient`) - [x] **99/99 tests passing**, `mvn test` BUILD SUCCESS -**Implementation details**: Layered defense-in-depth — transport auth (HTTP Basic) → integrity (checksum re-verify + JAR signature) → authorization (approval guard). Each layer fails closed. All 5 PRs stacked on `plugin-architecture-phase-5` branch and merged to `plugin-architecture` via stacked-to-main chain. This completes Phase 5. See PRs #39, #40, #41, #42, #43. +**Implementation details**: Layered defense-in-depth — transport auth (HTTP Basic) → integrity +(checksum re-verify + JAR signature) → authorization (approval guard). Each layer fails closed. All +5 PRs stacked on `plugin-architecture-phase-5` branch and merged to `plugin-architecture` via +stacked-to-main chain. This completes Phase 5. See PRs #39, #40, #41, #42, #43. ### Phase 6: Example, Documentation, Health & Metrics ✅ COMPLETE (May 2026) -- [x] **PluginHealthIndicator** — `@Component` implementing `HealthIndicator`; queries `PluginRegistryService.getPlugins().size()`, `JobDefinitionRepository.countByLoadStatus("FAILED")`, `Path.of(jarDir).toFile().canRead()`; returns DOWN if failed > 0 or dir inaccessible; exposed at `/actuator/health/plugins` via `management.endpoint.health.group.plugins.include=pluginHealthIndicator` -- [x] **PluginMetrics** — `@Component` with `@PostConstruct` registering Micrometer counters (`plugins_uploaded_total`, `plugins_loaded_total`, `plugins_unloaded_total`), gauge (`plugins_loaded`), timers (`plugins_load_duration_seconds`, `plugins_unload_duration_seconds`); tagged `plugins_executed_total` counter with `job_name` tag; injected into `JarUploadService`, `DynamicJobLoaderService`, `JobsManagerService` -- [x] **Service instrumentation** — `JarUploadService.uploadJar()` calls `pluginMetrics.incrementUpload()` after DB persist; `DynamicJobLoaderService` calls `incrementLoad()` + `recordLoadDuration()` on load, `incrementUnload()` + `recordUnloadDuration()` on unload; `JobsManagerService.syncRunJobWithParams()` calls `pluginMetrics.incrementExecuted(request.getJobBeanName())` -- [x] **Example plugin project** — `example-payment-job/` standalone Maven module at repo root; `PaymentJobPlugin` implementing `BatchJobPlugin` with CSV-reading tasklet (`data/sample-payments.csv`); SPI file `META-INF/services/com.fronzec.api.BatchJobPlugin`; `provided` scope for `batch-job-api:0.0.1-SNAPSHOT`, Spring Batch 6.0.0, Spring Framework 7.0.1, SLF4J 2.0.16; maven-shade-plugin 3.6.0 for platform dep shading; README with build/upload/load/execute/unload instructions -- [x] **Developer guide rewrite** — `fr-batch-service/docs/job-plugins/developer-guide.md` now covers: lifecycle ASCII diagram (upload→approve→load→execute→unload→audit), REST API reference, JAR signing config, approval workflow, security roles matrix, health/metrics endpoints, 7-entry troubleshooting FAQ -- [x] **Production checklist** — `fr-batch-service/docs/production-checklist.md` created; checkbox-based checklist covering security, storage, monitoring, scaling, approval workflow, disaster recovery, pre-deployment validation, operational runbooks, final sign-off -- [x] **120/120 tests passing**, `mvn test` BUILD SUCCESS -- [x] **Spring Boot 4.0 adaptation**: Health classes use `org.springframework.boot.health.contributor.*` (package migrated from `org.springframework.boot.actuate.health`); `TestRestTemplate` removed (use direct bean injection in integration tests) -- [x] **Version corrections**: Developer guide updated to reflect Spring Batch 6.0.0 and Spring Framework 7.1.0 (transitive from Spring Boot 4.0.0 parent POM) -**Implementation details**: Four independent deliverables — health indicator, metrics, example plugin, documentation — all additive with no API or schema changes. Health and metrics leverage existing Spring Boot Actuator infrastructure. Example plugin mirrors `DynamicTestPlugin` structure but as a standalone Maven module. 2 PRs stacked on `plugin-architecture-phase-6` branch and merged via stacked-to-main chain. See PRs #44, #45. +- [x] **PluginHealthIndicator** — `@Component` implementing `HealthIndicator`; queries + `PluginRegistryService.getPlugins().size()`, + `JobDefinitionRepository.countByLoadStatus("FAILED")`, `Path.of(jarDir).toFile().canRead()`; + returns DOWN if failed > 0 or dir inaccessible; exposed at `/actuator/health/plugins` via + `management.endpoint.health.group.plugins.include=pluginHealthIndicator` +- [x] **PluginMetrics** — `@Component` with `@PostConstruct` registering Micrometer counters + (`plugins_uploaded_total`, `plugins_loaded_total`, `plugins_unloaded_total`), gauge + (`plugins_loaded`), timers (`plugins_load_duration_seconds`, `plugins_unload_duration_seconds`); + tagged `plugins_executed_total` counter with `job_name` tag; injected into `JarUploadService`, + `DynamicJobLoaderService`, `JobsManagerService` +- [x] **Service instrumentation** — `JarUploadService.uploadJar()` calls + `pluginMetrics.incrementUpload()` after DB persist; `DynamicJobLoaderService` calls + `incrementLoad()` + `recordLoadDuration()` on load, `incrementUnload()` + `recordUnloadDuration()` + on unload; `JobsManagerService.syncRunJobWithParams()` calls + `pluginMetrics.incrementExecuted(request.getJobBeanName())` +- [x] **Example plugin project** — `example-payment-job/` standalone Maven module at repo root; + `PaymentJobPlugin` implementing `BatchJobPlugin` with CSV-reading tasklet + (`data/sample-payments.csv`); SPI file `META-INF/services/com.fronzec.api.BatchJobPlugin`; + `provided` scope for `batch-job-api:0.0.1-SNAPSHOT`, Spring Batch 6.0.0, Spring Framework 7.0.1, + SLF4J 2.0.16; maven-shade-plugin 3.6.0 for platform dep shading; README with + build/upload/load/execute/unload instructions +- [x] **Developer guide rewrite** — `fr-batch-service/docs/job-plugins/developer-guide.md` now + covers: lifecycle ASCII diagram (upload→approve→load→execute→unload→audit), REST API reference, + JAR signing config, approval workflow, security roles matrix, health/metrics endpoints, 7-entry + troubleshooting FAQ +- [x] **Production checklist** — `fr-batch-service/docs/production-checklist.md` created; + checkbox-based checklist covering security, storage, monitoring, scaling, approval workflow, + disaster recovery, pre-deployment validation, operational runbooks, final sign-off +- [x] **120/120 tests passing**, `mvn test` BUILD SUCCESS +- [x] **Spring Boot 4.0 adaptation**: Health classes use + `org.springframework.boot.health.contributor.*` (package migrated from + `org.springframework.boot.actuate.health`); `TestRestTemplate` removed (use direct bean injection + in integration tests) +- [x] **Version corrections**: Developer guide updated to reflect Spring Batch 6.0.0 and Spring + Framework 7.1.0 (transitive from Spring Boot 4.0.0 parent POM) + +**Implementation details**: Four independent deliverables — health indicator, metrics, example +plugin, documentation — all additive with no API or schema changes. Health and metrics leverage +existing Spring Boot Actuator infrastructure. Example plugin mirrors `DynamicTestPlugin` structure +but as a standalone Maven module. 2 PRs stacked on `plugin-architecture-phase-6` branch and merged +via stacked-to-main chain. See PRs #44, #45. ### Phase 7: Production Deployment (Future) + - [ ] Deploy to staging environment - [ ] Load testing - [ ] Operational runbook @@ -1146,6 +1307,7 @@ public class JobsManagerService { ## Success Metrics ### Phase 1-2 Achieved (May 2026) + - [x] Job registration decoupled from `JobsManagerService` — no more hardcoded map - [x] Any `@Component` implementing `BatchJobPlugin` is auto-discovered at startup - [x] `batch-job-api` published to GitHub Packages @@ -1154,6 +1316,7 @@ public class JobsManagerService { - [x] Zero production code changes needed to add a new plugin (just add a `@Component`) ### Technical Metrics (for future phases) + - [ ] Job load time < 5 seconds (dynamic loading) - [ ] Memory overhead per loaded job < 50MB - [ ] Zero memory leaks after 100 load/unload cycles @@ -1161,46 +1324,60 @@ public class JobsManagerService { - [ ] Support for 50+ concurrent loaded jobs ### Business Metrics -- [x] Reduce time to deploy new job from days to hours — **achieved**: new job = new `@Component`, no redeploy architecture -- [ ] Enable 3+ teams to develop jobs independently — **partial**: contract exists, but plugins still live in same repo -- [ ] 90% of new jobs deployed without platform changes — **partial**: no platform changes for classpath plugins; external JARs deferred + +- [x] Reduce time to deploy new job from days to hours — **achieved**: new job = new `@Component`, + no redeploy architecture +- [ ] Enable 3+ teams to develop jobs independently — **partial**: contract exists, but plugins + still live in same repo +- [ ] 90% of new jobs deployed without platform changes — **partial**: no platform changes for + classpath plugins; external JARs deferred - [ ] Zero production incidents due to job loading — **not yet in production** ## Open Questions 1. **Job Versioning Strategy**: Should we support running multiple versions simultaneously? + - **Decision Needed**: Define versioning policy (semantic versioning required?) -2. **Job Dependency Management**: How to handle shared job libraries? +1. **Job Dependency Management**: How to handle shared job libraries? + - **Options**: Shared classloader, parent context beans, explicit dependencies -3. **Job Approval Process**: Should all uploaded jobs require approval before enabling? - - **Options**: Auto-approve for trusted teams, manual approval for all, approval based on environment +1. **Job Approval Process**: Should all uploaded jobs require approval before enabling? + + - **Options**: Auto-approve for trusted teams, manual approval for all, approval based on + environment + +1. **Storage Strategy**: Where to store JAR files long-term? -4. **Storage Strategy**: Where to store JAR files long-term? - **Options**: Local filesystem, S3, NFS, Artifactory -5. **Hot Reload Impact**: Should running job instances be stopped when reloading? +1. **Hot Reload Impact**: Should running job instances be stopped when reloading? + - **Options**: Graceful drain, force stop, wait for completion -6. **Multi-instance Deployment**: How to handle job loading across multiple service instances? +1. **Multi-instance Deployment**: How to handle job loading across multiple service instances? + - **Options**: Shared storage + database coordination, leader election, manual sync ## References ### Related Documentation + - [Spring Batch Documentation](https://docs.spring.io/spring-batch/docs/current/reference/html/) - [Java ClassLoader Documentation](https://docs.oracle.com/javase/8/docs/api/java/lang/ClassLoader.html) - [Spring Boot DevTools (hot reload reference)](https://docs.spring.io/spring-boot/docs/current/reference/html/using.html#using.devtools) - [OSGi Dynamic Module System](https://www.osgi.org/developer/architecture/) (inspiration) ### Similar Implementations + - Jenkins Plugin System - Elasticsearch Plugin Architecture - Apache Kafka Connect Plugins - SonarQube Plugin System ### Security References + - [OWASP - Deserialization of untrusted data](https://owasp.org/www-community/vulnerabilities/Deserialization_of_untrusted_data) - [Java Security Manager (deprecated but concepts relevant)](https://docs.oracle.com/javase/tutorial/essential/environment/security.html) @@ -1213,34 +1390,53 @@ public class JobsManagerService { ## Review and Approval -| Role | Name | Date | Status | -|------|------|------|--------| -| Architect | TBD | TBD | Pending | -| Security | TBD | TBD | Pending | -| Operations | TBD | TBD | Pending | -| Product | TBD | TBD | Pending | +| Role | Name | Date | Status | +| ---------- | ---- | ---- | ------- | +| Architect | TBD | TBD | Pending | +| Security | TBD | TBD | Pending | +| Operations | TBD | TBD | Pending | +| Product | TBD | TBD | Pending | ---- +______________________________________________________________________ -**Last Updated**: 2026-05-31 (Phase 6 complete; example, documentation, health & metrics operational) -**Next Review**: Before starting Phase 7 (production deployment) +**Last Updated**: 2026-05-31 (Phase 6 complete; example, documentation, health & metrics +operational) **Next Review**: Before starting Phase 7 (production deployment) ### Related PRs -- [#27](https://github.com/fronzec/spring-batch-projects/pull/27) — Phase 1: Foundation (batch-job-api, docs, schema) -- [#28](https://github.com/fronzec/spring-batch-projects/pull/28) — Phase 2: Plugin registry + Job1Plugin migration -- [#29](https://github.com/fronzec/spring-batch-projects/pull/29) — Phase 2: sc09-test-fixture (integration test strengthening + H2 fix) -- [#30](https://github.com/fronzec/spring-batch-projects/pull/30) — Phase 3 PR 1: Flyway + Job Registry JPA Layer -- [#32](https://github.com/fronzec/spring-batch-projects/pull/32) — Phase 3 PR 3: CRUD Endpoints + Enable/Disable -- [#33](https://github.com/fronzec/spring-batch-projects/pull/33) — Phase 3 PR 4: PluginRegistryService Extension + Merged API -- [#34](https://github.com/fronzec/spring-batch-projects/pull/34) — Phase 3 PR 5: GlobalExceptionHandler + Integration Tests (Capstone) -- [#35](https://github.com/fronzec/spring-batch-projects/pull/35) — Phase 4 PR 1: DynamicJobClassLoader + unit tests -- [#36](https://github.com/fronzec/spring-batch-projects/pull/36) — Phase 4 PR 2: DynamicJobLoaderService + lifecycle -- [#37](https://github.com/fronzec/spring-batch-projects/pull/37) — Phase 4 PR 3: Controller endpoints + PluginAutoLoader -- [#38](https://github.com/fronzec/spring-batch-projects/pull/38) — Phase 4 PR 4: Integration tests + backward compatibility (Capstone) -- [#39](https://github.com/fronzec/spring-batch-projects/pull/39) — Phase 5 PR 1: ChecksumUtil extraction + load re-verification -- [#40](https://github.com/fronzec/spring-batch-projects/pull/40) — Phase 5 PR 2: Spring Security baseline (HTTP Basic, role-based access) -- [#41](https://github.com/fronzec/spring-batch-projects/pull/41) — Phase 5 PR 3: Approval workflow + V2 Flyway migration -- [#42](https://github.com/fronzec/spring-batch-projects/pull/42) — Phase 5 PR 4: JAR signature verification (strict/permissive modes) -- [#43](https://github.com/fronzec/spring-batch-projects/pull/43) — Phase 5 PR 5: Audit service + JobExecutionListener -- [#44](https://github.com/fronzec/spring-batch-projects/pull/44) — Phase 6 PR 1: PluginHealthIndicator + PluginMetrics + service instrumentation -- [#45](https://github.com/fronzec/spring-batch-projects/pull/45) — Phase 6 PR 2: Example plugin project (example-payment-job/) + developer guide rewrite + production checklist + +- [#27](https://github.com/fronzec/spring-batch-projects/pull/27) — Phase 1: Foundation + (batch-job-api, docs, schema) +- [#28](https://github.com/fronzec/spring-batch-projects/pull/28) — Phase 2: Plugin registry + + Job1Plugin migration +- [#29](https://github.com/fronzec/spring-batch-projects/pull/29) — Phase 2: sc09-test-fixture + (integration test strengthening + H2 fix) +- [#30](https://github.com/fronzec/spring-batch-projects/pull/30) — Phase 3 PR 1: Flyway + Job + Registry JPA Layer +- [#32](https://github.com/fronzec/spring-batch-projects/pull/32) — Phase 3 PR 3: CRUD Endpoints + + Enable/Disable +- [#33](https://github.com/fronzec/spring-batch-projects/pull/33) — Phase 3 PR 4: + PluginRegistryService Extension + Merged API +- [#34](https://github.com/fronzec/spring-batch-projects/pull/34) — Phase 3 PR 5: + GlobalExceptionHandler + Integration Tests (Capstone) +- [#35](https://github.com/fronzec/spring-batch-projects/pull/35) — Phase 4 PR 1: + DynamicJobClassLoader + unit tests +- [#36](https://github.com/fronzec/spring-batch-projects/pull/36) — Phase 4 PR 2: + DynamicJobLoaderService + lifecycle +- [#37](https://github.com/fronzec/spring-batch-projects/pull/37) — Phase 4 PR 3: Controller + endpoints + PluginAutoLoader +- [#38](https://github.com/fronzec/spring-batch-projects/pull/38) — Phase 4 PR 4: Integration tests + \+ backward compatibility (Capstone) +- [#39](https://github.com/fronzec/spring-batch-projects/pull/39) — Phase 5 PR 1: ChecksumUtil + extraction + load re-verification +- [#40](https://github.com/fronzec/spring-batch-projects/pull/40) — Phase 5 PR 2: Spring Security + baseline (HTTP Basic, role-based access) +- [#41](https://github.com/fronzec/spring-batch-projects/pull/41) — Phase 5 PR 3: Approval workflow + \+ V2 Flyway migration +- [#42](https://github.com/fronzec/spring-batch-projects/pull/42) — Phase 5 PR 4: JAR signature + verification (strict/permissive modes) +- [#43](https://github.com/fronzec/spring-batch-projects/pull/43) — Phase 5 PR 5: Audit service + + JobExecutionListener +- [#44](https://github.com/fronzec/spring-batch-projects/pull/44) — Phase 6 PR 1: + PluginHealthIndicator + PluginMetrics + service instrumentation +- [#45](https://github.com/fronzec/spring-batch-projects/pull/45) — Phase 6 PR 2: Example plugin + project (example-payment-job/) + developer guide rewrite + production checklist diff --git a/fr-batch-service/docs/job-plugins/developer-guide.md b/fr-batch-service/docs/job-plugins/developer-guide.md index 92334a13..c5a78400 100644 --- a/fr-batch-service/docs/job-plugins/developer-guide.md +++ b/fr-batch-service/docs/job-plugins/developer-guide.md @@ -1,8 +1,9 @@ # Job Plugins Developer Guide -This guide covers the complete lifecycle for developing, packaging, uploading, approving, loading, and executing external Spring Batch job plugins for the `fr-batch-service` platform. +This guide covers the complete lifecycle for developing, packaging, uploading, approving, loading, +and executing external Spring Batch job plugins for the `fr-batch-service` platform. ---- +______________________________________________________________________ ## Plugin Lifecycle Overview @@ -21,14 +22,14 @@ This guide covers the complete lifecycle for developing, packaging, uploading, a ``` 1. **Develop & Build** — Create a Maven project that implements `BatchJobPlugin` -2. **Upload** — `POST /jobs/upload` sends the shaded JAR to the platform -3. **Approve** — `PUT /jobs/definitions/{id}/approve` marks the job ready for loading -4. **Load** — `POST /jobs/definitions/{id}/load` instantiates the plugin and registers it -5. **Execute** — `POST /jobs/start-single/sync` (or `/async`) runs the job with parameters -6. **Unload** — `POST /jobs/definitions/{id}/unload` unregisters and closes the classloader -7. **Audit** — All lifecycle events are logged to the audit trail +1. **Upload** — `POST /jobs/upload` sends the shaded JAR to the platform +1. **Approve** — `PUT /jobs/definitions/{id}/approve` marks the job ready for loading +1. **Load** — `POST /jobs/definitions/{id}/load` instantiates the plugin and registers it +1. **Execute** — `POST /jobs/start-single/sync` (or `/async`) runs the job with parameters +1. **Unload** — `POST /jobs/definitions/{id}/unload` unregisters and closes the classloader +1. **Audit** — All lifecycle events are logged to the audit trail ---- +______________________________________________________________________ ## BatchJobPlugin Contract @@ -59,7 +60,7 @@ public interface JobMetadata { } ``` ---- +______________________________________________________________________ ## Plugin Project Structure (Maven) @@ -78,10 +79,13 @@ example-payment-job/ ### pom.xml — Key Concepts -- **`provided` scope**: `batch-job-api`, `spring-batch-core`, `spring-context`, and `spring-tx` must be `provided` — the platform supplies them at runtime. -- **`maven-shade-plugin`**: Bundles job-specific dependencies (e.g., CSV parsers, HTTP clients) while excluding platform libraries to prevent classpath conflicts. +- **`provided` scope**: `batch-job-api`, `spring-batch-core`, `spring-context`, and `spring-tx` must + be `provided` — the platform supplies them at runtime. +- **`maven-shade-plugin`**: Bundles job-specific dependencies (e.g., CSV parsers, HTTP clients) + while excluding platform libraries to prevent classpath conflicts. - **Java 21**: The platform runs on Java 21; plugins must target the same release. -- **No Spring Boot parent**: Plugins are standalone Maven projects; they don't inherit from Spring Boot's parent POM. +- **No Spring Boot parent**: Plugins are standalone Maven projects; they don't inherit from Spring + Boot's parent POM. ```xml @@ -153,7 +157,8 @@ example-payment-job/ ### SPI Registration -Create the file `src/main/resources/META-INF/services/com.fronzec.api.BatchJobPlugin` containing the fully-qualified class name of your plugin: +Create the file `src/main/resources/META-INF/services/com.fronzec.api.BatchJobPlugin` containing the +fully-qualified class name of your plugin: ``` com.example.payment.PaymentJobPlugin @@ -161,23 +166,24 @@ com.example.payment.PaymentJobPlugin This is how the platform discovers your plugin via `ServiceLoader` at load time. ---- +______________________________________________________________________ ## REST API Reference -All endpoints are prefixed with `/api/batch-service`. Authentication uses HTTP Basic (see Security Roles below). +All endpoints are prefixed with `/api/batch-service`. Authentication uses HTTP Basic (see Security +Roles below). ### Job Definition Management -| Method | Path | Auth | Description | -|--------|------|------|-------------| -| `POST` | `/jobs/upload` | `PLUGIN_ADMIN` | Upload a JAR and register a job definition | -| `GET` | `/jobs/definitions` | `PLUGIN_VIEWER`+ | List all job definitions | -| `GET` | `/jobs/definitions/{id}` | `PLUGIN_VIEWER`+ | Get a single definition | -| `PUT` | `/jobs/definitions/{id}/enable` | `PLUGIN_ADMIN` | Enable a definition | -| `PUT` | `/jobs/definitions/{id}/disable` | `PLUGIN_ADMIN` | Disable a definition | -| `DELETE` | `/jobs/definitions/{id}` | `PLUGIN_ADMIN` | Delete definition and JAR file | -| `GET` | `/jobs/plugins` | Public | List merged plugin metadata | +| Method | Path | Auth | Description | +| -------- | -------------------------------- | ---------------- | ------------------------------------------ | +| `POST` | `/jobs/upload` | `PLUGIN_ADMIN` | Upload a JAR and register a job definition | +| `GET` | `/jobs/definitions` | `PLUGIN_VIEWER`+ | List all job definitions | +| `GET` | `/jobs/definitions/{id}` | `PLUGIN_VIEWER`+ | Get a single definition | +| `PUT` | `/jobs/definitions/{id}/enable` | `PLUGIN_ADMIN` | Enable a definition | +| `PUT` | `/jobs/definitions/{id}/disable` | `PLUGIN_ADMIN` | Disable a definition | +| `DELETE` | `/jobs/definitions/{id}` | `PLUGIN_ADMIN` | Delete definition and JAR file | +| `GET` | `/jobs/plugins` | Public | List merged plugin metadata | ### Upload @@ -191,6 +197,7 @@ curl -u admin:admin123 \ ``` **Response** (`201 Created`): + ```json { "id": 1, @@ -208,12 +215,13 @@ curl -u admin:admin123 \ ### Approval Workflow -By default, `app.plugins.approval.auto-approve=true` in development. In production, set it to `false` and use explicit approval: +By default, `app.plugins.approval.auto-approve=true` in development. In production, set it to +`false` and use explicit approval: -| Method | Path | Auth | Description | -|--------|------|------|-------------| -| `PUT` | `/jobs/definitions/{id}/approve` | `PLUGIN_ADMIN` | Approve a job definition | -| `PUT` | `/jobs/definitions/{id}/reject` | `PLUGIN_ADMIN` | Reject a job definition | +| Method | Path | Auth | Description | +| ------ | -------------------------------- | -------------- | ------------------------ | +| `PUT` | `/jobs/definitions/{id}/approve` | `PLUGIN_ADMIN` | Approve a job definition | +| `PUT` | `/jobs/definitions/{id}/reject` | `PLUGIN_ADMIN` | Reject a job definition | ```bash curl -u admin:admin123 \ @@ -227,12 +235,12 @@ curl -u admin:admin123 \ ### Load and Unload -| Method | Path | Auth | Description | -|--------|------|------|-------------| -| `POST` | `/jobs/definitions/{id}/load` | `PLUGIN_ADMIN` | Load plugin from JAR into registry | +| Method | Path | Auth | Description | +| ------ | ------------------------------- | -------------- | ------------------------------------ | +| `POST` | `/jobs/definitions/{id}/load` | `PLUGIN_ADMIN` | Load plugin from JAR into registry | | `POST` | `/jobs/definitions/{id}/unload` | `PLUGIN_ADMIN` | Unregister plugin, close classloader | -| `POST` | `/jobs/definitions/{id}/reload` | `PLUGIN_ADMIN` | Atomic unload + load | -| `POST` | `/jobs/load-all` | `PLUGIN_ADMIN` | Load all enabled definitions | +| `POST` | `/jobs/definitions/{id}/reload` | `PLUGIN_ADMIN` | Atomic unload + load | +| `POST` | `/jobs/load-all` | `PLUGIN_ADMIN` | Load all enabled definitions | ```bash # Load @@ -254,16 +262,17 @@ curl -u admin:admin123 -X POST \ ### Job Execution -| Method | Path | Auth | Description | -|--------|------|------|-------------| -| `POST` | `/jobs/start-single/sync` | `PLUGIN_ADMIN` | Run a single job synchronously | -| `POST` | `/jobs/start-single/async` | `PLUGIN_ADMIN` | Run a single job asynchronously | -| `POST` | `/jobs/start-all/sync` | `PLUGIN_ADMIN` | Run all loaded jobs synchronously | -| `POST` | `/jobs/start-all/async` | `PLUGIN_ADMIN` | Run all loaded jobs asynchronously | -| `POST` | `/jobs/stop-all` | `PLUGIN_ADMIN` | Stop all running jobs | -| `GET` | `/jobs/running-all` | `PLUGIN_VIEWER`+ | List running job executions | +| Method | Path | Auth | Description | +| ------ | -------------------------- | ---------------- | ---------------------------------- | +| `POST` | `/jobs/start-single/sync` | `PLUGIN_ADMIN` | Run a single job synchronously | +| `POST` | `/jobs/start-single/async` | `PLUGIN_ADMIN` | Run a single job asynchronously | +| `POST` | `/jobs/start-all/sync` | `PLUGIN_ADMIN` | Run all loaded jobs synchronously | +| `POST` | `/jobs/start-all/async` | `PLUGIN_ADMIN` | Run all loaded jobs asynchronously | +| `POST` | `/jobs/stop-all` | `PLUGIN_ADMIN` | Stop all running jobs | +| `GET` | `/jobs/running-all` | `PLUGIN_VIEWER`+ | List running job executions | **Execute a single job synchronously**: + ```bash curl -u admin:admin123 \ -X POST \ @@ -279,14 +288,15 @@ curl -u admin:admin123 \ ``` **Request body fields** (snake_case due to Jackson naming strategy): -| Field | Required | Description | -|-------|----------|-------------| -| `job_bean_name` | Yes | The job name from `getJobName()` | -| `date` | Yes | ISO-8601 date string | -| `execution_attempt_number` | Yes | Unique attempt number | -| `description` | No | Non-identifying descriptor | ---- +| Field | Required | Description | +| -------------------------- | -------- | -------------------------------- | +| `job_bean_name` | Yes | The job name from `getJobName()` | +| `date` | Yes | ISO-8601 date string | +| `execution_attempt_number` | Yes | Unique attempt number | +| `description` | No | Non-identifying descriptor | + +______________________________________________________________________ ## JAR Signing Configuration @@ -301,12 +311,13 @@ app.plugins.signature.mode=strict ``` **JAR signing with jarsigner** (for production): + ```bash jarsigner -keystore my-keystore.jks -storepass secret \ -keypass secret target/example-payment-job-1.0.0.jar my-key-alias ``` ---- +______________________________________________________________________ ## Approval Workflow @@ -314,9 +325,9 @@ The approval system prevents unverified plugins from being loaded into the platf ### Modes -| Property | Value | Behavior | -|----------|-------|----------| -| `app.plugins.approval.auto-approve` | `true` (dev) | Jobs auto-approved on upload | +| Property | Value | Behavior | +| ----------------------------------- | -------------- | ---------------------------- | +| `app.plugins.approval.auto-approve` | `true` (dev) | Jobs auto-approved on upload | | `app.plugins.approval.auto-approve` | `false` (prod) | Jobs require manual approval | ### Approval States @@ -331,31 +342,31 @@ PENDING ──▶ APPROVED ──▶ LOADED ──▶ EXECUTED ### Production Approval Flow 1. Developer uploads JAR → `PENDING` -2. Approver reviews metadata and signature → `APPROVED` or `REJECTED` -3. If approved, operator loads the job → `LOADED` -4. Job is available for execution +1. Approver reviews metadata and signature → `APPROVED` or `REJECTED` +1. If approved, operator loads the job → `LOADED` +1. Job is available for execution ---- +______________________________________________________________________ ## Security Roles The platform uses HTTP Basic authentication with two roles: -| Role | Username | Password (dev) | Permissions | -|------|----------|----------------|-------------| -| `PLUGIN_ADMIN` | `admin` | `admin123` | Full CRUD, load, unload, execute, approve | -| `PLUGIN_VIEWER` | `viewer` | `viewer123` | Read-only: list definitions, plugins, running jobs | +| Role | Username | Password (dev) | Permissions | +| --------------- | -------- | -------------- | -------------------------------------------------- | +| `PLUGIN_ADMIN` | `admin` | `admin123` | Full CRUD, load, unload, execute, approve | +| `PLUGIN_VIEWER` | `viewer` | `viewer123` | Read-only: list definitions, plugins, running jobs | ### Endpoint Access Matrix -| Endpoint Pattern | Public | PLUGIN_VIEWER | PLUGIN_ADMIN | -|------------------|--------|---------------|--------------| -| `GET /actuator/health` | ✅ | ✅ | ✅ | -| `GET /jobs/plugins` | ✅ | ✅ | ✅ | -| `GET /jobs/**` | ❌ | ✅ | ✅ | -| `POST /jobs/**` | ❌ | ❌ | ✅ | -| `PUT /jobs/**` | ❌ | ❌ | ✅ | -| `DELETE /jobs/**` | ❌ | ❌ | ✅ | +| Endpoint Pattern | Public | PLUGIN_VIEWER | PLUGIN_ADMIN | +| ---------------------- | ------ | ------------- | ------------ | +| `GET /actuator/health` | ✅ | ✅ | ✅ | +| `GET /jobs/plugins` | ✅ | ✅ | ✅ | +| `GET /jobs/**` | ❌ | ✅ | ✅ | +| `POST /jobs/**` | ❌ | ❌ | ✅ | +| `PUT /jobs/**` | ❌ | ❌ | ✅ | +| `DELETE /jobs/**` | ❌ | ❌ | ✅ | ### Production Password Configuration @@ -368,7 +379,7 @@ app.security.viewer-password={bcrypt}$2a$10$... And update `SecurityConfig` to use `BCryptPasswordEncoder` instead of `NoOpPasswordEncoder`. ---- +______________________________________________________________________ ## Health and Metrics Endpoints @@ -381,6 +392,7 @@ GET /api/batch-service/actuator/health/plugins ``` **Response** (UP): + ```json { "status": "UP", @@ -393,6 +405,7 @@ GET /api/batch-service/actuator/health/plugins ``` **Response** (DOWN): + ```json { "status": "DOWN", @@ -405,6 +418,7 @@ GET /api/batch-service/actuator/health/plugins ``` The health indicator reports DOWN when: + - One or more job definitions have `loadStatus = FAILED` - The JAR storage directory is not readable @@ -416,15 +430,15 @@ Metrics are exposed at: GET /api/batch-service/actuator/prometheus ``` -| Metric | Type | Description | -|--------|------|-------------| -| `plugins_uploaded_total` | Counter | Total JARs uploaded | -| `plugins_loaded_total` | Counter | Total plugin load operations | -| `plugins_unloaded_total` | Counter | Total plugin unload operations | -| `plugins_executed_total{job_name="..."}` | Counter | Executions per job name | -| `plugins_loaded_count` | Gauge | Currently loaded plugins | -| `plugins_load_duration_seconds` | Timer | Load operation duration | -| `plugins_unload_duration_seconds` | Timer | Unload operation duration | +| Metric | Type | Description | +| ---------------------------------------- | ------- | ------------------------------ | +| `plugins_uploaded_total` | Counter | Total JARs uploaded | +| `plugins_loaded_total` | Counter | Total plugin load operations | +| `plugins_unloaded_total` | Counter | Total plugin unload operations | +| `plugins_executed_total{job_name="..."}` | Counter | Executions per job name | +| `plugins_loaded_count` | Gauge | Currently loaded plugins | +| `plugins_load_duration_seconds` | Timer | Load operation duration | +| `plugins_unload_duration_seconds` | Timer | Unload operation duration | ### Prometheus Scrape Config @@ -436,7 +450,7 @@ scrape_configs: - targets: ['localhost:8080'] ``` ---- +______________________________________________________________________ ## Troubleshooting @@ -444,13 +458,16 @@ scrape_configs: **Cause**: The file is not a valid JAR (ZIP) or is corrupted. -**Fix**: Verify the file starts with ZIP magic bytes (`PK\x03\x04`). Run `mvn clean package` and re-upload. Check the file extension is `.jar`. +**Fix**: Verify the file starts with ZIP magic bytes (`PK\x03\x04`). Run `mvn clean package` and +re-upload. Check the file extension is `.jar`. ### Q: Load fails with "JobLoadException: SPI file not found" -**Cause**: The `META-INF/services/com.fronzec.api.BatchJobPlugin` file is missing from the JAR or contains an incorrect class name. +**Cause**: The `META-INF/services/com.fronzec.api.BatchJobPlugin` file is missing from the JAR or +contains an incorrect class name. **Fix**: + ```bash # Check SPI file exists in the JAR jar tf target/example-payment-job-1.0.0.jar | grep META-INF/services @@ -461,9 +478,12 @@ cat META-INF/services/com.fronzec.api.BatchJobPlugin ### Q: Load fails with "NoClassDefFoundError" for platform classes -**Cause**: Platform dependencies (Spring Batch, Spring Framework) were shaded into the plugin JAR, causing classloader conflicts. +**Cause**: Platform dependencies (Spring Batch, Spring Framework) were shaded into the plugin JAR, +causing classloader conflicts. + +**Fix**: Ensure `spring-batch-core`, `spring-context`, and `spring-tx` are excluded in the shade +plugin configuration and declared as `provided` scope. Verify with: -**Fix**: Ensure `spring-batch-core`, `spring-context`, and `spring-tx` are excluded in the shade plugin configuration and declared as `provided` scope. Verify with: ```bash jar tf target/example-payment-job-1.0.0.jar | grep "spring" # Should return empty or only job-specific Spring classes @@ -473,22 +493,28 @@ jar tf target/example-payment-job-1.0.0.jar | grep "spring" **Cause**: The plugin was not loaded before execution, or was loaded under a different name. -**Fix**: Verify the job name in `getJobName()` matches the `job_bean_name` in the request. Check load status via `GET /jobs/definitions/{id}` — it should show `loadStatus: LOADED`. +**Fix**: Verify the job name in `getJobName()` matches the `job_bean_name` in the request. Check +load status via `GET /jobs/definitions/{id}` — it should show `loadStatus: LOADED`. ### Q: Unload fails with "JobUnloadConflictException" **Cause**: The job has active running executions and `force=false` was used. -**Fix**: Either wait for executions to complete, use `force=true`, or stop the running executions first via `POST /jobs/stop-all`. +**Fix**: Either wait for executions to complete, use `force=true`, or stop the running executions +first via `POST /jobs/stop-all`. ### Q: Health endpoint shows DOWN **Cause**: A job definition has `loadStatus = FAILED` or the JAR directory is not accessible. -**Fix**: Check `GET /jobs/definitions` for entries with `loadStatus: FAILED`. Investigate the application logs for the failure reason. Verify the JAR directory exists and is readable: `ls -la ./plugins/jars/`. +**Fix**: Check `GET /jobs/definitions` for entries with `loadStatus: FAILED`. Investigate the +application logs for the failure reason. Verify the JAR directory exists and is readable: +`ls -la ./plugins/jars/`. ### Q: Metrics not appearing in Prometheus **Cause**: The Prometheus endpoint may not be exposed or the scrape path is incorrect. -**Fix**: Verify `management.endpoints.web.exposure.include=health,info,env,prometheus` in `application.properties`. Ensure the Prometheus scrape config uses the correct `metrics_path` with the context path prefix (`/api/batch-service/actuator/prometheus`). +**Fix**: Verify `management.endpoints.web.exposure.include=health,info,env,prometheus` in +`application.properties`. Ensure the Prometheus scrape config uses the correct `metrics_path` with +the context path prefix (`/api/batch-service/actuator/prometheus`). diff --git a/fr-batch-service/docs/production-checklist.md b/fr-batch-service/docs/production-checklist.md index 2b18f075..07e750b9 100644 --- a/fr-batch-service/docs/production-checklist.md +++ b/fr-batch-service/docs/production-checklist.md @@ -1,176 +1,233 @@ # Production Readiness Checklist -This checklist must be completed before deploying the `fr-batch-service` plugin system to production. +This checklist must be completed before deploying the `fr-batch-service` plugin system to +production. ---- +______________________________________________________________________ ## 🔒 Security ### JAR Signing -- [ ] **Strict signature enforcement enabled**: Set `app.plugins.signature.mode=strict` in production configuration. -- [ ] **Signing certificate trusted**: All uploaded JARs must be signed with a certificate from the trusted keystore. -- [ ] **Keystore secured**: The platform's truststore/keystore is stored in a secure location (not in source control). -- [ ] **Signature verification tested**: Upload a test JAR signed with an untrusted certificate and confirm rejection. -- [ ] **Unsigned JAR rejection verified**: Upload an unsigned JAR in strict mode and confirm HTTP 400 with appropriate error message. +- [ ] **Strict signature enforcement enabled**: Set `app.plugins.signature.mode=strict` in + production configuration. +- [ ] **Signing certificate trusted**: All uploaded JARs must be signed with a certificate from the + trusted keystore. +- [ ] **Keystore secured**: The platform's truststore/keystore is stored in a secure location (not + in source control). +- [ ] **Signature verification tested**: Upload a test JAR signed with an untrusted certificate and + confirm rejection. +- [ ] **Unsigned JAR rejection verified**: Upload an unsigned JAR in strict mode and confirm HTTP + 400 with appropriate error message. ### Authentication & Authorization -- [ ] **BCrypt passwords**: Replace `{noop}` plain-text passwords with `{bcrypt}` hashes. Use `PasswordEncoderFactories.createDelegatingPasswordEncoder()` or `BCryptPasswordEncoder` directly. -- [ ] **Strong passwords in use**: Admin and viewer passwords meet complexity requirements (≥12 chars, mix of upper/lower/digit/special). -- [ ] **HTTPS enabled**: All API traffic goes over TLS. Configure SSL in `application.properties` or via a reverse proxy (nginx, Traefik). -- [ ] **Role separation verified**: `PLUGIN_VIEWER` cannot execute upload, load, unload, approve, or execute operations. -- [ ] **Audit log tamper-proof**: Audit events are stored in a database table with restricted write access. +- [ ] **BCrypt passwords**: Replace `{noop}` plain-text passwords with `{bcrypt}` hashes. Use + `PasswordEncoderFactories.createDelegatingPasswordEncoder()` or `BCryptPasswordEncoder` directly. +- [ ] **Strong passwords in use**: Admin and viewer passwords meet complexity requirements (≥12 + chars, mix of upper/lower/digit/special). +- [ ] **HTTPS enabled**: All API traffic goes over TLS. Configure SSL in `application.properties` or + via a reverse proxy (nginx, Traefik). +- [ ] **Role separation verified**: `PLUGIN_VIEWER` cannot execute upload, load, unload, approve, or + execute operations. +- [ ] **Audit log tamper-proof**: Audit events are stored in a database table with restricted write + access. ### Application Security -- [ ] **CSRF disabled confirmation**: CSRF protection is intentionally disabled for the REST API. Documented and accepted as a design decision. +- [ ] **CSRF disabled confirmation**: CSRF protection is intentionally disabled for the REST API. + Documented and accepted as a design decision. - [ ] **Session management stateless**: `SessionCreationPolicy.STATELESS` is configured. -- [ ] **No sensitive data in logs**: Verify that logs do not contain passwords, tokens, or PII from uploaded JAR metadata. -- [ ] **File upload limits configured**: `spring.servlet.multipart.max-file-size` is set appropriately for expected JAR sizes (e.g., 50MB). +- [ ] **No sensitive data in logs**: Verify that logs do not contain passwords, tokens, or PII from + uploaded JAR metadata. +- [ ] **File upload limits configured**: `spring.servlet.multipart.max-file-size` is set + appropriately for expected JAR sizes (e.g., 50MB). ---- +______________________________________________________________________ ## 📦 Storage ### JAR Directory -- [ ] **Directory exists and permissions correct**: `fr-batch-service.plugins.jar-dir` points to a writable directory (e.g., `/var/lib/fr-batch-service/plugins/jars/`). -- [ ] **Disk space monitoring**: Set up alerting when the JAR storage directory exceeds 80% of the partition capacity. -- [ ] **JAR cleanup policy**: Stale or disabled JARs are cleaned up periodically (manual or automated). -- [ ] **Backup strategy**: The JAR directory and database are backed up regularly. Include backup verification steps. +- [ ] **Directory exists and permissions correct**: `fr-batch-service.plugins.jar-dir` points to a + writable directory (e.g., `/var/lib/fr-batch-service/plugins/jars/`). +- [ ] **Disk space monitoring**: Set up alerting when the JAR storage directory exceeds 80% of the + partition capacity. +- [ ] **JAR cleanup policy**: Stale or disabled JARs are cleaned up periodically (manual or + automated). +- [ ] **Backup strategy**: The JAR directory and database are backed up regularly. Include backup + verification steps. ### Database -- [ ] **Flyway migrations validated**: All migration scripts in `db/migration` have been applied successfully. -- [ ] **Connection pool sized for production**: Review HikariCP `maximumPoolSize` and `minimumIdle` settings for expected load. -- [ ] **Database backup schedule**: Automated backups (e.g., daily full, hourly incremental) with point-in-time recovery capability. -- [ ] **JPA DDL auto disabled**: `spring.jpa.hibernate.ddl_auto=none` ensures no accidental schema changes. -- [ ] **Open session in view disabled**: `spring.jpa.open-in-view=false` avoids database connection leaks. +- [ ] **Flyway migrations validated**: All migration scripts in `db/migration` have been applied + successfully. +- [ ] **Connection pool sized for production**: Review HikariCP `maximumPoolSize` and `minimumIdle` + settings for expected load. +- [ ] **Database backup schedule**: Automated backups (e.g., daily full, hourly incremental) with + point-in-time recovery capability. +- [ ] **JPA DDL auto disabled**: `spring.jpa.hibernate.ddl_auto=none` ensures no accidental schema + changes. +- [ ] **Open session in view disabled**: `spring.jpa.open-in-view=false` avoids database connection + leaks. ---- +______________________________________________________________________ ## 📊 Monitoring ### Health Checks -- [ ] **Plugin health endpoint accessible**: `GET /api/batch-service/actuator/health/plugins` returns accurate UP/DOWN status. -- [ ] **Health check integrated with load balancer**: The load balancer or orchestrator probes the health endpoint and removes unhealthy instances. -- [ ] **Alerting configured for DOWN state**: When the plugin health indicator returns DOWN, an alert fires (PagerDuty, Slack, email). +- [ ] **Plugin health endpoint accessible**: `GET /api/batch-service/actuator/health/plugins` + returns accurate UP/DOWN status. +- [ ] **Health check integrated with load balancer**: The load balancer or orchestrator probes the + health endpoint and removes unhealthy instances. +- [ ] **Alerting configured for DOWN state**: When the plugin health indicator returns DOWN, an + alert fires (PagerDuty, Slack, email). ### Prometheus Metrics - [ ] **Prometheus endpoint exposed**: `management.endpoints.web.exposure.include=prometheus`. -- [ ] **Prometheus scrape job configured**: A scrape config targets the `/api/batch-service/actuator/prometheus` path with a reasonable interval (e.g., 30s). +- [ ] **Prometheus scrape job configured**: A scrape config targets the + `/api/batch-service/actuator/prometheus` path with a reasonable interval (e.g., 30s). - [ ] **Key metric alerts configured**: -| Metric | Alert Condition | Severity | -|--------|----------------|----------| -| `plugins_uploaded_total` | Sudden spike (DoS indicator) | Warning | -| `plugins_loaded_count` | Drops to 0 (all plugins unloaded) | Critical | -| `plugins_load_duration_seconds` | Exceeds 30s (classloader issues) | Warning | -| `plugins_unload_duration_seconds` | Exceeds 10s (stuck unload) | Warning | -| `plugins_executed_total` | Flat for >1h for key jobs | Warning | -| JVM heap usage | >85% | Warning | -| JVM GC pause time | >1s | Warning | +| Metric | Alert Condition | Severity | +| --------------------------------- | --------------------------------- | -------- | +| `plugins_uploaded_total` | Sudden spike (DoS indicator) | Warning | +| `plugins_loaded_count` | Drops to 0 (all plugins unloaded) | Critical | +| `plugins_load_duration_seconds` | Exceeds 30s (classloader issues) | Warning | +| `plugins_unload_duration_seconds` | Exceeds 10s (stuck unload) | Warning | +| `plugins_executed_total` | Flat for >1h for key jobs | Warning | +| JVM heap usage | >85% | Warning | +| JVM GC pause time | >1s | Warning | -- [ ] **Grafana dashboard created**: Build a dashboard showing plugin metrics over time, with panels for counter rates, gauge values, timer percentiles, and health status. +- [ ] **Grafana dashboard created**: Build a dashboard showing plugin metrics over time, with panels + for counter rates, gauge values, timer percentiles, and health status. ### Logging -- [ ] **Structured logging configured**: JSON log format for ingestion into centralized logging (ELK, Loki, Splunk). -- [ ] **Plugin lifecycle events logged**: Upload, load, unload, execution start/end, and errors are logged at INFO/WARN/ERROR levels. -- [ ] **Audit events logged separately**: Audit trail events have distinct log markers for compliance filtering. +- [ ] **Structured logging configured**: JSON log format for ingestion into centralized logging + (ELK, Loki, Splunk). +- [ ] **Plugin lifecycle events logged**: Upload, load, unload, execution start/end, and errors are + logged at INFO/WARN/ERROR levels. +- [ ] **Audit events logged separately**: Audit trail events have distinct log markers for + compliance filtering. ---- +______________________________________________________________________ ## ⚖️ Scaling ### Plugin Limits -- [ ] **Maximum loaded plugins defined**: Establish a limit (e.g., 50 loaded plugins) based on memory testing. Monitor `plugins_loaded_count`. -- [ ] **Classloader memory profiling completed**: Each loaded plugin creates a `DynamicJobClassLoader`. Profile memory usage under simulated load (e.g., 20 concurrent plugins). -- [ ] **Metaspace monitoring**: Monitor `jvm_memory_used_bytes{area="metaspace"}`; excessive loaded classes may exhaust Metaspace. +- [ ] **Maximum loaded plugins defined**: Establish a limit (e.g., 50 loaded plugins) based on + memory testing. Monitor `plugins_loaded_count`. +- [ ] **Classloader memory profiling completed**: Each loaded plugin creates a + `DynamicJobClassLoader`. Profile memory usage under simulated load (e.g., 20 concurrent plugins). +- [ ] **Metaspace monitoring**: Monitor `jvm_memory_used_bytes{area="metaspace"}`; excessive loaded + classes may exhaust Metaspace. ### Known Limitations -- [ ] **No multi-instance coordination**: Plugin load/unload operations are local to a single JVM instance. In a multi-instance deployment: +- [ ] **No multi-instance coordination**: Plugin load/unload operations are local to a single JVM + instance. In a multi-instance deployment: - Each instance must independently load its plugins. - No distributed consensus on load state — instances may diverge. - Load/unload via the load balancer hits only one instance unless sticky sessions are configured. -- [ ] **No horizontal plugin execution**: A single plugin job execution runs on a single instance. For distributed processing, use Spring Batch remote partitioning instead. -- [ ] **Plugin JAR size limit**: Very large JARs (>100MB) may cause high upload latency and increased Metaspace consumption at load time. +- [ ] **No horizontal plugin execution**: A single plugin job execution runs on a single instance. + For distributed processing, use Spring Batch remote partitioning instead. +- [ ] **Plugin JAR size limit**: Very large JARs (>100MB) may cause high upload latency and + increased Metaspace consumption at load time. ### Instance Sizing -- [ ] **Heap size**: Minimum 512MB per instance plus additional heap per loaded plugin (estimate 10-20MB per plugin for class metadata). -- [ ] **Metaspace**: Set `-XX:MaxMetaspaceSize=256m` minimum; increase based on the number and size of plugin JARs. -- [ ] **Thread pool**: Spring Batch's default `SimpleAsyncTaskExecutor` creates unbounded threads. Configure a bounded executor for production. +- [ ] **Heap size**: Minimum 512MB per instance plus additional heap per loaded plugin (estimate + 10-20MB per plugin for class metadata). +- [ ] **Metaspace**: Set `-XX:MaxMetaspaceSize=256m` minimum; increase based on the number and size + of plugin JARs. +- [ ] **Thread pool**: Spring Batch's default `SimpleAsyncTaskExecutor` creates unbounded threads. + Configure a bounded executor for production. ---- +______________________________________________________________________ ## ✅ Approval Workflow - [ ] **Auto-approve disabled**: Set `app.plugins.approval.auto-approve=false` in production. -- [ ] **Approval policy documented**: Define who can approve job definitions, what checks are required (code review, security scan, performance test results). -- [ ] **Approval audit trail verified**: Every approval/rejection is recorded with timestamp, approver identity, and job name in the audit log. -- [ ] **Rejected jobs not loadable**: Verify that REJECTED job definitions cannot be loaded or executed. -- [ ] **Approval gating before load**: Load operation checks `approvalStatus == APPROVED` before proceeding. +- [ ] **Approval policy documented**: Define who can approve job definitions, what checks are + required (code review, security scan, performance test results). +- [ ] **Approval audit trail verified**: Every approval/rejection is recorded with timestamp, + approver identity, and job name in the audit log. +- [ ] **Rejected jobs not loadable**: Verify that REJECTED job definitions cannot be loaded or + executed. +- [ ] **Approval gating before load**: Load operation checks `approvalStatus == APPROVED` before + proceeding. ---- +______________________________________________________________________ ## 🔄 Disaster Recovery ### JAR Recovery -- [ ] **JAR re-upload procedure documented**: If the JAR directory is lost, instructions exist for re-uploading JARs from artifact storage or rebuilding from source. -- [ ] **JAR checksum verification**: After re-upload, verify the SHA-256 checksum matches the value stored in the database (if DB was not lost). +- [ ] **JAR re-upload procedure documented**: If the JAR directory is lost, instructions exist for + re-uploading JARs from artifact storage or rebuilding from source. +- [ ] **JAR checksum verification**: After re-upload, verify the SHA-256 checksum matches the value + stored in the database (if DB was not lost). ### Database Recovery -- [ ] **Database backup restoration tested**: A full restore from backup has been practiced and documented. -- [ ] **Flyway migration replay verified**: After restore, `spring.flyway.enabled=true` applies any pending migrations on startup. -- [ ] **Recovery Time Objective (RTO) defined**: Maximum acceptable downtime for the plugin system (e.g., 4 hours). +- [ ] **Database backup restoration tested**: A full restore from backup has been practiced and + documented. +- [ ] **Flyway migration replay verified**: After restore, `spring.flyway.enabled=true` applies any + pending migrations on startup. +- [ ] **Recovery Time Objective (RTO) defined**: Maximum acceptable downtime for the plugin system + (e.g., 4 hours). ### Rollback Plan - [ ] **Plugin rollback procedure**: If a loaded plugin causes issues: 1. `POST /jobs/definitions/{id}/unload?force=true` — unload the problematic plugin - 2. `DELETE /jobs/definitions/{id}` — remove the definition - 3. Upload the previous working version + 1. `DELETE /jobs/definitions/{id}` — remove the definition + 1. Upload the previous working version - [ ] **Rollback tested**: A rollback drill has been completed within the RTO window. ---- +______________________________________________________________________ ## 🧪 Pre-Deployment Validation - [ ] **All unit tests pass**: `mvn test -f fr-batch-service/pom.xml` — 120+ tests, 0 failures. -- [ ] **Integration tests pass**: All `@SpringBootTest` integration tests pass against a test database. -- [ ] **Plugin end-to-end test**: Full lifecycle test (upload → approve → load → execute → unload → delete) with a real plugin JAR. -- [ ] **Security scan**: Dependency vulnerability scan (`mvn dependency-check:check` or OWASP) with no critical/high findings. -- [ ] **Load test**: Simulate concurrent upload (5), load (5), and execution (10) operations. Verify no deadlocks, OOM, or excessive latency. +- [ ] **Integration tests pass**: All `@SpringBootTest` integration tests pass against a test + database. +- [ ] **Plugin end-to-end test**: Full lifecycle test (upload → approve → load → execute → unload → + delete) with a real plugin JAR. +- [ ] **Security scan**: Dependency vulnerability scan (`mvn dependency-check:check` or OWASP) with + no critical/high findings. +- [ ] **Load test**: Simulate concurrent upload (5), load (5), and execution (10) operations. Verify + no deadlocks, OOM, or excessive latency. ---- +______________________________________________________________________ ## 📝 Operational Runbooks -- [ ] **Startup procedure**: Documented steps for starting the service (JVM args, env vars, dependency startup order). -- [ ] **Shutdown procedure**: Graceful shutdown with `management.endpoint.shutdown.enabled=true` and step drain for running jobs. -- [ ] **Plugin lifecycle runbook**: Step-by-step for uploading, approving, loading, and executing a new plugin. -- [ ] **Incident response**: Triage steps for common failures (health DOWN, load failure, execution error, signature rejection). +- [ ] **Startup procedure**: Documented steps for starting the service (JVM args, env vars, + dependency startup order). +- [ ] **Shutdown procedure**: Graceful shutdown with `management.endpoint.shutdown.enabled=true` and + step drain for running jobs. +- [ ] **Plugin lifecycle runbook**: Step-by-step for uploading, approving, loading, and executing a + new plugin. +- [ ] **Incident response**: Triage steps for common failures (health DOWN, load failure, execution + error, signature rejection). - [ ] **Contact information**: On-call rotation or escalation contacts for plugin system issues. ---- +______________________________________________________________________ ## ☑️ Final Sign-off -| Role | Name | Date | Signature | -|------|------|------|-----------| -| Developer | | | | -| Security Review | | | | -| Operations Review | | | | -| Product Owner | | | | +| Role | Name | Date | Signature | +| ----------------- | ---- | ---- | --------- | +| Developer | | | | +| Security Review | | | | +| Operations Review | | | | +| Product Owner | | | | -**Date of deployment**: __________________ +**Date of deployment**: \_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_ -**Deployed by**: __________________ +**Deployed by**: \_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_ diff --git a/fr-batch-service/docs/runbook.md b/fr-batch-service/docs/runbook.md index 0ba7639b..52357472 100644 --- a/fr-batch-service/docs/runbook.md +++ b/fr-batch-service/docs/runbook.md @@ -20,16 +20,19 @@ └─────────────┘ ``` -- **fr-batch-service**: Spring Boot 3.x application serving the plugin management REST API and executing Spring Batch jobs loaded from dynamic plugin JARs. -- **MySQL 8.2**: Persistent store for job definitions, approval records, audit log, and Spring Batch metadata tables. +- **fr-batch-service**: Spring Boot 3.x application serving the plugin management REST API and + executing Spring Batch jobs loaded from dynamic plugin JARs. +- **MySQL 8.2**: Persistent store for job definitions, approval records, audit log, and Spring Batch + metadata tables. - **Prometheus**: Scrapes Micrometer metrics every 15 s from the `/actuator/prometheus` endpoint. - **Grafana**: Pre-built dashboard monitoring plugin lifecycle, JVM health, and HTTP error rates. ---- +______________________________________________________________________ ## 2. Startup ### Prerequisites + - Docker Engine ≥ 24.0 and Docker Compose ≥ 2.20 installed. - Environment variable `DB_PASSWORD` set (root password for MySQL). - Optional: `GRAFANA_PASSWORD` (defaults to `admin`). @@ -57,9 +60,11 @@ curl -s http://localhost:8080/api/batch-service/jobs/plugins | jq . ``` ### Expected result -All 4 containers show `healthy` in `docker compose ps`. The health endpoint returns `{"status":"UP"}`. Grafana login works and the plugin dashboard shows live metrics. ---- +All 4 containers show `healthy` in `docker compose ps`. The health endpoint returns +`{"status":"UP"}`. Grafana login works and the plugin dashboard shows live metrics. + +______________________________________________________________________ ## 3. Shutdown @@ -82,16 +87,18 @@ docker compose down ``` ### Force shutdown (emergency) + ```bash docker compose down -v # WARNING: destroys volumes — use only in emergency ``` ### Verification + ```bash docker compose ps # Should show "Exited" for all services ``` ---- +______________________________________________________________________ ## 4. Plugin Lifecycle @@ -111,6 +118,7 @@ curl -X POST -u admin:admin123 \ **Expected**: HTTP 201 Created. Response includes `jobDefinitionId` and `checksumSha256`. **Error handling**: + - HTTP 400 — invalid JAR, missing manifest, or signature verification failed (strict mode). - HTTP 409 — job name already exists. Use `PUT` to update the existing definition. - HTTP 413 — JAR exceeds `max-file-size` (50MB default). @@ -126,7 +134,8 @@ curl -X PUT -u admin:admin123 \ **Expected**: HTTP 200. `approvalStatus` changes to `APPROVED`. -**Note**: In production, `app.plugins.approval.auto-approve=false`, so this step is required before loading. +**Note**: In production, `app.plugins.approval.auto-approve=false`, so this step is required before +loading. ### 4.3 Load a plugin @@ -138,6 +147,7 @@ curl -X POST -u admin:admin123 \ **Expected**: HTTP 200. Plugin is loaded into the classloader and registered as a Spring Batch job. **Error handling**: + - HTTP 403 — definition not yet approved. - HTTP 409 — already loaded. - HTTP 500 — classloader error (check logs for `DynamicJobClassLoader` stack trace). @@ -161,9 +171,11 @@ curl -X POST -u admin:admin123 \ **Expected**: HTTP 200. Plugin classloader is closed; job is no longer available for execution. **Error handling**: -- HTTP 409 — job is currently executing. Add `?force=true` to force-unload (may interrupt running jobs). ---- +- HTTP 409 — job is currently executing. Add `?force=true` to force-unload (may interrupt running + jobs). + +______________________________________________________________________ ## 5. Backup & Restore @@ -197,7 +209,7 @@ docker compose up -d app curl -s http://localhost:8080/api/batch-service/actuator/health ``` ---- +______________________________________________________________________ ## 6. Incident Response @@ -206,6 +218,7 @@ curl -s http://localhost:8080/api/batch-service/actuator/health **Symptoms**: `/actuator/health` returns `DOWN`. Load balancer removes instance. **Diagnosis**: + ```bash # Check which component is down curl -s http://localhost:8080/api/batch-service/actuator/health | jq . @@ -218,16 +231,18 @@ docker compose exec app df -h /data/plugins/jars ``` **Resolution**: + 1. If MySQL is unreachable → check MySQL logs: `docker compose logs mysql` -2. If disk full → see §6.4 -3. If plugin indicator DOWN → see §6.2 -4. Restart app: `docker compose restart app` +1. If disk full → see §6.4 +1. If plugin indicator DOWN → see §6.2 +1. Restart app: `docker compose restart app` ### 6.2 Plugin Load Failure **Symptoms**: `POST /load` returns HTTP 500. Plugin appears as UNLOADED. **Diagnosis**: + ```bash # Check app logs for classloader errors docker compose logs app | grep -i "classloader\|LoadFailure" @@ -240,15 +255,18 @@ docker compose exec mysql mysql -u root -p"${DB_PASSWORD}" -e \ ``` **Resolution**: + 1. Corrupted JAR → re-upload: `POST /jobs/upload` -2. Missing dependency in JAR → check `MANIFEST.MF` classpath -3. Signature mismatch → re-sign JAR or temporarily set `app.plugins.signature.mode=permissive` during investigation +1. Missing dependency in JAR → check `MANIFEST.MF` classpath +1. Signature mismatch → re-sign JAR or temporarily set `app.plugins.signature.mode=permissive` + during investigation ### 6.3 Execution Error **Symptoms**: Job fails with `FAILED` status. Exception in logs. **Diagnosis**: + ```bash # View recent job executions curl -s -u admin:admin123 \ @@ -259,30 +277,36 @@ docker compose logs app | grep "executionId={id}" ``` **Resolution**: + 1. Review job logic in plugin JAR (business error) -2. Force-stop stuck execution: `POST /jobs/executions/{id}/stop` -3. Unload and reload the plugin if it left corrupted state -4. If job requires data fix, run ad-hoc SQL after unload +1. Force-stop stuck execution: `POST /jobs/executions/{id}/stop` +1. Unload and reload the plugin if it left corrupted state +1. If job requires data fix, run ad-hoc SQL after unload ### 6.4 Disk Full **Symptoms**: Uploads fail with HTTP 507. Write errors in logs. **Diagnosis**: + ```bash docker compose exec app df -h /data/plugins/jars docker compose exec app du -sh /data/plugins/jars/* ``` **Resolution**: + 1. Identify stale/orphaned JARs (not referenced in `job_definitions` table) -2. Remove orphaned JARs inside the container (never on the host): `docker compose exec app sh -c 'test -f /data/plugins/jars/{orphanId}.jar && rm /data/plugins/jars/{orphanId}.jar'` -3. Unload and delete unused job definitions: `DELETE /jobs/definitions/{id}` -4. Expand Docker volume: `docker volume create --opt size=10G plugin-jars` (requires volume recreation) +1. Remove orphaned JARs inside the container (never on the host): + `docker compose exec app sh -c 'test -f /data/plugins/jars/{orphanId}.jar && rm /data/plugins/jars/{orphanId}.jar'` +1. Unload and delete unused job definitions: `DELETE /jobs/definitions/{id}` +1. Expand Docker volume: `docker volume create --opt size=10G plugin-jars` (requires volume + recreation) -**Escalation**: If disk usage exceeds 90% despite cleanup, expand the host partition or migrate to a larger volume. +**Escalation**: If disk usage exceeds 90% despite cleanup, expand the host partition or migrate to a +larger volume. ---- +______________________________________________________________________ ## 7. Monitoring @@ -330,9 +354,10 @@ groups: ### Grafana Dashboard Import `fr-batch-service/docs/grafana-dashboard.json` via: + 1. Grafana → Dashboards → New → Import -2. Upload JSON file or paste content -3. Select the Prometheus datasource +1. Upload JSON file or paste content +1. Select the Prometheus datasource ### Log Locations @@ -341,12 +366,13 @@ Import `fr-batch-service/docs/grafana-dashboard.json` via: - **Prometheus logs**: `docker compose logs prometheus` - **Grafana logs**: `docker compose logs grafana` -For centralized logging in production, configure a logging driver (`json-file`, `fluentd`, `loki`) in `docker-compose.yml`. +For centralized logging in production, configure a logging driver (`json-file`, `fluentd`, `loki`) +in `docker-compose.yml`. ### Escalation Contacts -| Role | Contact | -|------|---------| -| Primary on-call | [TBD — update before deployment] | -| Secondary on-call | [TBD] | -| Platform engineering | [TBD] | +| Role | Contact | +| -------------------- | -------------------------------- | +| Primary on-call | [TBD — update before deployment] | +| Secondary on-call | [TBD] | +| Platform engineering | [TBD] | From 845b5cbdc9c290b1730bdfaabb664f33492d9e89 Mon Sep 17 00:00:00 2001 From: Eduardo Flores Date: Fri, 16 Oct 2026 19:39:11 -0600 Subject: [PATCH 2/2] feat(plugins): coordinate definition delete with the plugin lifecycle DELETE /jobs/definitions/{id} now runs DynamicJobLoaderService.deleteJob under the per-definition lifecycle lock: - refuse with 409 when the definition has execution history (force does not bypass it; disable the definition instead) - unload first, refusing with 409 on running executions unless force=true - delete parameter templates and the definition in one transaction - delete the JAR only after commit; a failure leaves a logged orphan JAR named in the audit instead of failing the request - audit JOB_DELETED SUCCESS only after completion, FAILURE otherwise - unknown ids return 404 through the global error handler Closes #94 --- .../ADR/001-dynamic-job-loading-system.md | 38 +-- .../docs/job-plugins/developer-guide.md | 55 +++- fr-batch-service/docs/production-checklist.md | 9 +- fr-batch-service/docs/runbook.md | 8 +- .../loader/DynamicJobLoaderService.java | 165 ++++++++++- .../loader/JobDeleteConflictException.java | 20 ++ .../JobExecutionAuditRepository.java | 8 + .../web/GlobalExceptionHandler.java | 13 + .../web/JobManagementController.java | 74 ++--- .../loader/DynamicJobLoaderServiceTest.java | 277 ++++++++++++++++++ .../DynamicJobLoadingIntegrationTest.java | 65 ++++ .../web/PluginManagementIntegrationTest.java | 7 + 12 files changed, 653 insertions(+), 86 deletions(-) create mode 100644 fr-batch-service/src/main/java/com/fronzec/frbatchservice/batchjobs/plugins/loader/JobDeleteConflictException.java diff --git a/fr-batch-service/docs/ADR/001-dynamic-job-loading-system.md b/fr-batch-service/docs/ADR/001-dynamic-job-loading-system.md index 5be72462..26396365 100644 --- a/fr-batch-service/docs/ADR/001-dynamic-job-loading-system.md +++ b/fr-batch-service/docs/ADR/001-dynamic-job-loading-system.md @@ -722,27 +722,27 @@ public class JobManagementController { return ResponseEntity.ok(JobDefinitionDto.from(jobDef)); } - @DeleteMapping("/{jobName}") - public ResponseEntity deleteJob(@PathVariable String jobName) { - JobDefinition jobDef = jobDefinitionRepository.findByJobName(jobName) - .orElseThrow(() -> new JobNotFoundException(jobName)); - - // Unload if loaded - jobLoaderService.unloadJob(jobName); - - // Delete JAR file - try { - Files.deleteIfExists(Paths.get(jobDef.getJarFilePath())); - } catch (IOException e) { - log.warn("Failed to delete JAR file", e); - } - - // Delete from database - jobDefinitionRepository.delete(jobDef); - - return ResponseEntity.ok(StatusResponse.success("Job deleted")); + // Implemented as DELETE /jobs/definitions/{id}[?force=true] -> 204 No Content, + // delegating to DynamicJobLoaderService.deleteJob(id, force). + @DeleteMapping("/definitions/{id}") + public ResponseEntity deleteDefinition( + @PathVariable long id, @RequestParam(defaultValue = "false") boolean force) { + jobLoaderService.deleteJob(id, force); + return ResponseEntity.noContent().build(); } } + +// DynamicJobLoaderService.deleteJob(id, force), all under the per-definition lifecycle lock: +// 1. Unknown id -> NoSuchElementException (404). +// 2. Execution history in job_executions_audit -> JobDeleteConflictException (409) before +// anything is unloaded; force=true does not bypass it. Disable the definition instead. +// 3. Unload (reuses the unload path): running executions without force -> 409, nothing +// deleted; force=true skips the guard but does not stop running executions. +// 4. One transaction: delete parameter templates, then the definition row. +// 5. After commit: Files.deleteIfExists(jar). A missing JAR is fine; an I/O failure still +// succeeds, logs WARN, and names the orphan JAR in the audit detail. +// 6. Audit JOB_DELETED SUCCESS only after completion; any failure up to the commit audits +// JOB_DELETED FAILURE with id, jobName, and reason, then rethrows. ``` ### 6. Example Job Plugin Project diff --git a/fr-batch-service/docs/job-plugins/developer-guide.md b/fr-batch-service/docs/job-plugins/developer-guide.md index c5a78400..7ef59aa8 100644 --- a/fr-batch-service/docs/job-plugins/developer-guide.md +++ b/fr-batch-service/docs/job-plugins/developer-guide.md @@ -175,15 +175,15 @@ Roles below). ### Job Definition Management -| Method | Path | Auth | Description | -| -------- | -------------------------------- | ---------------- | ------------------------------------------ | -| `POST` | `/jobs/upload` | `PLUGIN_ADMIN` | Upload a JAR and register a job definition | -| `GET` | `/jobs/definitions` | `PLUGIN_VIEWER`+ | List all job definitions | -| `GET` | `/jobs/definitions/{id}` | `PLUGIN_VIEWER`+ | Get a single definition | -| `PUT` | `/jobs/definitions/{id}/enable` | `PLUGIN_ADMIN` | Enable a definition | -| `PUT` | `/jobs/definitions/{id}/disable` | `PLUGIN_ADMIN` | Disable a definition | -| `DELETE` | `/jobs/definitions/{id}` | `PLUGIN_ADMIN` | Delete definition and JAR file | -| `GET` | `/jobs/plugins` | Public | List merged plugin metadata | +| Method | Path | Auth | Description | +| -------- | ------------------------------------- | ---------------- | ----------------------------------------------------------------------- | +| `POST` | `/jobs/upload` | `PLUGIN_ADMIN` | Upload a JAR and register a job definition | +| `GET` | `/jobs/definitions` | `PLUGIN_VIEWER`+ | List all job definitions | +| `GET` | `/jobs/definitions/{id}` | `PLUGIN_VIEWER`+ | Get a single definition | +| `PUT` | `/jobs/definitions/{id}/enable` | `PLUGIN_ADMIN` | Enable a definition | +| `PUT` | `/jobs/definitions/{id}/disable` | `PLUGIN_ADMIN` | Disable a definition | +| `DELETE` | `/jobs/definitions/{id}[?force=true]` | `PLUGIN_ADMIN` | Unload, delete definition, then delete JAR file (see [Delete](#delete)) | +| `GET` | `/jobs/plugins` | Public | List merged plugin metadata | ### Upload @@ -260,6 +260,43 @@ curl -u admin:admin123 -X POST \ http://localhost:8080/api/batch-service/jobs/load-all ``` +### Delete + +`DELETE /jobs/definitions/{id}` is a single operation that runs under the same per-definition lock +as load, unload, and reload, so it cannot race with them for the same ID: + +1. A definition with execution history (rows in `job_executions_audit`) is rejected before anything + is unloaded. Disable it instead (`PUT /jobs/definitions/{id}/disable`). +1. The plugin is unloaded automatically (unregistered and its classloader released). A definition + that is not loaded is deleted without error. +1. The parameter templates and the definition row are deleted in one database transaction. +1. The JAR file is deleted only after that transaction commits. + +| Outcome | Response | +| ------------------------------------------------------- | ---------------------------------------------------------------------- | +| Deleted | `204 No Content` | +| Unknown ID | `404 Not Found` | +| Execution history exists (with or without `force=true`) | `409 Conflict`; nothing is unloaded or deleted; disable the definition | +| Running executions and `force=false` (default) | `409 Conflict`; nothing is deleted | +| Database delete fails | Error response; the transaction is rolled back and the JAR is kept | + +`force=true` skips only the running-execution guard; it does not stop running executions and does +not bypass the execution-history guard. A missing JAR file does not fail the request. If the JAR +cannot be removed after commit, the request still returns `204`, a `WARN` is logged, and the +`JOB_DELETED` success audit names the orphan JAR path. `JOB_DELETED` is audited only once the +outcome is known: `SUCCESS` after completion, `FAILURE` (with the definition ID and reason) when the +delete is rejected or fails. + +```bash +# Delete (fails with 409 while executions are running) +curl -u admin:admin123 -X DELETE \ + http://localhost:8080/api/batch-service/jobs/definitions/1 + +# Delete even if executions are running (they are not stopped) +curl -u admin:admin123 -X DELETE \ + "http://localhost:8080/api/batch-service/jobs/definitions/1?force=true" +``` + ### Job Execution | Method | Path | Auth | Description | diff --git a/fr-batch-service/docs/production-checklist.md b/fr-batch-service/docs/production-checklist.md index 07e750b9..ac9c8036 100644 --- a/fr-batch-service/docs/production-checklist.md +++ b/fr-batch-service/docs/production-checklist.md @@ -184,8 +184,13 @@ ______________________________________________________________________ ### Rollback Plan - [ ] **Plugin rollback procedure**: If a loaded plugin causes issues: - 1. `POST /jobs/definitions/{id}/unload?force=true` — unload the problematic plugin - 1. `DELETE /jobs/definitions/{id}` — remove the definition + 1. `DELETE /jobs/definitions/{id}?force=true` — unload the problematic plugin and remove the + definition in one operation (running executions are not stopped; the JAR is deleted after the + database commit, and an orphan JAR is logged and named in the `JOB_DELETED` audit) + 1. If DELETE returns `409` because the definition has execution history (`force=true` does not + bypass this), disable it with `PUT /jobs/definitions/{id}/disable` and unload it with + `POST /jobs/definitions/{id}/unload?force=true` instead. Note that a replacement upload with + the same job name is rejected as a duplicate while that definition exists. 1. Upload the previous working version - [ ] **Rollback tested**: A rollback drill has been completed within the RTO window. diff --git a/fr-batch-service/docs/runbook.md b/fr-batch-service/docs/runbook.md index 52357472..350d6ad0 100644 --- a/fr-batch-service/docs/runbook.md +++ b/fr-batch-service/docs/runbook.md @@ -299,7 +299,13 @@ docker compose exec app du -sh /data/plugins/jars/* 1. Identify stale/orphaned JARs (not referenced in `job_definitions` table) 1. Remove orphaned JARs inside the container (never on the host): `docker compose exec app sh -c 'test -f /data/plugins/jars/{orphanId}.jar && rm /data/plugins/jars/{orphanId}.jar'` -1. Unload and delete unused job definitions: `DELETE /jobs/definitions/{id}` +1. Delete unused job definitions with `DELETE /jobs/definitions/{id}`; the plugin is unloaded + automatically and the JAR is removed after the database commit. A `409` either means the + definition has execution history (it cannot be deleted, even with `?force=true`; disable it with + `PUT /jobs/definitions/{id}/disable` instead) or that executions are running (retry later or use + `?force=true`; running executions are not stopped). The error message says which. If the JAR + cannot be removed, the delete still succeeds and the `JOB_DELETED` audit names the orphan JAR + path to clean up with step 2. 1. Expand Docker volume: `docker volume create --opt size=10G plugin-jars` (requires volume recreation) diff --git a/fr-batch-service/src/main/java/com/fronzec/frbatchservice/batchjobs/plugins/loader/DynamicJobLoaderService.java b/fr-batch-service/src/main/java/com/fronzec/frbatchservice/batchjobs/plugins/loader/DynamicJobLoaderService.java index e43ac4f0..e0af4129 100644 --- a/fr-batch-service/src/main/java/com/fronzec/frbatchservice/batchjobs/plugins/loader/DynamicJobLoaderService.java +++ b/fr-batch-service/src/main/java/com/fronzec/frbatchservice/batchjobs/plugins/loader/DynamicJobLoaderService.java @@ -9,9 +9,15 @@ import com.fronzec.frbatchservice.batchjobs.plugins.entity.JobDefinitionEntity; import com.fronzec.frbatchservice.batchjobs.plugins.metrics.PluginMetrics; import com.fronzec.frbatchservice.batchjobs.plugins.repository.JobDefinitionRepository; +import com.fronzec.frbatchservice.batchjobs.plugins.repository.JobExecutionAuditRepository; +import com.fronzec.frbatchservice.batchjobs.plugins.repository.JobParameterTemplateRepository; import com.fronzec.frbatchservice.batchjobs.plugins.util.ChecksumUtil; import java.io.File; +import java.io.IOException; import java.net.URL; +import java.nio.file.Files; +import java.nio.file.InvalidPathException; +import java.nio.file.Path; import java.time.LocalDateTime; import java.util.Collections; import java.util.LinkedHashMap; @@ -28,10 +34,11 @@ import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.stereotype.Service; import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; /** * Orchestrates the full lifecycle of dynamically-loaded batch job plugins: load from JAR, - * instantiate, configure, register, and unload. + * instantiate, configure, register, unload, and delete. * *

Owns the {@link DynamicJobClassLoader} lifecycle via a {@link ConcurrentHashMap} keyed by * definition ID. Delegates registry operations to {@link PluginRegistryService}. @@ -48,6 +55,11 @@ public class DynamicJobLoaderService { private final PlatformTransactionManager transactionManager; private final AuditService auditService; private final PluginMetrics pluginMetrics; + private final JobParameterTemplateRepository jobParameterTemplateRepository; + private final JobExecutionAuditRepository jobExecutionAuditRepository; + + /** Runs the database part of {@link #deleteJob} (templates + definition) as one transaction. */ + private final TransactionTemplate deleteTransaction; /** Tracks active classloaders by definition ID for cleanup on unload. */ private final Map classLoaders = new ConcurrentHashMap<>(); @@ -68,7 +80,9 @@ public DynamicJobLoaderService( JobRepository jobRepository, PlatformTransactionManager transactionManager, AuditService auditService, - PluginMetrics pluginMetrics) { + PluginMetrics pluginMetrics, + JobParameterTemplateRepository jobParameterTemplateRepository, + JobExecutionAuditRepository jobExecutionAuditRepository) { this.jobDefinitionRepository = jobDefinitionRepository; this.pluginRegistryService = pluginRegistryService; this.applicationContext = applicationContext; @@ -76,6 +90,9 @@ public DynamicJobLoaderService( this.transactionManager = transactionManager; this.auditService = auditService; this.pluginMetrics = pluginMetrics; + this.jobParameterTemplateRepository = jobParameterTemplateRepository; + this.jobExecutionAuditRepository = jobExecutionAuditRepository; + this.deleteTransaction = new TransactionTemplate(transactionManager); } /** @@ -433,6 +450,150 @@ private LoadResult doUnloadJob(Long definitionId, boolean force) { entity.getJobName(), LoadResult.UNLOADED, "Successfully unloaded"); } + /** + * Deletes a job definition as one lifecycle-coordinated operation: unloads the plugin, deletes + * its parameter templates and definition row in a single database transaction, and only after + * that transaction commits removes the JAR file from disk. + * + *

The whole operation runs under the per-definition lifecycle lock, so it cannot interleave + * with {@link #loadJob}, {@link #unloadJob}, or {@link #reloadJob} for the same ID. A definition + * that is not currently loaded is deleted without error. + * + *

Before anything is unloaded, a definition with execution history in {@code + * job_executions_audit} is rejected with {@link JobDeleteConflictException}; such definitions + * should be disabled instead. {@code force=true} does not bypass this guard: it only skips the + * running-execution guard, and it does not stop running executions. + * + *

A missing JAR file is not an error. If the JAR cannot be removed after commit, the delete + * still succeeds, a warning is logged, and the {@code JOB_DELETED} success audit names the + * orphan JAR path. Any failure up to and including the commit emits a {@code JOB_DELETED} + * failure audit and propagates the original exception. + * + * @param definitionId the database ID of the job definition to delete + * @param force if {@code true}, skip the running-execution guard + * @throws NoSuchElementException if the definition does not exist + * @throws JobDeleteConflictException if the definition has execution history, regardless of + * {@code force}; nothing is unloaded or deleted + * @throws JobUnloadConflictException if the job has running executions and {@code force} is + * {@code false}; nothing is deleted + * @throws org.springframework.dao.DataAccessException if the database delete fails; the + * transaction is rolled back and the JAR file is kept + */ + public void deleteJob(Long definitionId, boolean force) { + if (jobDefinitionRepository.findById(definitionId).isEmpty()) { + throw new NoSuchElementException("Job definition not found for id: " + definitionId); + } + withDefinitionLock(definitionId, () -> doDeleteJob(definitionId, force)); + } + + /** Performs the actual delete while the caller owns the per-definition lifecycle lock. */ + private void doDeleteJob(Long definitionId, boolean force) { + JobDefinitionEntity entity = + jobDefinitionRepository + .findById(definitionId) + .orElseThrow( + () -> + new NoSuchElementException( + "Job definition not found for id: " + definitionId)); + String jobName = entity.getJobName(); + String jarFilePath = entity.getJarFilePath(); + + boolean unloaded = false; + try { + // History guard runs before unload so a rejected delete leaves the plugin untouched. + // force only bypasses the running-execution guard, never this one. + if (jobExecutionAuditRepository.existsByJobDefinitionId(definitionId)) { + throw new JobDeleteConflictException( + "Job definition \"" + + jobName + + "\" (id=" + + definitionId + + ") has execution history and cannot be deleted; disable it instead" + + " (PUT /jobs/definitions/" + + definitionId + + "/disable). force=true does not bypass this check."); + } + doUnloadJob(definitionId, force); + unloaded = true; + // doUnloadJob persisted the entity as UNLOADED, so delete by id inside the transaction + // instead of deleting the now-stale instance read above. + deleteTransaction.executeWithoutResult( + status -> { + jobParameterTemplateRepository.deleteByJobDefinitionId(definitionId); + jobDefinitionRepository.deleteById(definitionId); + }); + } catch (RuntimeException e) { + String outcome = + unloaded + ? "The plugin was unloaded but the definition, its parameter templates, and its" + + " JAR were kept." + : "Nothing was deleted."; + log.error( + "Failed to delete job definition id={} jobName='{}' force={}: {}", + definitionId, + jobName, + force, + e.getMessage(), + e); + auditService.logEvent( + new AuditEvent( + AuditEventType.JOB_DELETED, + jobName, + AuditService.currentUserId(), + "Delete failed for job definition id=" + + definitionId + + " (force=" + + force + + "): " + + e.getMessage() + + ". " + + outcome, + AuditEvent.FAILURE, + LocalDateTime.now())); + throw e; + } + + String jarOutcome = deleteJarAfterCommit(definitionId, jarFilePath); + log.info("Job definition deleted: id={}, jobName='{}', {}", definitionId, jobName, jarOutcome); + auditService.logEvent( + new AuditEvent( + AuditEventType.JOB_DELETED, + jobName, + AuditService.currentUserId(), + "Job definition deleted: id=" + definitionId + " (force=" + force + "); " + jarOutcome, + AuditEvent.SUCCESS, + LocalDateTime.now())); + } + + /** + * Removes the JAR file after the delete transaction committed. File-system problems never fail + * the delete: the definition is already gone, so the JAR is reported as an orphan instead. + * + * @return a human-readable JAR outcome for the success audit detail + */ + private String deleteJarAfterCommit(Long definitionId, String jarFilePath) { + if (jarFilePath == null || jarFilePath.isBlank()) { + log.warn("Job definition id={} had no JAR path; nothing to remove from disk", definitionId); + return "no JAR path recorded"; + } + try { + if (Files.deleteIfExists(Path.of(jarFilePath))) { + log.info("JAR file deleted from disk: {}", jarFilePath); + return "JAR deleted: " + jarFilePath; + } + log.info("JAR file already absent from disk: {}", jarFilePath); + return "JAR already absent: " + jarFilePath; + } catch (IOException | InvalidPathException | SecurityException e) { + log.warn( + "Job definition id={} deleted but its JAR could not be removed; orphan JAR left on" + + " disk: {} ({})", + definitionId, + jarFilePath, + e.toString()); + return "orphan JAR left on disk: " + jarFilePath + " (" + e + ")"; + } + } + /** * Reloads a job by unloading (forced) then loading sequentially. The operation is sequential, so * the status never appears partially ready. diff --git a/fr-batch-service/src/main/java/com/fronzec/frbatchservice/batchjobs/plugins/loader/JobDeleteConflictException.java b/fr-batch-service/src/main/java/com/fronzec/frbatchservice/batchjobs/plugins/loader/JobDeleteConflictException.java new file mode 100644 index 00000000..af891cfe --- /dev/null +++ b/fr-batch-service/src/main/java/com/fronzec/frbatchservice/batchjobs/plugins/loader/JobDeleteConflictException.java @@ -0,0 +1,20 @@ +/* 2026 */ +package com.fronzec.frbatchservice.batchjobs.plugins.loader; + +/** + * Thrown when a job definition cannot be deleted because its current state forbids it, for example + * because it has execution history in {@code job_executions_audit}. Unlike {@link + * JobUnloadConflictException}, {@code force=true} does not bypass this conflict. Maps to HTTP 409 + * Conflict in the controller layer. + */ +public class JobDeleteConflictException extends RuntimeException { + + /** + * Creates a delete conflict with an operator-facing message. + * + * @param message why the delete was rejected and what to do instead + */ + public JobDeleteConflictException(String message) { + super(message); + } +} diff --git a/fr-batch-service/src/main/java/com/fronzec/frbatchservice/batchjobs/plugins/repository/JobExecutionAuditRepository.java b/fr-batch-service/src/main/java/com/fronzec/frbatchservice/batchjobs/plugins/repository/JobExecutionAuditRepository.java index e2cf05a4..cf187815 100644 --- a/fr-batch-service/src/main/java/com/fronzec/frbatchservice/batchjobs/plugins/repository/JobExecutionAuditRepository.java +++ b/fr-batch-service/src/main/java/com/fronzec/frbatchservice/batchjobs/plugins/repository/JobExecutionAuditRepository.java @@ -9,4 +9,12 @@ public interface JobExecutionAuditRepository extends JpaRepository { List findByJobDefinitionId(long jobDefinitionId); + + /** + * Reports whether any execution history references the given job definition. + * + * @param jobDefinitionId the job definition ID + * @return {@code true} if at least one {@code job_executions_audit} row references it + */ + boolean existsByJobDefinitionId(long jobDefinitionId); } diff --git a/fr-batch-service/src/main/java/com/fronzec/frbatchservice/web/GlobalExceptionHandler.java b/fr-batch-service/src/main/java/com/fronzec/frbatchservice/web/GlobalExceptionHandler.java index fb3baeb3..9e1ce6a4 100644 --- a/fr-batch-service/src/main/java/com/fronzec/frbatchservice/web/GlobalExceptionHandler.java +++ b/fr-batch-service/src/main/java/com/fronzec/frbatchservice/web/GlobalExceptionHandler.java @@ -1,6 +1,7 @@ /* 2024-2025 */ package com.fronzec.frbatchservice.web; +import com.fronzec.frbatchservice.batchjobs.plugins.loader.JobDeleteConflictException; import com.fronzec.frbatchservice.batchjobs.plugins.loader.JobLoadException; import com.fronzec.frbatchservice.batchjobs.plugins.loader.JobUnloadConflictException; import com.fronzec.frbatchservice.batchjobs.plugins.service.DuplicateJobDefinitionException; @@ -113,6 +114,18 @@ public ResponseEntity handleUnloadConflict( .body(buildError(409, "Conflict", ex.getMessage(), request)); } + /** + * Maps {@link JobDeleteConflictException} — a definition that cannot be deleted, such as one + * with execution history; {@code force} does not bypass it — to {@code 409 Conflict}. + */ + @ExceptionHandler(JobDeleteConflictException.class) + public ResponseEntity handleDeleteConflict( + JobDeleteConflictException ex, HttpServletRequest request) { + log.warn("Job delete conflict: {}", ex.getMessage()); + return ResponseEntity.status(HttpStatus.CONFLICT) + .body(buildError(409, "Conflict", ex.getMessage(), request)); + } + /** * Maps {@link IllegalStateException} — e.g. a definition that is disabled or * already loaded — to {@code 409 Conflict}. diff --git a/fr-batch-service/src/main/java/com/fronzec/frbatchservice/web/JobManagementController.java b/fr-batch-service/src/main/java/com/fronzec/frbatchservice/web/JobManagementController.java index 07666e68..b4ff2702 100644 --- a/fr-batch-service/src/main/java/com/fronzec/frbatchservice/web/JobManagementController.java +++ b/fr-batch-service/src/main/java/com/fronzec/frbatchservice/web/JobManagementController.java @@ -8,14 +8,10 @@ import com.fronzec.frbatchservice.batchjobs.plugins.loader.DynamicJobLoaderService; import com.fronzec.frbatchservice.batchjobs.plugins.loader.LoadResult; import com.fronzec.frbatchservice.batchjobs.plugins.repository.JobDefinitionRepository; -import com.fronzec.frbatchservice.batchjobs.plugins.repository.JobParameterTemplateRepository; import com.fronzec.frbatchservice.batchjobs.plugins.service.JarUploadService; import com.fronzec.frbatchservice.web.dto.ApprovalRequest; import com.fronzec.frbatchservice.web.dto.JarUploadResponse; import com.fronzec.frbatchservice.web.dto.JobDefinitionResponse; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.time.LocalDateTime; import java.util.List; import java.util.Map; @@ -24,7 +20,6 @@ import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.transaction.annotation.Transactional; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -52,7 +47,7 @@ *

  • {@code GET /definitions/{id}} — get single definition
    • *
  • {@code PUT /definitions/{id}/enable} — enable a definition
    • *
  • {@code PUT /definitions/{id}/disable} — disable a definition
    • - *
  • {@code DELETE /definitions/{id}} — remove DB row + JAR file from disk
    • + *
  • {@code DELETE /definitions/{id}[?force=true]} — unload, delete DB rows, then remove JAR
    • *
  • {@code POST /definitions/{id}/load} — load plugin from JAR into registry
    • *
  • {@code POST /definitions/{id}/unload} — unregister plugin and close classloader
    • *
  • {@code POST /definitions/{id}/reload} — atomic unload + load
    • @@ -67,19 +62,16 @@ public class JobManagementController { private final JarUploadService jarUploadService; private final JobDefinitionRepository jobDefinitionRepository; - private final JobParameterTemplateRepository jobParameterTemplateRepository; private final DynamicJobLoaderService loaderService; private final AuditService auditService; public JobManagementController( JarUploadService jarUploadService, JobDefinitionRepository jobDefinitionRepository, - JobParameterTemplateRepository jobParameterTemplateRepository, DynamicJobLoaderService loaderService, AuditService auditService) { this.jarUploadService = jarUploadService; this.jobDefinitionRepository = jobDefinitionRepository; - this.jobParameterTemplateRepository = jobParameterTemplateRepository; this.loaderService = loaderService; this.auditService = auditService; } @@ -308,54 +300,30 @@ public ResponseEntity rejectDefinition( // ─── CRUD: delete ────────────────────────────────────────────────────────── /** - * Hard-deletes a job definition and its parameter template rows, and removes - * the associated JAR file from disk. + * Deletes a job definition as one lifecycle-coordinated operation, delegated to + * {@link DynamicJobLoaderService#deleteJob(Long, boolean)}: the plugin is unloaded + * first (unregistered and its classloader released), the parameter templates and + * definition row are deleted in one database transaction, and the JAR file is + * removed from disk only after that transaction commits. * *

    Returns {@code 204 No Content} on success, {@code 404 Not Found} if the - * definition does not exist, or {@code 500 Internal Server Error} if file - * deletion fails due to an I/O error. + * definition does not exist, and {@code 409 Conflict} if the definition has + * execution history (disable it instead; {@code force} does not bypass this) or + * if the job has running executions and {@code force} is {@code false}. On a + * conflict nothing is unloaded or deleted. A missing JAR file, or a JAR that + * cannot be removed after commit, does not fail the request; an orphan JAR is + * logged and named in the {@code JOB_DELETED} audit. + * + * @param id the job definition ID + * @param force if {@code true}, delete even when executions are running; running + * executions are not stopped, and the execution-history guard still + * applies + * @return {@code 204 No Content} on success */ - @Transactional @DeleteMapping("/definitions/{id}") - public ResponseEntity deleteDefinition(@PathVariable long id) { - Optional opt = jobDefinitionRepository.findById(id); - if (opt.isEmpty()) { - log.warn("Cannot delete — job definition not found: id={}", id); - return ResponseEntity.notFound().build(); - } - - JobDefinitionEntity entity = opt.get(); - String jarFilePath = entity.getJarFilePath(); - String jobName = entity.getJobName(); - - // ── audit before deletion (entity is gone afterwards) ────────────── - auditService.logEvent( - new AuditEvent( - AuditEventType.JOB_DELETED, - jobName, - AuditService.currentUserId(), - "Job definition deleted: id=" + id, - AuditEvent.SUCCESS, - LocalDateTime.now())); - - // Delete the JAR file from disk (if it exists) - try { - Path jarPath = Path.of(jarFilePath); - boolean deleted = Files.deleteIfExists(jarPath); - if (deleted) { - log.info("JAR file deleted from disk: {}", jarPath); - } else { - log.warn("JAR file not found on disk (already removed?): {}", jarPath); - } - } catch (IOException e) { - log.error("Failed to delete JAR file from disk: {} — {}", jarFilePath, e.getMessage()); - throw new RuntimeException("Failed to delete JAR file for definition " + id, e); - } - - // Cascade-delete parameter templates, then delete the entity - jobParameterTemplateRepository.deleteByJobDefinitionId(id); - jobDefinitionRepository.delete(entity); - log.info("Job definition deleted: id={}, jobName={}", id, entity.getJobName()); + public ResponseEntity deleteDefinition( + @PathVariable long id, @RequestParam(defaultValue = "false") boolean force) { + loaderService.deleteJob(id, force); return ResponseEntity.noContent().build(); } diff --git a/fr-batch-service/src/test/java/com/fronzec/frbatchservice/batchjobs/plugins/loader/DynamicJobLoaderServiceTest.java b/fr-batch-service/src/test/java/com/fronzec/frbatchservice/batchjobs/plugins/loader/DynamicJobLoaderServiceTest.java index 39f7c059..ef698ea3 100644 --- a/fr-batch-service/src/test/java/com/fronzec/frbatchservice/batchjobs/plugins/loader/DynamicJobLoaderServiceTest.java +++ b/fr-batch-service/src/test/java/com/fronzec/frbatchservice/batchjobs/plugins/loader/DynamicJobLoaderServiceTest.java @@ -10,10 +10,13 @@ import com.fronzec.frbatchservice.batchjobs.plugins.PluginRegistrationException; import com.fronzec.frbatchservice.batchjobs.plugins.PluginRegistryService; import com.fronzec.frbatchservice.batchjobs.plugins.audit.AuditEvent; +import com.fronzec.frbatchservice.batchjobs.plugins.audit.AuditEventType; import com.fronzec.frbatchservice.batchjobs.plugins.audit.AuditService; import com.fronzec.frbatchservice.batchjobs.plugins.entity.JobDefinitionEntity; import com.fronzec.frbatchservice.batchjobs.plugins.metrics.PluginMetrics; import com.fronzec.frbatchservice.batchjobs.plugins.repository.JobDefinitionRepository; +import com.fronzec.frbatchservice.batchjobs.plugins.repository.JobExecutionAuditRepository; +import com.fronzec.frbatchservice.batchjobs.plugins.repository.JobParameterTemplateRepository; import com.fronzec.frbatchservice.batchjobs.plugins.util.ChecksumUtil; import java.io.IOException; import java.nio.file.Files; @@ -29,6 +32,8 @@ import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.*; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.mockito.*; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.batch.core.job.Job; @@ -38,6 +43,7 @@ import org.springframework.batch.core.step.builder.StepBuilder; import org.springframework.batch.infrastructure.repeat.RepeatStatus; import org.springframework.context.ApplicationContext; +import org.springframework.dao.DataIntegrityViolationException; import org.springframework.transaction.PlatformTransactionManager; @ExtendWith(MockitoExtension.class) @@ -50,6 +56,8 @@ class DynamicJobLoaderServiceTest { @Mock private PlatformTransactionManager transactionManager; @Mock private AuditService auditService; @Mock private PluginMetrics pluginMetrics; + @Mock private JobParameterTemplateRepository jobParameterTemplateRepository; + @Mock private JobExecutionAuditRepository jobExecutionAuditRepository; @InjectMocks private DynamicJobLoaderService service; @@ -534,6 +542,275 @@ void unloadJob_definitionNotFound_throwsNoSuchElementException() { assertThrows(NoSuchElementException.class, () -> service.unloadJob(999L, false)); } + // ── Delete ──────────────────────────────────────────────────────────────── + + /** Returns the single {@code JOB_DELETED} audit event emitted by the service. */ + private AuditEvent singleDeletedAuditEvent() { + ArgumentCaptor captor = ArgumentCaptor.forClass(AuditEvent.class); + verify(auditService, atLeastOnce()).logEvent(captor.capture()); + List deleted = + captor.getAllValues().stream() + .filter(event -> event.type() == AuditEventType.JOB_DELETED) + .toList(); + assertEquals(1, deleted.size(), "Expected exactly one JOB_DELETED audit event: " + deleted); + return deleted.get(0); + } + + @Test + void deleteJob_loadedPlugin_unloadsDeletesRowsInOneTransactionThenDeletesJar() throws Exception { + JobDefinitionEntity entity = createEntity(40L, "delete-loaded-job", true, LoadResult.LOADED); + DynamicJobClassLoader classLoader = mock(DynamicJobClassLoader.class); + classLoadersOf(service).put(40L, classLoader); + Path jar = Path.of(validJarPath); + + when(jobDefinitionRepository.findById(40L)).thenReturn(Optional.of(entity)); + when(jobExecutionAuditRepository.existsByJobDefinitionId(40L)).thenReturn(false); + when(jobRepository.findRunningJobExecutions("delete-loaded-job")) + .thenReturn(Collections.emptySet()); + doAnswer( + invocation -> { + assertTrue(Files.exists(jar), "JAR must still exist when the transaction commits"); + return null; + }) + .when(transactionManager) + .commit(any()); + + service.deleteJob(40L, false); + + InOrder order = + inOrder(pluginRegistryService, jobParameterTemplateRepository, jobDefinitionRepository, + transactionManager); + order.verify(pluginRegistryService).unregisterDynamicPlugin("delete-loaded-job"); + order.verify(jobParameterTemplateRepository).deleteByJobDefinitionId(40L); + order.verify(jobDefinitionRepository).deleteById(40L); + order.verify(transactionManager).commit(any()); + verify(classLoader).cleanup(); + assertFalse(classLoadersOf(service).containsKey(40L)); + assertFalse(Files.exists(jar), "JAR must be deleted after commit"); + assertTrue(lifecycleLocksOf(service).isEmpty()); + + AuditEvent deleted = singleDeletedAuditEvent(); + assertEquals(AuditEvent.SUCCESS, deleted.outcome()); + assertEquals("delete-loaded-job", deleted.jobName()); + } + + @Test + void deleteJob_notLoadedDefinition_succeeds() { + JobDefinitionEntity entity = createEntity(41L, "delete-unloaded-job", true, LoadResult.UNLOADED); + + when(jobDefinitionRepository.findById(41L)).thenReturn(Optional.of(entity)); + when(jobExecutionAuditRepository.existsByJobDefinitionId(41L)).thenReturn(false); + when(jobRepository.findRunningJobExecutions("delete-unloaded-job")) + .thenReturn(Collections.emptySet()); + + service.deleteJob(41L, false); + + verify(jobParameterTemplateRepository).deleteByJobDefinitionId(41L); + verify(jobDefinitionRepository).deleteById(41L); + assertFalse(Files.exists(Path.of(validJarPath))); + assertEquals(AuditEvent.SUCCESS, singleDeletedAuditEvent().outcome()); + } + + @Test + void deleteJob_runningExecutionsWithoutForce_throwsConflictAndDeletesNothing() throws Exception { + JobDefinitionEntity entity = createEntity(42L, "delete-running-job", true, LoadResult.LOADED); + DynamicJobClassLoader classLoader = mock(DynamicJobClassLoader.class); + classLoadersOf(service).put(42L, classLoader); + + when(jobDefinitionRepository.findById(42L)).thenReturn(Optional.of(entity)); + when(jobExecutionAuditRepository.existsByJobDefinitionId(42L)).thenReturn(false); + when(jobRepository.findRunningJobExecutions("delete-running-job")) + .thenReturn(Set.of(mock(JobExecution.class))); + + assertThrows(JobUnloadConflictException.class, () -> service.deleteJob(42L, false)); + + verify(pluginRegistryService, never()).unregisterDynamicPlugin(any()); + verify(classLoader, never()).cleanup(); + assertSame(classLoader, classLoadersOf(service).get(42L)); + verifyNoInteractions(jobParameterTemplateRepository, transactionManager); + verify(jobDefinitionRepository, never()).deleteById(any()); + assertTrue(Files.exists(Path.of(validJarPath)), "JAR must be kept on conflict"); + + AuditEvent deleted = singleDeletedAuditEvent(); + assertEquals(AuditEvent.FAILURE, deleted.outcome()); + assertTrue(deleted.details().contains("id=42"), deleted.details()); + assertTrue(deleted.details().contains("running execution"), deleted.details()); + } + + @ParameterizedTest(name = "force={0}") + @ValueSource(booleans = {false, true}) + void deleteJob_executionHistory_throwsDeleteConflictBeforeUnloadEvenWhenForced(boolean force) + throws Exception { + JobDefinitionEntity entity = createEntity(48L, "delete-history-job", true, LoadResult.LOADED); + DynamicJobClassLoader classLoader = mock(DynamicJobClassLoader.class); + classLoadersOf(service).put(48L, classLoader); + + when(jobDefinitionRepository.findById(48L)).thenReturn(Optional.of(entity)); + when(jobExecutionAuditRepository.existsByJobDefinitionId(48L)).thenReturn(true); + + JobDeleteConflictException ex = + assertThrows(JobDeleteConflictException.class, () -> service.deleteJob(48L, force)); + assertTrue(ex.getMessage().contains("execution history"), ex.getMessage()); + assertTrue(ex.getMessage().contains("disable"), ex.getMessage()); + + verifyNoInteractions( + pluginRegistryService, jobRepository, jobParameterTemplateRepository, transactionManager); + verify(classLoader, never()).cleanup(); + assertSame(classLoader, classLoadersOf(service).get(48L)); + verify(jobDefinitionRepository, never()).save(any()); + verify(jobDefinitionRepository, never()).deleteById(any()); + assertEquals(LoadResult.LOADED, entity.getLoadStatus()); + assertTrue(Files.exists(Path.of(validJarPath)), "JAR must be kept on a history conflict"); + assertTrue(lifecycleLocksOf(service).isEmpty()); + + AuditEvent deleted = singleDeletedAuditEvent(); + assertEquals(AuditEvent.FAILURE, deleted.outcome()); + assertTrue(deleted.details().contains("id=48"), deleted.details()); + assertTrue(deleted.details().contains("execution history"), deleted.details()); + verify(auditService, never()) + .logEvent(argThat(event -> event.type() == AuditEventType.JOB_UNLOADED)); + } + + @Test + void deleteJob_runningExecutionsWithForce_deletesWithoutTouchingExecutions() { + JobDefinitionEntity entity = createEntity(43L, "delete-forced-job", true, LoadResult.LOADED); + + when(jobDefinitionRepository.findById(43L)).thenReturn(Optional.of(entity)); + when(jobExecutionAuditRepository.existsByJobDefinitionId(43L)).thenReturn(false); + + service.deleteJob(43L, true); + + verify(pluginRegistryService).unregisterDynamicPlugin("delete-forced-job"); + verify(jobDefinitionRepository).deleteById(43L); + verifyNoInteractions(jobRepository); + assertEquals(AuditEvent.SUCCESS, singleDeletedAuditEvent().outcome()); + } + + @Test + void deleteJob_missingJar_succeeds() { + JobDefinitionEntity entity = createEntity(44L, "delete-missing-jar-job", true, null); + String missingJar = tempDir.resolve("already-gone.jar").toString(); + entity.setJarFilePath(missingJar); + + when(jobDefinitionRepository.findById(44L)).thenReturn(Optional.of(entity)); + when(jobExecutionAuditRepository.existsByJobDefinitionId(44L)).thenReturn(false); + + service.deleteJob(44L, true); + + verify(jobDefinitionRepository).deleteById(44L); + AuditEvent deleted = singleDeletedAuditEvent(); + assertEquals(AuditEvent.SUCCESS, deleted.outcome()); + assertTrue(deleted.details().contains(missingJar), deleted.details()); + } + + @Test + void deleteJob_jarDeleteFails_stillSucceedsAndNamesOrphanJarInAudit() throws Exception { + // A non-empty directory cannot be removed by Files.deleteIfExists, which simulates an + // I/O failure on the JAR path after the database transaction committed. + Path undeletable = tempDir.resolve("undeletable.jar"); + Files.createDirectory(undeletable); + Files.createFile(undeletable.resolve("blocker")); + JobDefinitionEntity entity = createEntity(45L, "delete-orphan-job", true, null); + entity.setJarFilePath(undeletable.toString()); + + when(jobDefinitionRepository.findById(45L)).thenReturn(Optional.of(entity)); + when(jobExecutionAuditRepository.existsByJobDefinitionId(45L)).thenReturn(false); + + assertDoesNotThrow(() -> service.deleteJob(45L, true)); + + verify(jobDefinitionRepository).deleteById(45L); + assertTrue(Files.exists(undeletable)); + AuditEvent deleted = singleDeletedAuditEvent(); + assertEquals(AuditEvent.SUCCESS, deleted.outcome()); + assertTrue(deleted.details().contains("orphan"), deleted.details()); + assertTrue(deleted.details().contains(undeletable.toString()), deleted.details()); + } + + @Test + void deleteJob_databaseDeleteFails_auditsFailureRollsBackAndKeepsJar() { + JobDefinitionEntity entity = createEntity(46L, "delete-db-failure-job", true, null); + DataIntegrityViolationException failure = new DataIntegrityViolationException("db down"); + + when(jobDefinitionRepository.findById(46L)).thenReturn(Optional.of(entity)); + when(jobExecutionAuditRepository.existsByJobDefinitionId(46L)).thenReturn(false); + doThrow(failure).when(jobDefinitionRepository).deleteById(46L); + + DataIntegrityViolationException thrown = + assertThrows(DataIntegrityViolationException.class, () -> service.deleteJob(46L, true)); + + assertSame(failure, thrown); + verify(transactionManager).rollback(any()); + verify(transactionManager, never()).commit(any()); + assertTrue(Files.exists(Path.of(validJarPath)), "JAR must be kept when the DB delete fails"); + + AuditEvent deleted = singleDeletedAuditEvent(); + assertEquals(AuditEvent.FAILURE, deleted.outcome()); + assertEquals("delete-db-failure-job", deleted.jobName()); + assertTrue(deleted.details().contains("id=46"), deleted.details()); + assertTrue(deleted.details().contains("db down"), deleted.details()); + } + + @Test + void deleteJob_nonexistentId_throwsNotFoundAndRetainsNoLock() throws Exception { + when(jobDefinitionRepository.findById(999L)).thenReturn(Optional.empty()); + + assertThrows(NoSuchElementException.class, () -> service.deleteJob(999L, false)); + + assertTrue(lifecycleLocksOf(service).isEmpty()); + verify(jobDefinitionRepository, never()).deleteById(any()); + verifyNoInteractions(pluginRegistryService, jobParameterTemplateRepository); + } + + @Test + void deleteJob_holdsDefinitionLockThroughDatabaseDelete() throws Exception { + JobDefinitionEntity entity = createEntity(47L, "delete-lock-job", true, LoadResult.LOADED); + when(jobDefinitionRepository.findById(47L)).thenReturn(Optional.of(entity)); + when(jobExecutionAuditRepository.existsByJobDefinitionId(47L)).thenReturn(false); + + AtomicInteger unregisterCalls = new AtomicInteger(); + CountDownLatch secondUnregisterEntered = new CountDownLatch(1); + doAnswer( + invocation -> { + if (unregisterCalls.incrementAndGet() == 2) { + secondUnregisterEntered.countDown(); + } + return null; + }) + .when(pluginRegistryService) + .unregisterDynamicPlugin("delete-lock-job"); + + CountDownLatch dbDeleteEntered = new CountDownLatch(1); + CountDownLatch releaseDbDelete = new CountDownLatch(1); + doAnswer( + invocation -> { + dbDeleteEntered.countDown(); + assertTrue(releaseDbDelete.await(5, TimeUnit.SECONDS)); + return null; + }) + .when(jobDefinitionRepository) + .deleteById(47L); + + ExecutorService pool = Executors.newFixedThreadPool(2); + try { + Future delete = pool.submit(() -> service.deleteJob(47L, true)); + assertTrue(dbDeleteEntered.await(5, TimeUnit.SECONDS)); + + Future concurrentUnload = pool.submit(() -> service.unloadJob(47L, true)); + assertFalse( + secondUnregisterEntered.await(250, TimeUnit.MILLISECONDS), + "Concurrent unload must not enter while delete owns the lifecycle lock"); + + releaseDbDelete.countDown(); + delete.get(5, TimeUnit.SECONDS); + concurrentUnload.get(5, TimeUnit.SECONDS); + assertEquals(2, unregisterCalls.get()); + assertTrue(lifecycleLocksOf(service).isEmpty()); + } finally { + releaseDbDelete.countDown(); + pool.shutdownNow(); + } + } + // ── Reload ──────────────────────────────────────────────────────────────── @Test diff --git a/fr-batch-service/src/test/java/com/fronzec/frbatchservice/batchjobs/plugins/loader/DynamicJobLoadingIntegrationTest.java b/fr-batch-service/src/test/java/com/fronzec/frbatchservice/batchjobs/plugins/loader/DynamicJobLoadingIntegrationTest.java index 72bcb536..668ef6cd 100644 --- a/fr-batch-service/src/test/java/com/fronzec/frbatchservice/batchjobs/plugins/loader/DynamicJobLoadingIntegrationTest.java +++ b/fr-batch-service/src/test/java/com/fronzec/frbatchservice/batchjobs/plugins/loader/DynamicJobLoadingIntegrationTest.java @@ -14,6 +14,7 @@ import com.fronzec.frbatchservice.batchjobs.plugins.PluginRegistryService; import com.fronzec.frbatchservice.batchjobs.plugins.entity.JobDefinitionEntity; import com.fronzec.frbatchservice.batchjobs.plugins.repository.JobDefinitionRepository; +import com.fronzec.frbatchservice.batchjobs.plugins.repository.JobExecutionAuditRepository; import com.fronzec.frbatchservice.restclients.ApiClient; import com.fronzec.frbatchservice.restclients.DataCalculatedResponse; import com.fronzec.frbatchservice.web.SingleJobDataRequest; @@ -82,6 +83,7 @@ class DynamicJobLoadingIntegrationTest { @Autowired private JobsManagerService jobsManagerService; @Autowired private PluginRegistryService pluginRegistryService; @Autowired private DynamicJobLoaderService loaderService; + @Autowired private JobExecutionAuditRepository jobExecutionAuditRepository; @MockitoBean private ApiClient apiClient; @@ -434,4 +436,67 @@ void loadAllEnabled_reloadsDefinitionWithStaleLoadedStatus() { .isEqualTo("Successfully loaded"); assertThat(pluginRegistryService.getRegisteredJobNames()).contains(JOB_NAME); } + + /* ── 14. DELETE with execution history → 409 before unload, nothing changes ── */ + + @Test + @Order(14) + void deleteWithExecutionHistory_returnsConflict_andKeepsPluginRowAndJar() throws Exception { + // Test 11 executed the job, so job_executions_audit rows reference this definition. + // The history guard runs before unload, so even force=true must leave everything intact. + assertThat(jobExecutionAuditRepository.existsByJobDefinitionId(definitionId)).isTrue(); + assertThat(pluginRegistryService.getRegisteredJobNames()).contains(JOB_NAME); + Path storedJar = + Path.of(jobDefinitionRepository.findById(definitionId).orElseThrow().getJarFilePath()); + + mockMvc + .perform(delete(JOBS_BASE + "/definitions/" + definitionId).param("force", "true")) + .andExpect(status().isConflict()) + .andExpect(jsonPath("$.status").value(409)) + .andExpect(jsonPath("$.error").value("Conflict")) + .andExpect(jsonPath("$.message").value(containsString("execution history"))) + .andExpect(jsonPath("$.path").value(JOBS_BASE + "/definitions/" + definitionId)); + + assertThat(jobDefinitionRepository.existsById(definitionId)).isTrue(); + assertThat(storedJar).exists(); + assertThat(pluginRegistryService.getRegisteredJobNames()).contains(JOB_NAME); + } + + /* ── 15. DELETE a loaded plugin → 204, unregistered, row and JAR removed ── */ + + @Test + @Order(15) + void deleteLoadedPlugin_returnsNoContent_andUnregistersJob() throws Exception { + // Remove the execution history that blocked test 14 so this test isolates the + // lifecycle behaviour of deleting a loaded plugin. + jobExecutionAuditRepository.deleteAll( + jobExecutionAuditRepository.findByJobDefinitionId(definitionId)); + if (!pluginRegistryService.getRegisteredJobNames().contains(JOB_NAME)) { + loaderService.loadJob(definitionId); + } + assertThat(pluginRegistryService.getRegisteredJobNames()).contains(JOB_NAME); + Path storedJar = + Path.of(jobDefinitionRepository.findById(definitionId).orElseThrow().getJarFilePath()); + assertThat(storedJar).exists(); + + mockMvc + .perform(delete(JOBS_BASE + "/definitions/" + definitionId)) + .andExpect(status().isNoContent()); + + assertThat(pluginRegistryService.getRegisteredJobNames()).doesNotContain(JOB_NAME); + assertThat(jobDefinitionRepository.existsById(definitionId)).isFalse(); + assertThat(storedJar).doesNotExist(); + } + + /* ── 16. DELETE an unknown definition → 404 error envelope ────────────── */ + + @Test + @Order(16) + void deleteNonExistentDefinition_returnsNotFound() throws Exception { + mockMvc + .perform(delete(JOBS_BASE + "/definitions/99999")) + .andExpect(status().isNotFound()) + .andExpect(jsonPath("$.status").value(404)) + .andExpect(jsonPath("$.error").value("Not Found")); + } } diff --git a/fr-batch-service/src/test/java/com/fronzec/frbatchservice/web/PluginManagementIntegrationTest.java b/fr-batch-service/src/test/java/com/fronzec/frbatchservice/web/PluginManagementIntegrationTest.java index 6180622e..bfab122a 100644 --- a/fr-batch-service/src/test/java/com/fronzec/frbatchservice/web/PluginManagementIntegrationTest.java +++ b/fr-batch-service/src/test/java/com/fronzec/frbatchservice/web/PluginManagementIntegrationTest.java @@ -241,6 +241,13 @@ void deleteDefinition_returnsNoContent_andSubsequentGetReturnsNotFound() throws mockMvc .perform(get(JOBS_BASE + "/definitions/" + createdDefinitionId)) .andExpect(status().isNotFound()); + + // A repeated DELETE of the same id is now an unknown definition: 404 error envelope + mockMvc + .perform(delete(JOBS_BASE + "/definitions/" + createdDefinitionId)) + .andExpect(status().isNotFound()) + .andExpect(jsonPath("$.status").value(404)) + .andExpect(jsonPath("$.path").value(JOBS_BASE + "/definitions/" + createdDefinitionId)); } // ─── merged plugins endpoint ───────────────────────────────────────────────