From bda060bee468b5479d41a296bd20f58a0bee2f5f Mon Sep 17 00:00:00 2001 From: Janaka Premathilaka Date: Fri, 10 Apr 2026 14:15:44 -0700 Subject: [PATCH] Add Phase 5 AI application patterns and evaluation harness --- README.md | 11 ++ docs/phase-5-ai-era-patterns.md | 42 +++++++ .../org/example/ai/EvaluationHarness.java | 56 ++++++++++ .../org/example/ai/FallbackModelStrategy.java | 32 ++++++ src/main/java/org/example/ai/LlmClient.java | 5 + .../example/ai/PolicyEnforcedLlmClient.java | 91 ++++++++++++++++ .../java/org/example/ai/PromptBuilder.java | 27 +++++ .../org/example/ai/RetrievalPipeline.java | 40 +++++++ .../java/org/example/ai/ToolAgentFacade.java | 45 ++++++++ .../example/AiApplicationPatternsTest.java | 103 ++++++++++++++++++ 10 files changed, 452 insertions(+) create mode 100644 docs/phase-5-ai-era-patterns.md create mode 100644 src/main/java/org/example/ai/EvaluationHarness.java create mode 100644 src/main/java/org/example/ai/FallbackModelStrategy.java create mode 100644 src/main/java/org/example/ai/LlmClient.java create mode 100644 src/main/java/org/example/ai/PolicyEnforcedLlmClient.java create mode 100644 src/main/java/org/example/ai/PromptBuilder.java create mode 100644 src/main/java/org/example/ai/RetrievalPipeline.java create mode 100644 src/main/java/org/example/ai/ToolAgentFacade.java create mode 100644 src/test/java/example/AiApplicationPatternsTest.java diff --git a/README.md b/README.md index 2297052..4316f3a 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Phase tracking: - [Phase 2 — Expand from GoF to enterprise patterns](docs/phase-2-enterprise-patterns.md) - [Phase 3 — Modern Java implementation variants](docs/phase-3-modern-java-variants.md) - [Phase 4 — Cloud-native + observability examples](docs/phase-4-cloud-native-observability.md) +- [Phase 5 — AI-era additions (high practical value)](docs/phase-5-ai-era-patterns.md) --- @@ -109,6 +110,16 @@ Package: `src/main/java/org/example/cloudnative/observability` - **Saga correlation IDs** (trace attributes and correlated execution) - **Resilience fault-injection scenarios** (Testcontainers + WireMock) +### 10) AI application patterns +Package: `src/main/java/org/example/ai` + +- **Prompt Builder pattern** +- **Retrieval pipeline (RAG-lite) orchestration** +- **Tool/Agent orchestration facade** +- **Fallback model strategy** +- **Policy & safety wrappers** (input validation, output filtering, PII masking) +- **Evaluation harness** (golden set + statistical scoring) + --- ## Pattern index matrix diff --git a/docs/phase-5-ai-era-patterns.md b/docs/phase-5-ai-era-patterns.md new file mode 100644 index 0000000..de1662c --- /dev/null +++ b/docs/phase-5-ai-era-patterns.md @@ -0,0 +1,42 @@ +# Phase 5 — AI-era additions (high practical value) + +This phase introduces practical AI application patterns that teams can use to structure prompt workflows, retrieval pipelines, model orchestration, policy enforcement, and repeatable evaluation. + +## AI application patterns module + +Package: `org.example.ai` + +### 1) Prompt Builder pattern +- `PromptBuilder` composes reusable prompt templates with parameter placeholders. +- Keeps prompt assembly explicit and testable. + +### 2) Retrieval pipeline (RAG-lite) orchestration +- `RetrievalPipeline` separates retrieval and reranking steps. +- Builds bounded context windows suitable for lightweight RAG flows. + +### 3) Tool/Agent orchestration facade +- `ToolAgentFacade` exposes one entrypoint for tool calls and LLM-based responses. +- Supports a simple `tool::` command route plus LLM answering path. + +### 4) Fallback model strategy +- `FallbackModelStrategy` tries model clients in order. +- Moves to the next model when previous responses fail or return empty output. + +### 5) Policy & safety wrappers +- `PolicyEnforcedLlmClient` decorates `LlmClient` with: + - input validation (`NonEmptyInputPolicy`, `MaxLengthInputPolicy`), + - PII masking (`PiiMaskingInputPolicy`), + - output filtering (`BasicOutputFilterPolicy`). + +### 6) Evaluation harness +- `EvaluationHarness` provides: + - deterministic golden set checks (accuracy + failure diagnostics), + - statistical score aggregation (average/min/max). + +## Validation + +`AiApplicationPatternsTest` demonstrates end-to-end usage of: +- prompt + RAG-lite + facade orchestration, +- fallback model behavior, +- policy/safety wrappers, +- deterministic and statistical evaluation patterns. diff --git a/src/main/java/org/example/ai/EvaluationHarness.java b/src/main/java/org/example/ai/EvaluationHarness.java new file mode 100644 index 0000000..27eeeaa --- /dev/null +++ b/src/main/java/org/example/ai/EvaluationHarness.java @@ -0,0 +1,56 @@ +package org.example.ai; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; + +public final class EvaluationHarness { + + private EvaluationHarness() { + } + + public record GoldenExample(String input, String expectedOutput) { + } + + public record GoldenSetResult(double accuracy, List failures) { + } + + public record StatisticalResult(double averageScore, double minScore, double maxScore) { + } + + public static GoldenSetResult evaluateGoldenSet(List examples, Function model) { + int correct = 0; + List failures = new ArrayList<>(); + + for (GoldenExample example : examples) { + String actual = model.apply(example.input()); + if (example.expectedOutput().equals(actual)) { + correct++; + } else { + failures.add("input=" + example.input() + ", expected=" + example.expectedOutput() + ", actual=" + actual); + } + } + + double accuracy = examples.isEmpty() ? 0.0 : (double) correct / examples.size(); + return new GoldenSetResult(accuracy, failures); + } + + public static StatisticalResult evaluateStatistically(List inputs, Function scorer) { + if (inputs.isEmpty()) { + throw new IllegalArgumentException("inputs must not be empty"); + } + + double sum = 0.0; + double min = Double.POSITIVE_INFINITY; + double max = Double.NEGATIVE_INFINITY; + + for (String input : inputs) { + double score = scorer.apply(input); + sum += score; + min = Math.min(min, score); + max = Math.max(max, score); + } + + return new StatisticalResult(sum / inputs.size(), min, max); + } +} diff --git a/src/main/java/org/example/ai/FallbackModelStrategy.java b/src/main/java/org/example/ai/FallbackModelStrategy.java new file mode 100644 index 0000000..773b657 --- /dev/null +++ b/src/main/java/org/example/ai/FallbackModelStrategy.java @@ -0,0 +1,32 @@ +package org.example.ai; + +import java.util.List; + +public class FallbackModelStrategy implements LlmClient { + + private final List models; + + public FallbackModelStrategy(List models) { + this.models = models; + } + + @Override + public String complete(String prompt) { + RuntimeException lastError = null; + for (LlmClient model : models) { + try { + String response = model.complete(prompt); + if (response != null && !response.isBlank()) { + return response; + } + } catch (RuntimeException ex) { + lastError = ex; + } + } + + if (lastError != null) { + throw lastError; + } + throw new IllegalStateException("No model produced a response"); + } +} diff --git a/src/main/java/org/example/ai/LlmClient.java b/src/main/java/org/example/ai/LlmClient.java new file mode 100644 index 0000000..a639ee7 --- /dev/null +++ b/src/main/java/org/example/ai/LlmClient.java @@ -0,0 +1,5 @@ +package org.example.ai; + +public interface LlmClient { + String complete(String prompt); +} diff --git a/src/main/java/org/example/ai/PolicyEnforcedLlmClient.java b/src/main/java/org/example/ai/PolicyEnforcedLlmClient.java new file mode 100644 index 0000000..76c7d9e --- /dev/null +++ b/src/main/java/org/example/ai/PolicyEnforcedLlmClient.java @@ -0,0 +1,91 @@ +package org.example.ai; + +import java.util.List; +import java.util.regex.Pattern; + +public class PolicyEnforcedLlmClient implements LlmClient { + + public interface InputPolicy { + void validate(String input); + } + + public interface OutputPolicy { + String filter(String output); + } + + public static class NonEmptyInputPolicy implements InputPolicy { + @Override + public void validate(String input) { + if (input == null || input.isBlank()) { + throw new IllegalArgumentException("Prompt must not be blank"); + } + } + } + + public static class MaxLengthInputPolicy implements InputPolicy { + private final int maxLength; + + public MaxLengthInputPolicy(int maxLength) { + this.maxLength = maxLength; + } + + @Override + public void validate(String input) { + if (input.length() > maxLength) { + throw new IllegalArgumentException("Prompt exceeds max length " + maxLength); + } + } + } + + public static class PiiMaskingInputPolicy implements InputPolicy { + private static final Pattern EMAIL = Pattern.compile("[A-Za-z0-9+_.-]+@[A-Za-z0-9.-]+"); + + @Override + public void validate(String input) { + // no-op for validation; mask is applied via sanitize + } + + public String sanitize(String input) { + return EMAIL.matcher(input).replaceAll("[EMAIL_MASKED]"); + } + } + + public static class BasicOutputFilterPolicy implements OutputPolicy { + @Override + public String filter(String output) { + return output.replaceAll("(?i)password\\s*[:=]\\s*\\S+", "password:[REDACTED]"); + } + } + + private final LlmClient delegate; + private final List inputPolicies; + private final List outputPolicies; + private final PiiMaskingInputPolicy piiMaskingInputPolicy; + + public PolicyEnforcedLlmClient( + LlmClient delegate, + List inputPolicies, + List outputPolicies, + PiiMaskingInputPolicy piiMaskingInputPolicy + ) { + this.delegate = delegate; + this.inputPolicies = inputPolicies; + this.outputPolicies = outputPolicies; + this.piiMaskingInputPolicy = piiMaskingInputPolicy; + } + + @Override + public String complete(String prompt) { + for (InputPolicy policy : inputPolicies) { + policy.validate(prompt); + } + + String sanitizedPrompt = piiMaskingInputPolicy.sanitize(prompt); + String output = delegate.complete(sanitizedPrompt); + String filteredOutput = output; + for (OutputPolicy policy : outputPolicies) { + filteredOutput = policy.filter(filteredOutput); + } + return filteredOutput; + } +} diff --git a/src/main/java/org/example/ai/PromptBuilder.java b/src/main/java/org/example/ai/PromptBuilder.java new file mode 100644 index 0000000..18f6416 --- /dev/null +++ b/src/main/java/org/example/ai/PromptBuilder.java @@ -0,0 +1,27 @@ +package org.example.ai; + +import java.util.HashMap; +import java.util.Map; + +public class PromptBuilder { + + private final String template; + private final Map values = new HashMap<>(); + + public PromptBuilder(String template) { + this.template = template; + } + + public PromptBuilder with(String key, String value) { + values.put(key, value); + return this; + } + + public String build() { + String built = template; + for (Map.Entry entry : values.entrySet()) { + built = built.replace("{{" + entry.getKey() + "}}", entry.getValue()); + } + return built; + } +} diff --git a/src/main/java/org/example/ai/RetrievalPipeline.java b/src/main/java/org/example/ai/RetrievalPipeline.java new file mode 100644 index 0000000..6feacd0 --- /dev/null +++ b/src/main/java/org/example/ai/RetrievalPipeline.java @@ -0,0 +1,40 @@ +package org.example.ai; + +import java.util.List; +import java.util.stream.Collectors; + +public class RetrievalPipeline { + + public interface Retriever { + List retrieve(String query, int topK); + } + + public interface Reranker { + List rerank(String query, List documents); + } + + public record RetrievedDocument(String id, String content, double score) { + } + + private final Retriever retriever; + private final Reranker reranker; + + public RetrievalPipeline(Retriever retriever, Reranker reranker) { + this.retriever = retriever; + this.reranker = reranker; + } + + public String buildContext(String query, int topK, int maxChars) { + List retrieved = retriever.retrieve(query, topK); + List reranked = reranker.rerank(query, retrieved); + + String context = reranked.stream() + .map(RetrievedDocument::content) + .collect(Collectors.joining("\n---\n")); + + if (context.length() <= maxChars) { + return context; + } + return context.substring(0, maxChars); + } +} diff --git a/src/main/java/org/example/ai/ToolAgentFacade.java b/src/main/java/org/example/ai/ToolAgentFacade.java new file mode 100644 index 0000000..b5ae27f --- /dev/null +++ b/src/main/java/org/example/ai/ToolAgentFacade.java @@ -0,0 +1,45 @@ +package org.example.ai; + +import java.util.Map; + +public class ToolAgentFacade { + + public interface Tool { + String execute(String input); + } + + private final RetrievalPipeline retrievalPipeline; + private final LlmClient llmClient; + private final Map tools; + + public ToolAgentFacade(RetrievalPipeline retrievalPipeline, LlmClient llmClient, Map tools) { + this.retrievalPipeline = retrievalPipeline; + this.llmClient = llmClient; + this.tools = tools; + } + + public String answer(String userQuery) { + if (userQuery.startsWith("tool:")) { + return invokeTool(userQuery); + } + + String context = retrievalPipeline.buildContext(userQuery, 3, 500); + String prompt = new PromptBuilder("Use context to answer.\\nContext:\\n{{context}}\\nQuestion: {{question}}") + .with("context", context) + .with("question", userQuery) + .build(); + return llmClient.complete(prompt); + } + + private String invokeTool(String query) { + String[] parts = query.split(":", 3); + if (parts.length < 3) { + throw new IllegalArgumentException("Tool query must be tool::"); + } + Tool tool = tools.get(parts[1]); + if (tool == null) { + throw new IllegalArgumentException("Unknown tool: " + parts[1]); + } + return tool.execute(parts[2]); + } +} diff --git a/src/test/java/example/AiApplicationPatternsTest.java b/src/test/java/example/AiApplicationPatternsTest.java new file mode 100644 index 0000000..d8c52df --- /dev/null +++ b/src/test/java/example/AiApplicationPatternsTest.java @@ -0,0 +1,103 @@ +package example; + +import org.example.ai.EvaluationHarness; +import org.example.ai.FallbackModelStrategy; +import org.example.ai.LlmClient; +import org.example.ai.PolicyEnforcedLlmClient; +import org.example.ai.RetrievalPipeline; +import org.example.ai.ToolAgentFacade; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class AiApplicationPatternsTest { + + @Test + public void ragLiteAndToolFacadeOrchestrateCorrectly() { + RetrievalPipeline pipeline = new RetrievalPipeline( + (query, topK) -> List.of( + new RetrievalPipeline.RetrievedDocument("d1", "Java uses classes and objects.", 0.9), + new RetrievalPipeline.RetrievedDocument("d2", "Design patterns improve maintainability.", 0.8) + ), + (query, docs) -> docs + ); + + LlmClient llm = prompt -> prompt.contains("Design patterns") ? "Patterns are reusable solutions" : "unknown"; + + ToolAgentFacade facade = new ToolAgentFacade( + pipeline, + llm, + Map.of("uppercase", String::toUpperCase) + ); + + String answer = facade.answer("How do patterns help?"); + assertEquals("Patterns are reusable solutions", answer); + + String toolResult = facade.answer("tool:uppercase:hello"); + assertEquals("HELLO", toolResult); + } + + @Test + public void fallbackModelStrategyUsesSecondaryModelOnPrimaryFailure() { + LlmClient primary = prompt -> { + throw new RuntimeException("primary unavailable"); + }; + LlmClient secondary = prompt -> "secondary response"; + + FallbackModelStrategy strategy = new FallbackModelStrategy(List.of(primary, secondary)); + assertEquals("secondary response", strategy.complete("Explain retry strategy")); + } + + @Test + public void policyWrapperValidatesMasksAndFilters() { + AtomicReference seenPrompt = new AtomicReference<>(); + LlmClient base = prompt -> { + seenPrompt.set(prompt); + return "Use password=secret for admin"; + }; + + PolicyEnforcedLlmClient secured = new PolicyEnforcedLlmClient( + base, + List.of( + new PolicyEnforcedLlmClient.NonEmptyInputPolicy(), + new PolicyEnforcedLlmClient.MaxLengthInputPolicy(200) + ), + List.of(new PolicyEnforcedLlmClient.BasicOutputFilterPolicy()), + new PolicyEnforcedLlmClient.PiiMaskingInputPolicy() + ); + + String output = secured.complete("Email me at user@example.com"); + assertTrue(seenPrompt.get().contains("[EMAIL_MASKED]")); + assertEquals("Use password:[REDACTED] for admin", output); + assertThrows(IllegalArgumentException.class, () -> secured.complete(" ")); + } + + @Test + public void evaluationHarnessSupportsGoldenAndStatisticalEvaluation() { + EvaluationHarness.GoldenSetResult golden = EvaluationHarness.evaluateGoldenSet( + List.of( + new EvaluationHarness.GoldenExample("A", "yes"), + new EvaluationHarness.GoldenExample("B", "no") + ), + input -> input.equals("A") ? "yes" : "maybe" + ); + + assertEquals(0.5, golden.accuracy()); + assertEquals(1, golden.failures().size()); + + EvaluationHarness.StatisticalResult statistical = EvaluationHarness.evaluateStatistically( + List.of("tiny", "bigger", "largest"), + input -> Math.min(1.0, input.length() / 10.0) + ); + + assertEquals(0.5666666666666667, statistical.averageScore()); + assertEquals(0.4, statistical.minScore()); + assertEquals(0.7, statistical.maxScore()); + } +}