diff --git a/README.md b/README.md index 55a83f9..2297052 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Phase tracking: - [Phase 1 — Production-ready reference baseline](docs/phase-1-production-readiness.md) - [Phase 2 — Expand from GoF to enterprise patterns](docs/phase-2-enterprise-patterns.md) - [Phase 3 — Modern Java implementation variants](docs/phase-3-modern-java-variants.md) +- [Phase 4 — Cloud-native + observability examples](docs/phase-4-cloud-native-observability.md) --- @@ -100,6 +101,14 @@ Package: `src/main/java/org/example/architecture` - **Strangler Fig migration sample** (`strangler`) - **Anti-Corruption Layer** (`acl`) +### 9) Cloud-native + observability examples +Package: `src/main/java/org/example/cloudnative/observability` + +- **Command latency tracing** (OpenTelemetry spans + latency histogram) +- **Circuit breaker metrics** (success/failure/rejection counters) +- **Saga correlation IDs** (trace attributes and correlated execution) +- **Resilience fault-injection scenarios** (Testcontainers + WireMock) + --- ## Pattern index matrix diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml new file mode 100644 index 0000000..1140df9 --- /dev/null +++ b/deploy/docker-compose.yml @@ -0,0 +1,45 @@ +version: "3.9" + +services: + command-observability-demo: + image: ghcr.io/janaka2/designpatterns-command-demo:latest + environment: + OTEL_SERVICE_NAME: command-observability-demo + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 + ports: + - "8088:8080" + depends_on: + - otel-collector + + resilience-fault-demo: + image: ghcr.io/janaka2/designpatterns-resilience-demo:latest + environment: + UPSTREAM_BASE_URL: http://wiremock:8080 + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4317 + ports: + - "8090:8080" + depends_on: + - wiremock + - otel-collector + + wiremock: + image: wiremock/wiremock:3.13.1 + command: ["--verbose"] + ports: + - "8089:8080" + + otel-collector: + image: otel/opentelemetry-collector:0.122.1 + command: ["--config=/etc/otelcol/config.yaml"] + volumes: + - ./otel-collector-config.yaml:/etc/otelcol/config.yaml:ro + ports: + - "4317:4317" + - "4318:4318" + depends_on: + - jaeger + + jaeger: + image: jaegertracing/all-in-one:1.64.0 + ports: + - "16686:16686" diff --git a/deploy/k8s/command-observability-demo.yaml b/deploy/k8s/command-observability-demo.yaml new file mode 100644 index 0000000..4aa5c43 --- /dev/null +++ b/deploy/k8s/command-observability-demo.yaml @@ -0,0 +1,35 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: command-observability-demo +spec: + replicas: 1 + selector: + matchLabels: + app: command-observability-demo + template: + metadata: + labels: + app: command-observability-demo + spec: + containers: + - name: app + image: ghcr.io/janaka2/designpatterns-command-demo:latest + ports: + - containerPort: 8080 + env: + - name: OTEL_SERVICE_NAME + value: command-observability-demo + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://otel-collector:4317 +--- +apiVersion: v1 +kind: Service +metadata: + name: command-observability-demo +spec: + selector: + app: command-observability-demo + ports: + - port: 80 + targetPort: 8080 diff --git a/deploy/k8s/resilience-fault-demo.yaml b/deploy/k8s/resilience-fault-demo.yaml new file mode 100644 index 0000000..277a7cd --- /dev/null +++ b/deploy/k8s/resilience-fault-demo.yaml @@ -0,0 +1,55 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: resilience-fault-demo +spec: + replicas: 1 + selector: + matchLabels: + app: resilience-fault-demo + template: + metadata: + labels: + app: resilience-fault-demo + spec: + containers: + - name: app + image: ghcr.io/janaka2/designpatterns-resilience-demo:latest + ports: + - containerPort: 8080 + env: + - name: UPSTREAM_BASE_URL + value: http://wiremock:8080 + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://otel-collector:4317 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: wiremock +spec: + replicas: 1 + selector: + matchLabels: + app: wiremock + template: + metadata: + labels: + app: wiremock + spec: + containers: + - name: wiremock + image: wiremock/wiremock:3.13.1 + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: wiremock +spec: + selector: + app: wiremock + ports: + - port: 8080 + targetPort: 8080 diff --git a/deploy/otel-collector-config.yaml b/deploy/otel-collector-config.yaml new file mode 100644 index 0000000..b6d2484 --- /dev/null +++ b/deploy/otel-collector-config.yaml @@ -0,0 +1,21 @@ +receivers: + otlp: + protocols: + grpc: + http: + +exporters: + debug: + otlp/jaeger: + endpoint: jaeger:4317 + tls: + insecure: true + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [debug, otlp/jaeger] + metrics: + receivers: [otlp] + exporters: [debug] diff --git a/docs/phase-4-cloud-native-observability.md b/docs/phase-4-cloud-native-observability.md new file mode 100644 index 0000000..18b4b54 --- /dev/null +++ b/docs/phase-4-cloud-native-observability.md @@ -0,0 +1,46 @@ +# Phase 4 — Cloud-native + observability examples + +This phase adds observability-first examples and deployment-ready scaffolding for running selected pattern demos in containerized environments. + +## Observability-first pattern examples + +Package: `org.example.cloudnative.observability` + +### 1) Command latency trace +- `ObservedCommandExecutor` wraps command execution with an OpenTelemetry span (`command.execute`). +- Records latency metric `pattern.command.latency.ms` to make command performance visible in dashboards. + +### 2) Circuit breaker metrics +- `ObservedCircuitBreaker` wraps `CircuitBreaker` and emits counter metric `pattern.circuit_breaker.executions`. +- Counter outcome labels: `success`, `failure`, `rejected` (open circuit). + +### 3) Saga correlation IDs +- `CorrelatedSagaOrchestrator` accepts a `correlationId` and propagates it through: + - saga orchestration span, + - per-step action span, + - compensation spans on rollback. +- `CorrelatedSagaStep` keeps saga step contracts explicit and correlation-aware. + +## Resilience test scenarios + +`ResilienceFaultInjectionTest` introduces fault injection via **Testcontainers + WireMock**: + +- timeout behavior (`/timeout` fixed delay) +- partial/transient failures (`/inventory` stateful scenario) +- retries via `RetryExecutor` + +This verifies that resilience patterns remain understandable under stress and failure conditions. + +## Deployment-ready examples + +Minimal deployment assets are included under `deploy/`: + +- `deploy/docker-compose.yml` + - command observability demo + - resilience fault demo + - WireMock dependency simulator + - OpenTelemetry Collector + Jaeger +- `deploy/k8s/command-observability-demo.yaml` +- `deploy/k8s/resilience-fault-demo.yaml` + +These manifests are intentionally minimal so teams can copy them as a starting point for real service demos. diff --git a/pom.xml b/pom.xml index 57442e5..901c198 100644 --- a/pom.xml +++ b/pom.xml @@ -18,6 +18,8 @@ 10.12.4 4.8.6.6 0.8.12 + 1.46.0 + 1.20.6 @@ -56,6 +58,28 @@ ${reactor.version} test + + io.opentelemetry + opentelemetry-api + ${opentelemetry.version} + + + io.opentelemetry + opentelemetry-sdk + ${opentelemetry.version} + + + io.opentelemetry + opentelemetry-sdk-testing + ${opentelemetry.version} + test + + + org.testcontainers + testcontainers + ${testcontainers.version} + test + diff --git a/src/main/java/org/example/cloudnative/observability/CorrelatedSagaOrchestrator.java b/src/main/java/org/example/cloudnative/observability/CorrelatedSagaOrchestrator.java new file mode 100644 index 0000000..064502e --- /dev/null +++ b/src/main/java/org/example/cloudnative/observability/CorrelatedSagaOrchestrator.java @@ -0,0 +1,71 @@ +package org.example.cloudnative.observability; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public class CorrelatedSagaOrchestrator { + private final Tracer tracer; + + public CorrelatedSagaOrchestrator(Tracer tracer) { + this.tracer = Objects.requireNonNull(tracer); + } + + public void execute(String correlationId, List steps) { + Span sagaSpan = tracer.spanBuilder("saga.orchestration") + .setSpanKind(SpanKind.INTERNAL) + .setAttribute("pattern.type", "saga") + .setAttribute("saga.correlation_id", correlationId) + .startSpan(); + + List completed = new ArrayList<>(); + try { + for (CorrelatedSagaStep step : steps) { + runStep(correlationId, step); + completed.add(step); + } + sagaSpan.setStatus(StatusCode.OK); + } catch (RuntimeException ex) { + rollback(correlationId, completed); + sagaSpan.recordException(ex); + sagaSpan.setStatus(StatusCode.ERROR, "saga failed"); + throw new IllegalStateException("Correlated saga orchestration failed", ex); + } finally { + sagaSpan.end(); + } + } + + private void runStep(String correlationId, CorrelatedSagaStep step) { + Span span = tracer.spanBuilder("saga.step") + .setAttribute("saga.correlation_id", correlationId) + .setAttribute("saga.step", step.name()) + .startSpan(); + try { + step.action().accept(correlationId); + span.setStatus(StatusCode.OK); + } finally { + span.end(); + } + } + + private void rollback(String correlationId, List completed) { + for (int i = completed.size() - 1; i >= 0; i--) { + CorrelatedSagaStep step = completed.get(i); + Span compensationSpan = tracer.spanBuilder("saga.compensation") + .setAttribute("saga.correlation_id", correlationId) + .setAttribute("saga.step", step.name()) + .startSpan(); + try { + step.compensation().accept(correlationId); + compensationSpan.setStatus(StatusCode.OK); + } finally { + compensationSpan.end(); + } + } + } +} diff --git a/src/main/java/org/example/cloudnative/observability/CorrelatedSagaStep.java b/src/main/java/org/example/cloudnative/observability/CorrelatedSagaStep.java new file mode 100644 index 0000000..b0b9c9b --- /dev/null +++ b/src/main/java/org/example/cloudnative/observability/CorrelatedSagaStep.java @@ -0,0 +1,6 @@ +package org.example.cloudnative.observability; + +import java.util.function.Consumer; + +public record CorrelatedSagaStep(String name, Consumer action, Consumer compensation) { +} diff --git a/src/main/java/org/example/cloudnative/observability/ObservedCircuitBreaker.java b/src/main/java/org/example/cloudnative/observability/ObservedCircuitBreaker.java new file mode 100644 index 0000000..3bdd683 --- /dev/null +++ b/src/main/java/org/example/cloudnative/observability/ObservedCircuitBreaker.java @@ -0,0 +1,39 @@ +package org.example.cloudnative.observability; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.Meter; +import org.example.enterprise.resilience.CircuitBreaker; + +import java.util.Objects; +import java.util.function.Supplier; + +public class ObservedCircuitBreaker { + private static final AttributeKey OUTCOME = AttributeKey.stringKey("outcome"); + + private final CircuitBreaker delegate; + private final LongCounter executionCounter; + + public ObservedCircuitBreaker(CircuitBreaker delegate, Meter meter) { + this.delegate = Objects.requireNonNull(delegate); + this.executionCounter = Objects.requireNonNull(meter) + .counterBuilder("pattern.circuit_breaker.executions") + .setDescription("Circuit breaker execution outcomes") + .build(); + } + + public T execute(Supplier operation) { + try { + T result = delegate.execute(operation); + executionCounter.add(1, Attributes.of(OUTCOME, "success")); + return result; + } catch (IllegalStateException openCircuit) { + executionCounter.add(1, Attributes.of(OUTCOME, "rejected")); + throw openCircuit; + } catch (RuntimeException failure) { + executionCounter.add(1, Attributes.of(OUTCOME, "failure")); + throw failure; + } + } +} diff --git a/src/main/java/org/example/cloudnative/observability/ObservedCommandExecutor.java b/src/main/java/org/example/cloudnative/observability/ObservedCommandExecutor.java new file mode 100644 index 0000000..c1a7e9c --- /dev/null +++ b/src/main/java/org/example/cloudnative/observability/ObservedCommandExecutor.java @@ -0,0 +1,49 @@ +package org.example.cloudnative.observability; + +import io.opentelemetry.api.metrics.DoubleHistogram; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; +import io.opentelemetry.api.trace.Tracer; + +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +public class ObservedCommandExecutor { + private final Tracer tracer; + private final DoubleHistogram latencyHistogram; + + public ObservedCommandExecutor(Tracer tracer, Meter meter) { + this.tracer = Objects.requireNonNull(tracer); + this.latencyHistogram = Objects.requireNonNull(meter) + .histogramBuilder("pattern.command.latency.ms") + .setDescription("Command execution latency") + .setUnit("ms") + .build(); + } + + public T execute(String commandName, Supplier commandAction) { + long startNanos = System.nanoTime(); + Span span = tracer.spanBuilder("command.execute") + .setSpanKind(SpanKind.INTERNAL) + .setAttribute("pattern.type", "command") + .setAttribute("command.name", commandName) + .startSpan(); + + try { + T result = commandAction.get(); + span.setStatus(StatusCode.OK); + return result; + } catch (RuntimeException ex) { + span.recordException(ex); + span.setStatus(StatusCode.ERROR, "command failure"); + throw ex; + } finally { + double latencyMillis = TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - startNanos) / 1_000.0; + latencyHistogram.record(latencyMillis); + span.end(); + } + } +} diff --git a/src/test/java/example/ObservabilityPatternsTest.java b/src/test/java/example/ObservabilityPatternsTest.java new file mode 100644 index 0000000..49f5a04 --- /dev/null +++ b/src/test/java/example/ObservabilityPatternsTest.java @@ -0,0 +1,113 @@ +package example; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.metrics.export.InMemoryMetricReader; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import org.example.cloudnative.observability.CorrelatedSagaOrchestrator; +import org.example.cloudnative.observability.CorrelatedSagaStep; +import org.example.cloudnative.observability.ObservedCircuitBreaker; +import org.example.cloudnative.observability.ObservedCommandExecutor; +import org.example.enterprise.resilience.CircuitBreaker; +import org.junit.jupiter.api.Test; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ObservabilityPatternsTest { + + @Test + public void commandLatencyAndSagaCorrelationProduceSpans() { + InMemorySpanExporter spanExporter = InMemorySpanExporter.create(); + InMemoryMetricReader metricReader = InMemoryMetricReader.create(); + + SdkTracerProvider tracerProvider = SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) + .setResource(Resource.empty()) + .build(); + SdkMeterProvider meterProvider = SdkMeterProvider.builder() + .registerMetricReader(metricReader) + .setResource(Resource.empty()) + .build(); + + OpenTelemetry openTelemetry = OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .setMeterProvider(meterProvider) + .build(); + + Tracer tracer = openTelemetry.getTracer("patterns-test"); + Meter meter = openTelemetry.getMeter("patterns-test"); + + ObservedCommandExecutor commandExecutor = new ObservedCommandExecutor(tracer, meter); + String result = commandExecutor.execute("CreateOrder", () -> "ok"); + + AtomicReference seenCorrelation = new AtomicReference<>(); + CorrelatedSagaOrchestrator saga = new CorrelatedSagaOrchestrator(tracer); + saga.execute("corr-42", List.of( + new CorrelatedSagaStep("reserveInventory", seenCorrelation::set, value -> { + }) + )); + + List spans = spanExporter.getFinishedSpanItems(); + List spanNames = spans.stream().map(SpanData::getName).collect(Collectors.toList()); + assertTrue(spanNames.contains("command.execute")); + assertTrue(spanNames.contains("saga.orchestration")); + assertEquals("corr-42", seenCorrelation.get()); + assertEquals("ok", result); + + List metrics = metricReader.collectAllMetrics(); + assertTrue(metrics.stream().anyMatch(m -> m.getName().equals("pattern.command.latency.ms"))); + + tracerProvider.close(); + meterProvider.close(); + } + + @Test + public void circuitBreakerMetricsRecordOutcomes() { + InMemoryMetricReader metricReader = InMemoryMetricReader.create(); + SdkMeterProvider meterProvider = SdkMeterProvider.builder() + .registerMetricReader(metricReader) + .build(); + Meter meter = OpenTelemetrySdk.builder() + .setMeterProvider(meterProvider) + .build() + .getMeter("patterns-test"); + + CircuitBreaker breaker = new CircuitBreaker(1, Duration.ofMinutes(1), + Clock.fixed(Instant.parse("2026-01-01T00:00:00Z"), ZoneOffset.UTC)); + ObservedCircuitBreaker observed = new ObservedCircuitBreaker(breaker, meter); + + assertThrows(RuntimeException.class, () -> observed.execute(() -> { + throw new RuntimeException("down"); + })); + assertThrows(IllegalStateException.class, () -> observed.execute(() -> "never")); + + List metrics = metricReader.collectAllMetrics(); + MetricData counter = metrics.stream() + .filter(metric -> metric.getName().equals("pattern.circuit_breaker.executions")) + .findFirst() + .orElseThrow(); + + int points = counter.getLongSumData().getPoints().size(); + assertEquals(2, points); + + meterProvider.close(); + } +} diff --git a/src/test/java/example/ResilienceFaultInjectionTest.java b/src/test/java/example/ResilienceFaultInjectionTest.java new file mode 100644 index 0000000..804d030 --- /dev/null +++ b/src/test/java/example/ResilienceFaultInjectionTest.java @@ -0,0 +1,144 @@ +package example; + +import org.example.enterprise.resilience.ExponentialBackoffStrategy; +import org.example.enterprise.resilience.RetryExecutor; +import org.example.enterprise.resilience.TimeoutExecutor; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class ResilienceFaultInjectionTest { + + private final TimeoutExecutor timeoutExecutor = new TimeoutExecutor(); + + @AfterEach + public void cleanup() { + timeoutExecutor.shutdown(); + } + + @Test + public void wiremockFaultInjectionShowsTimeoutsAndRetries() throws Exception { + assumeTrue(DockerClientFactory.instance().isDockerAvailable(), "Docker is required for Testcontainers"); + + try (GenericContainer wiremock = new GenericContainer<>(DockerImageName.parse("wiremock/wiremock:3.13.1")) + .withExposedPorts(8080) + .waitingFor(Wait.forHttp("/__admin"))) { + wiremock.start(); + + String baseUrl = "http://" + wiremock.getHost() + ":" + wiremock.getMappedPort(8080); + configureTimeoutStub(baseUrl); + configureFlakyScenario(baseUrl); + + HttpClient client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(1)).build(); + + assertThrows(RuntimeException.class, () -> timeoutExecutor.execute(() -> httpGet(client, baseUrl + "/timeout"), + Duration.ofMillis(300))); + + RetryExecutor retryExecutor = new RetryExecutor(3, new ExponentialBackoffStrategy(0, 1)); + AtomicInteger attempts = new AtomicInteger(); + String body = retryExecutor.execute(() -> { + attempts.incrementAndGet(); + return httpGet(client, baseUrl + "/inventory"); + }); + + assertEquals("ok", body); + assertEquals(3, attempts.get()); + } + } + + private void configureTimeoutStub(String baseUrl) throws Exception { + String payload = """ + { + "request": {"method": "GET", "url": "/timeout"}, + "response": {"status": 200, "fixedDelayMilliseconds": 2000, "body": "late"} + } + """; + postMapping(baseUrl, payload); + } + + private void configureFlakyScenario(String baseUrl) throws Exception { + String firstFailure = """ + { + "scenarioName": "Inventory Retry", + "requiredScenarioState": "Started", + "newScenarioState": "second-failure", + "request": {"method": "GET", "url": "/inventory"}, + "response": {"status": 503, "body": "temporary-failure"} + } + """; + String secondFailure = """ + { + "scenarioName": "Inventory Retry", + "requiredScenarioState": "second-failure", + "newScenarioState": "healthy", + "request": {"method": "GET", "url": "/inventory"}, + "response": {"status": 500, "body": "partial-failure"} + } + """; + String success = """ + { + "scenarioName": "Inventory Retry", + "requiredScenarioState": "healthy", + "request": {"method": "GET", "url": "/inventory"}, + "response": {"status": 200, "body": "ok"} + } + """; + + postMapping(baseUrl, firstFailure); + postMapping(baseUrl, secondFailure); + postMapping(baseUrl, success); + } + + private void postMapping(String baseUrl, String payload) throws Exception { + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(baseUrl + "/__admin/mappings")) + .timeout(Duration.ofSeconds(5)) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(payload)) + .build(); + + HttpResponse response; + try { + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + } catch (IOException | InterruptedException e) { + throw new IllegalStateException("Failed to configure WireMock mapping", e); + } + + if (response.statusCode() >= 300) { + throw new IllegalStateException("WireMock mapping rejected: " + response.body()); + } + } + + private String httpGet(HttpClient client, String url) { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .timeout(Duration.ofSeconds(4)) + .GET() + .build(); + try { + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() >= 500) { + throw new IllegalStateException("Upstream failure " + response.statusCode()); + } + return response.body(); + } catch (IOException | InterruptedException e) { + throw new IllegalStateException("HTTP call failed", e); + } + } +}