From 89fd3a9cac1dbb242e09b10ad03fccbe3d019f44 Mon Sep 17 00:00:00 2001 From: Mateus Silva Date: Sun, 10 May 2026 23:49:43 -0300 Subject: [PATCH 01/23] feat: Add DoesNotContainAny and DoesNotLeakKnownValues to SensitiveDataAssert --- CHANGELOG.md | 5 ++ docs/testkit.md | 31 ++++++++ .../Assertions/SensitiveDataAssert.cs | 57 ++++++++++++++ .../SensitiveDataAssertTests.cs | 76 +++++++++++++++++++ 4 files changed, 169 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbcc1b6..82d58f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `SensitiveDataAssert.DoesNotContainAny` — checks a payload against explicit string values without requiring an annotated entity. +- `SensitiveDataAssert.DoesNotLeakKnownValues` — same as `DoesNotContainAny` but accepts `IEnumerable` for readability. + ## [1.0.0-preview.2] - 2026-05-10 ### Added diff --git a/docs/testkit.md b/docs/testkit.md index 97720d4..cf432c4 100644 --- a/docs/testkit.md +++ b/docs/testkit.md @@ -60,3 +60,34 @@ SensitiveDataAssert.DoesNotLeak(logSink.Output, request, loadedEntity, currentUs - Values that never lived on the entity in the first place (free-form notes, computed fields) For those, write specific assertions on the field values you expect to see. + +## SensitiveDataAssert.DoesNotContainAny + +Checks a payload against explicit string values — no entity or annotation needed. Use this when you know exactly which values should not leak. + +```csharp +using SensitiveFlow.TestKit.Assertions; + +[Fact] +public async Task GetCustomer_DoesNotLeakEmailOrPhone() +{ + var customer = new Customer { Email = "alice@example.com", Phone = "555-1234" }; + await SeedAsync(customer); + + var response = await _client.GetAsync($"/customers/{customer.Id}"); + var body = await response.Content.ReadAsStringAsync(); + + SensitiveDataAssert.DoesNotContainAny(body, customer.Email, customer.Phone); +} +``` + +Empty and null values are silently skipped — they would otherwise match every string. + +## SensitiveDataAssert.DoesNotLeakKnownValues + +Same as `DoesNotContainAny` but accepts an `IEnumerable` for readability when you already have a collection of known sensitive values. + +```csharp +var knownValues = new[] { customer.Email, customer.Phone, customer.TaxId }; +SensitiveDataAssert.DoesNotLeakKnownValues(body, knownValues); +``` diff --git a/src/SensitiveFlow.TestKit/Assertions/SensitiveDataAssert.cs b/src/SensitiveFlow.TestKit/Assertions/SensitiveDataAssert.cs index 7e893c1..1967241 100644 --- a/src/SensitiveFlow.TestKit/Assertions/SensitiveDataAssert.cs +++ b/src/SensitiveFlow.TestKit/Assertions/SensitiveDataAssert.cs @@ -68,4 +68,61 @@ public static void DoesNotLeak(string payload, params object[] entities) DoesNotLeak(payload, entity); } } + + /// + /// Fails the test if any of the specified appears verbatim + /// inside . Use this when you know exactly which values + /// should not leak — no entity or annotation needed. + /// + /// The string suspected of leaking — usually a serialized JSON, log line, or HTTP response body. + /// The exact string values that must NOT appear in . + /// + /// + /// SensitiveDataAssert.DoesNotContainAny( + /// jsonResponse, + /// customer.Email, + /// customer.Phone, + /// customer.TaxId); + /// + /// + public static void DoesNotContainAny(string payload, params string[] values) + { + ArgumentNullException.ThrowIfNull(payload); + ArgumentNullException.ThrowIfNull(values); + + var leaks = new List(); + foreach (var value in values) + { + if (string.IsNullOrEmpty(value)) + { + continue; + } + + if (payload.Contains(value, StringComparison.Ordinal)) + { + leaks.Add($"\"{value}\""); + } + } + + if (leaks.Count > 0) + { + throw new XunitException( + "Known sensitive values appeared in payload (redaction regression):" + + Environment.NewLine + + string.Join(Environment.NewLine, leaks.Select(l => " - " + l))); + } + } + + /// + /// Fails the test if any value from appears verbatim + /// inside . Alias for + /// with a more descriptive name for readability. + /// + /// The string suspected of leaking. + /// The collection of values that must NOT appear in . + public static void DoesNotLeakKnownValues(string payload, IEnumerable knownValues) + { + ArgumentNullException.ThrowIfNull(knownValues); + DoesNotContainAny(payload, knownValues.ToArray()); + } } diff --git a/tests/SensitiveFlow.TestKit.Tests/SensitiveDataAssertTests.cs b/tests/SensitiveFlow.TestKit.Tests/SensitiveDataAssertTests.cs index 96c7af0..3aa3693 100644 --- a/tests/SensitiveFlow.TestKit.Tests/SensitiveDataAssertTests.cs +++ b/tests/SensitiveFlow.TestKit.Tests/SensitiveDataAssertTests.cs @@ -98,6 +98,82 @@ public void DoesNotLeak_AcrossMultipleEntities_SkipsNullEntityEntries() act.Should().NotThrow(); } + [Fact] + public void DoesNotContainAny_WhenPayloadHasNoKnownValue_Passes() + { + var payload = "{\"Name\":\"A****\",\"Email\":\"a****@example.com\"}"; + + Action act = () => SensitiveDataAssert.DoesNotContainAny(payload, "alice@example.com", "555-1234"); + + act.Should().NotThrow(); + } + + [Fact] + public void DoesNotContainAny_WhenPayloadContainsKnownValue_Throws() + { + var payload = "{\"Name\":\"Alice\",\"Email\":\"a****@example.com\"}"; + + Action act = () => SensitiveDataAssert.DoesNotContainAny(payload, "alice@example.com", "Alice"); + + act.Should().Throw().WithMessage("*Alice*"); + } + + [Fact] + public void DoesNotContainAny_SkipsEmptyAndNullValues() + { + var payload = "clean payload"; + + Action act = () => SensitiveDataAssert.DoesNotContainAny(payload, "", null!, "clean"); + + act.Should().Throw().WithMessage("*clean*"); + } + + [Fact] + public void DoesNotContainAny_RejectsNullPayload() + { + var act = () => SensitiveDataAssert.DoesNotContainAny(null!, "value"); + + act.Should().Throw(); + } + + [Fact] + public void DoesNotContainAny_RejectsNullValuesArray() + { + var act = () => SensitiveDataAssert.DoesNotContainAny("payload", null!); + + act.Should().Throw(); + } + + [Fact] + public void DoesNotLeakKnownValues_WhenPayloadHasNoKnownValue_Passes() + { + var payload = "{\"Name\":\"A****\"}"; + var knownValues = new[] { "Alice", "alice@example.com", "555-1234" }; + + Action act = () => SensitiveDataAssert.DoesNotLeakKnownValues(payload, knownValues); + + act.Should().NotThrow(); + } + + [Fact] + public void DoesNotLeakKnownValues_WhenPayloadContainsKnownValue_Throws() + { + var payload = "leaked: Alice was here"; + var knownValues = new[] { "Alice", "Bob" }; + + Action act = () => SensitiveDataAssert.DoesNotLeakKnownValues(payload, knownValues); + + act.Should().Throw().WithMessage("*Alice*"); + } + + [Fact] + public void DoesNotLeakKnownValues_RejectsNullKnownValues() + { + var act = () => SensitiveDataAssert.DoesNotLeakKnownValues("payload", null!); + + act.Should().Throw(); + } + public class Customer { [PersonalData(Category = DataCategory.Identification)] From ed32182c0e5704c01813b379f906241a91abea87 Mon Sep 17 00:00:00 2001 From: Mateus Silva Date: Mon, 11 May 2026 05:44:05 -0300 Subject: [PATCH 02/23] feat: Add health checks for audit and token stores - Introduced `SensitiveFlowValidationOptions` to manage validation expectations. - Implemented `AuditStoreHealthCheck` and `TokenStoreHealthCheck` for health monitoring. - Added extension methods for registering health checks in `SensitiveFlowHealthCheckBuilderExtensions`. - Created a new project for health checks with necessary dependencies. - Updated retention services to utilize a customizable `TimeProvider`. - Added contract tests for anonymizers, maskers, pseudonymizers, and retention expiration handlers. - Implemented a command-line tool for generating sensitive data reports. - Enhanced test assertions for sensitive data handling. - Added validation tests for sensitive flow configurations. --- CHANGELOG.md | 13 +- Directory.Packages.props | 3 + README.md | 29 ++++ docs/README.md | 2 + docs/package-reference.md | 9 +- docs/policies-discovery-health.md | 87 ++++++++++++ docs/testkit.md | 19 +++ samples/Console.Sample/Program.cs | 32 +++++ .../MinimalApi.Sample.csproj | 1 + samples/MinimalApi.Sample/Program.cs | 69 ++++++++++ .../Controllers/CustomersController.cs | 22 +++ .../Controllers/SensitiveFlowController.cs | 36 +++++ samples/WebApi.Sample/Program.cs | 26 ++++ samples/WebApi.Sample/WebApi.Sample.csproj | 1 + sensitiveflow.slnx | 3 + .../Analyzers/SensitiveDataLoggingAnalyzer.cs | 5 + .../Extensions/SymbolExtensions.cs | 38 ++++++ .../AllowSensitiveLoggingAttribute.cs | 31 +++++ .../AllowSensitiveReturnAttribute.cs | 18 +++ .../Attributes/MaskAttribute.cs | 23 ++++ .../Attributes/OmitAttribute.cs | 14 ++ .../Attributes/PersonalDataAttribute.cs | 3 + .../Attributes/RedactAttribute.cs | 14 ++ .../Attributes/RedactionAttribute.cs | 36 +++++ .../Attributes/SensitiveDataAttribute.cs | 3 + .../SensitiveFlowIgnoreAttribute.cs | 16 +++ .../Correlation/AuditCorrelationSnapshot.cs | 16 +++ .../Correlation/IAuditCorrelationContext.cs | 17 +++ .../Correlation/SensitiveFlowCorrelation.cs | 19 +++ .../Discovery/SensitiveDataDiscovery.cs | 54 ++++++++ .../Discovery/SensitiveDataDiscoveryEntry.cs | 37 +++++ .../Discovery/SensitiveDataDiscoveryReport.cs | 57 ++++++++ .../Enums/DataSensitivity.cs | 20 +++ src/SensitiveFlow.Core/Enums/MaskKind.cs | 20 +++ .../Enums/OutputRedactionAction.cs | 23 ++++ .../Enums/RedactionContext.cs | 20 +++ .../SensitiveFlowConfigurationException.cs | 14 ++ .../Exceptions/SensitiveFlowException.cs | 19 +++ .../SensitiveFlowExceptionSanitizer.cs | 42 ++++++ .../SensitiveFlowSanitizedException.cs | 17 +++ .../Export/CsvDataExportFormatter.cs | 44 ++++++ .../Export/IDataExportFormatter.cs | 11 ++ .../Export/JsonDataExportFormatter.cs | 17 +++ .../Interfaces/IAuditOutbox.cs | 13 ++ .../Interfaces/IAuditOutboxSerializer.cs | 16 +++ .../Interfaces/IDataSubjectEraser.cs | 12 ++ .../Interfaces/IDataSubjectExporter.cs | 12 ++ .../Interfaces/IDataSubjectRequestHandler.cs | 11 ++ .../Interfaces/IHealthProbe.cs | 11 ++ .../Policies/SensitiveFlowPolicyAction.cs | 27 ++++ .../SensitiveFlowPolicyConfiguration.cs | 17 +++ .../Policies/SensitiveFlowPolicyRegistry.cs | 63 +++++++++ .../Policies/SensitiveFlowPolicyRule.cs | 22 +++ .../SensitiveFlowPolicyRuleBuilder.cs | 55 ++++++++ .../Profiles/SensitiveFlowOptions.cs | 53 +++++++ .../Profiles/SensitiveFlowProfile.cs | 19 +++ ...owValidationServiceCollectionExtensions.cs | 39 ++++++ .../SensitiveFlow.Diagnostics.csproj | 1 + .../SensitiveFlowConfigurationDiagnostic.cs | 17 +++ .../SensitiveFlowConfigurationReport.cs | 21 +++ .../SensitiveFlowConfigurationValidator.cs | 66 +++++++++ .../SensitiveFlowDiagnosticSeverity.cs | 17 +++ .../SensitiveFlowValidationOptions.cs | 20 +++ .../Checks/AuditStoreHealthCheck.cs | 43 ++++++ .../Checks/TokenStoreHealthCheck.cs | 39 ++++++ ...nsitiveFlowHealthCheckBuilderExtensions.cs | 40 ++++++ .../SensitiveFlow.HealthChecks.csproj | 13 ++ .../Services/RetentionEvaluator.cs | 10 +- .../Services/RetentionExecutor.cs | 49 ++++++- .../AnonymizerContractTests.cs | 28 ++++ .../Assertions/SensitiveDataAssert.cs | 61 +++++++++ .../AuditSnapshotStoreContractTests.cs | 68 +++++++++ .../MaskerContractTests.cs | 28 ++++ .../PseudonymizerContractTests.cs | 36 +++++ ...RetentionExpirationHandlerContractTests.cs | 25 ++++ .../SensitiveFlow.TestKit.csproj | 1 + src/SensitiveFlow.Tool/Program.cs | 3 + .../SensitiveFlow.Tool.csproj | 16 +++ .../SensitiveFlowToolRunner.cs | 49 +++++++ .../CoreContractTests.cs | 62 +++++++++ .../SensitiveFlow.Core.Tests.csproj | 3 +- .../SensitiveFlowToolRunnerTests.cs | 37 +++++ .../SensitiveFlowValidationTests.cs | 52 +++++++ .../SensitiveFlow.HealthChecks.Tests.csproj | 13 ++ .../SensitiveFlowHealthChecksTests.cs | 79 +++++++++++ .../RetentionExecutorTests.cs | 13 ++ .../ExpandedContractTests.cs | 129 ++++++++++++++++++ .../SensitiveDataAssertTests.cs | 40 ++++++ 88 files changed, 2436 insertions(+), 13 deletions(-) create mode 100644 docs/policies-discovery-health.md create mode 100644 samples/WebApi.Sample/Controllers/SensitiveFlowController.cs create mode 100644 src/SensitiveFlow.Core/Attributes/AllowSensitiveLoggingAttribute.cs create mode 100644 src/SensitiveFlow.Core/Attributes/AllowSensitiveReturnAttribute.cs create mode 100644 src/SensitiveFlow.Core/Attributes/MaskAttribute.cs create mode 100644 src/SensitiveFlow.Core/Attributes/OmitAttribute.cs create mode 100644 src/SensitiveFlow.Core/Attributes/RedactAttribute.cs create mode 100644 src/SensitiveFlow.Core/Attributes/RedactionAttribute.cs create mode 100644 src/SensitiveFlow.Core/Attributes/SensitiveFlowIgnoreAttribute.cs create mode 100644 src/SensitiveFlow.Core/Correlation/AuditCorrelationSnapshot.cs create mode 100644 src/SensitiveFlow.Core/Correlation/IAuditCorrelationContext.cs create mode 100644 src/SensitiveFlow.Core/Correlation/SensitiveFlowCorrelation.cs create mode 100644 src/SensitiveFlow.Core/Discovery/SensitiveDataDiscovery.cs create mode 100644 src/SensitiveFlow.Core/Discovery/SensitiveDataDiscoveryEntry.cs create mode 100644 src/SensitiveFlow.Core/Discovery/SensitiveDataDiscoveryReport.cs create mode 100644 src/SensitiveFlow.Core/Enums/DataSensitivity.cs create mode 100644 src/SensitiveFlow.Core/Enums/MaskKind.cs create mode 100644 src/SensitiveFlow.Core/Enums/OutputRedactionAction.cs create mode 100644 src/SensitiveFlow.Core/Enums/RedactionContext.cs create mode 100644 src/SensitiveFlow.Core/Exceptions/SensitiveFlowConfigurationException.cs create mode 100644 src/SensitiveFlow.Core/Exceptions/SensitiveFlowException.cs create mode 100644 src/SensitiveFlow.Core/Exceptions/SensitiveFlowExceptionSanitizer.cs create mode 100644 src/SensitiveFlow.Core/Exceptions/SensitiveFlowSanitizedException.cs create mode 100644 src/SensitiveFlow.Core/Export/CsvDataExportFormatter.cs create mode 100644 src/SensitiveFlow.Core/Export/IDataExportFormatter.cs create mode 100644 src/SensitiveFlow.Core/Export/JsonDataExportFormatter.cs create mode 100644 src/SensitiveFlow.Core/Interfaces/IAuditOutbox.cs create mode 100644 src/SensitiveFlow.Core/Interfaces/IAuditOutboxSerializer.cs create mode 100644 src/SensitiveFlow.Core/Interfaces/IDataSubjectEraser.cs create mode 100644 src/SensitiveFlow.Core/Interfaces/IDataSubjectExporter.cs create mode 100644 src/SensitiveFlow.Core/Interfaces/IDataSubjectRequestHandler.cs create mode 100644 src/SensitiveFlow.Core/Interfaces/IHealthProbe.cs create mode 100644 src/SensitiveFlow.Core/Policies/SensitiveFlowPolicyAction.cs create mode 100644 src/SensitiveFlow.Core/Policies/SensitiveFlowPolicyConfiguration.cs create mode 100644 src/SensitiveFlow.Core/Policies/SensitiveFlowPolicyRegistry.cs create mode 100644 src/SensitiveFlow.Core/Policies/SensitiveFlowPolicyRule.cs create mode 100644 src/SensitiveFlow.Core/Policies/SensitiveFlowPolicyRuleBuilder.cs create mode 100644 src/SensitiveFlow.Core/Profiles/SensitiveFlowOptions.cs create mode 100644 src/SensitiveFlow.Core/Profiles/SensitiveFlowProfile.cs create mode 100644 src/SensitiveFlow.Diagnostics/Extensions/SensitiveFlowValidationServiceCollectionExtensions.cs create mode 100644 src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationDiagnostic.cs create mode 100644 src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationReport.cs create mode 100644 src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs create mode 100644 src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowDiagnosticSeverity.cs create mode 100644 src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowValidationOptions.cs create mode 100644 src/SensitiveFlow.HealthChecks/Checks/AuditStoreHealthCheck.cs create mode 100644 src/SensitiveFlow.HealthChecks/Checks/TokenStoreHealthCheck.cs create mode 100644 src/SensitiveFlow.HealthChecks/Extensions/SensitiveFlowHealthCheckBuilderExtensions.cs create mode 100644 src/SensitiveFlow.HealthChecks/SensitiveFlow.HealthChecks.csproj create mode 100644 src/SensitiveFlow.TestKit/AnonymizerContractTests.cs create mode 100644 src/SensitiveFlow.TestKit/AuditSnapshotStoreContractTests.cs create mode 100644 src/SensitiveFlow.TestKit/MaskerContractTests.cs create mode 100644 src/SensitiveFlow.TestKit/PseudonymizerContractTests.cs create mode 100644 src/SensitiveFlow.TestKit/RetentionExpirationHandlerContractTests.cs create mode 100644 src/SensitiveFlow.Tool/Program.cs create mode 100644 src/SensitiveFlow.Tool/SensitiveFlow.Tool.csproj create mode 100644 src/SensitiveFlow.Tool/SensitiveFlowToolRunner.cs create mode 100644 tests/SensitiveFlow.Core.Tests/SensitiveFlowToolRunnerTests.cs create mode 100644 tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs create mode 100644 tests/SensitiveFlow.HealthChecks.Tests/SensitiveFlow.HealthChecks.Tests.csproj create mode 100644 tests/SensitiveFlow.HealthChecks.Tests/SensitiveFlowHealthChecksTests.cs create mode 100644 tests/SensitiveFlow.TestKit.Tests/ExpandedContractTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 82d58f6..6710852 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `SensitiveDataAssert.DoesNotContainAny` — checks a payload against explicit string values without requiring an annotated entity. -- `SensitiveDataAssert.DoesNotLeakKnownValues` — same as `DoesNotContainAny` but accepts `IEnumerable` for readability. +- `SensitiveFlow.Core` policy engine primitives: `SensitiveFlowOptions`, built-in profiles, category policy registry, output behavior attributes, contextual redaction attributes, data sensitivity levels, discovery reports, export formatters, audit correlation helpers, audit outbox interfaces, data-subject request interfaces, and privacy-safe exception types. +- `SensitiveFlow.Tool` command-line project with `sensitiveflow scan [output-directory]` for JSON/Markdown discovery reports. +- `SensitiveFlow.HealthChecks` package with audit/token store health checks. +- `SensitiveFlow.Diagnostics` startup validation via `AddSensitiveFlowValidation(...)` and `ValidateSensitiveFlow()`. +- `SensitiveFlow.Retention` dry-run execution via `RetentionExecutor.DryRunAsync(...)`. +- `SensitiveFlow.TestKit` expanded assertions: `ContainsMaskedEmail`, `DoesNotContainRawValues`, `JsonDoesNotExposeAnnotatedProperties`, and `LogsDoNotContainSensitiveValues`. +- `SensitiveFlow.TestKit` contract test bases for `IAuditSnapshotStore`, `IPseudonymizer`, `IMasker`, `IAnonymizer`, and `IRetentionExpirationHandler`. +- `SensitiveDataAssert.DoesNotContainAny` - checks a payload against explicit string values without requiring an annotated entity. +- `SensitiveDataAssert.DoesNotLeakKnownValues` - same as `DoesNotContainAny` but accepts `IEnumerable` for readability. ## [1.0.0-preview.2] - 2026-05-10 @@ -82,3 +89,5 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Documentation: getting-started, attributes, legal-bases, consent, audit, data-subject-rights, retention, data-map, incidents, ripd, international-transfer, efcore, aspnetcore, migration + + diff --git a/Directory.Packages.props b/Directory.Packages.props index 46eeb6e..1d54234 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -6,9 +6,12 @@ + + + diff --git a/README.md b/README.md index 90f19ad..475a5bc 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ Sensitive data flows through your application on every request: EF Core saves, H | `SensitiveFlow.AspNetCore` | Middleware for actor/IP context enrichment | Preview | | `SensitiveFlow.Logging` | ILogger decorator for PII redaction in logs | Preview | | `SensitiveFlow.Diagnostics` | OpenTelemetry bridge (ActivitySource + Meter) for audit/logging spans & metrics | Preview | +| `SensitiveFlow.HealthChecks` | Microsoft health checks for audit/token infrastructure | Preview | | `SensitiveFlow.Anonymization` | Masking, anonymization, pseudonymization, erasure, data export, and deterministic fingerprints | Preview | | `SensitiveFlow.Json` | `System.Text.Json` modifier that masks/redacts/omits annotated properties at serialization time | Preview | | `SensitiveFlow.Retention` | Retention metadata and expiration hook contracts | Preview | @@ -40,6 +41,7 @@ Sensitive data flows through your application on every request: EF Core saves, H | `SensitiveFlow.Analyzers.CodeFixes` | Quick-fix providers for SF0001/SF0002 (wrap with `.MaskEmail()` / `.MaskPhone()` / `.MaskName()`) | Preview | | `SensitiveFlow.SourceGenerators` | Source generator that precomputes sensitive/retention member metadata | Preview | | `SensitiveFlow.TestKit` | xUnit conformance bases for `IAuditStore` / `ITokenStore` plus a `SensitiveDataAssert` leak-detection helper | Preview | +| `SensitiveFlow.Tool` | `dotnet tool` command for discovery reports from annotated assemblies | Preview | ## Quick Start @@ -104,6 +106,11 @@ builder.Services.AddAuditStoreRetry(); builder.Services.AddSensitiveFlowEFCore(); // SensitiveFlow.EFCore builder.Services.AddSensitiveFlowAspNetCore(); // SensitiveFlow.AspNetCore builder.Services.AddSensitiveFlowLogging(); // SensitiveFlow.Logging +builder.Services.AddSensitiveFlowValidation(o => +{ + o.RequireAuditStore = true; + o.RequireTokenStore = true; +}); ``` ### 5. Add the middleware @@ -119,6 +126,28 @@ app.UseSensitiveFlowAudit(); optionsBuilder.AddInterceptors(app.Services.GetRequiredService()); ``` +### Optional: policies, reports, and health checks + +```csharp +var options = SensitiveFlowPolicyConfiguration.Create(options => +{ + options.UseProfile(SensitiveFlowProfile.Strict); + options.Policies.ForCategory(DataCategory.Contact) + .MaskInLogs() + .RedactInJson() + .AuditOnChange(); +}); + +var report = SensitiveDataDiscovery.Scan(typeof(Customer).Assembly); +File.WriteAllText("sensitiveflow-report.md", report.ToMarkdown()); + +builder.Services.AddSensitiveFlowHealthChecks() + .AddAuditStoreCheck() + .AddTokenStoreCheck(); + +var startupReport = app.Services.ValidateSensitiveFlow(); +``` + Every `SaveChanges` on a field annotated with `[PersonalData]` or `[SensitiveData]` now produces an `AuditRecord` automatically. diff --git a/docs/README.md b/docs/README.md index 45521c8..e0b5e8c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,6 +16,7 @@ Use this page as the documentation index. The files are grouped by task so contr - [JSON redaction](json.md): `System.Text.Json` output protection. - [Logging](logging.md): `ILogger` redaction provider and limits. - [Diagnostics](diagnostics.md): OpenTelemetry ActivitySource/Meter integration. +- [Policies, discovery, and health](policies-discovery-health.md): shared profiles, policy rules, discovery reports, startup validation, health checks, and CLI scans. ## Data protection workflows @@ -28,6 +29,7 @@ Use this page as the documentation index. The files are grouped by task so contr - [Analyzers](analyzers.md): Roslyn diagnostics for privacy anti-patterns. - [TestKit](testkit.md): contract tests for `IAuditStore` and `ITokenStore`, plus leak assertions. - [Source generators](package-reference.md#sensitiveflowsourcegenerators): generated metadata overview. +- [SensitiveFlow Tool](policies-discovery-health.md#cli-tool): `sensitiveflow scan` report generation. ## AI usage diff --git a/docs/package-reference.md b/docs/package-reference.md index 8b0e1ed..53ec387 100644 --- a/docs/package-reference.md +++ b/docs/package-reference.md @@ -17,10 +17,12 @@ This document summarizes each SensitiveFlow package individually: purpose, prima | `SensitiveFlow.Json` | You need automatic response serialization redaction. | `WithSensitiveDataRedaction()` on `System.Text.Json`. | Does not cover Newtonsoft.Json. | | `SensitiveFlow.Logging` | You need sensitive log value redaction. | `AddSensitiveFlowLogging()` and/or provider wrapper. | Not semantic PII detection; values must flow through known redaction paths. | | `SensitiveFlow.Diagnostics` | You want OpenTelemetry spans/metrics. | `AddSensitiveFlowDiagnostics()` after audit store/decorators. | Decorator order changes what latency is measured. | +| `SensitiveFlow.HealthChecks` | You want ASP.NET Core health checks for SensitiveFlow infrastructure. | `AddSensitiveFlowHealthChecks().AddAuditStoreCheck().AddTokenStoreCheck()`. | Token stores without `IHealthProbe` are resolution-only checks to avoid mutating data. | | `SensitiveFlow.Retention` | You evaluate retention policies. | `AddRetention()` / `AddRetentionExecutor()` and run a scheduled job. | It will not delete database rows automatically. | | `SensitiveFlow.Analyzers` | You want compile-time guardrails. | Add analyzer package to application projects. | Warnings still require engineering judgment. | | `SensitiveFlow.SourceGenerators` | You want generated sensitive metadata. | Add source generator package. | Keep generator tests aligned with reflection fallback. | | `SensitiveFlow.TestKit` | You implement custom stores or leak tests. | Inherit contract tests. | Contract tests need isolated fresh stores. | +| `SensitiveFlow.Tool` | You want CI/documentation reports from annotated assemblies. | `dotnet tool install SensitiveFlow.Tool`; run `sensitiveflow scan `. | Scans compiled assemblies, not source directories. | ## SensitiveFlow.Core @@ -31,9 +33,12 @@ Purpose: Primary APIs: - Attributes: `[PersonalData]`, `[SensitiveData]`, `[RetentionData]`. +- Output attributes: `[Redact]`, `[Mask]`, `[Omit]`, `[Redaction]`. - Models: `AuditRecord`, `AuditSnapshot`. -- Contracts: `IAuditStore`, `IBatchAuditStore`, `IAuditSnapshotStore`, `ITokenStore`, `IPseudonymizer`, `IMasker`, `IAnonymizer`, `IAuditContext`. -- Enums: `DataCategory`, `SensitiveDataCategory`, `RetentionPolicy`, `AuditOperation`, `AnonymizationType`. +- Contracts: `IAuditStore`, `IBatchAuditStore`, `IAuditSnapshotStore`, `ITokenStore`, `IPseudonymizer`, `IMasker`, `IAnonymizer`, `IAuditContext`, data-subject request interfaces, audit outbox interfaces. +- Enums: `DataCategory`, `SensitiveDataCategory`, `DataSensitivity`, `RetentionPolicy`, `AuditOperation`, `AnonymizationType`. +- Policies/profiles: `SensitiveFlowOptions`, `SensitiveFlowProfile`, `SensitiveFlowPolicyRegistry`. +- Discovery/export: `SensitiveDataDiscovery`, `JsonDataExportFormatter`, `CsvDataExportFormatter`. - Cache: `SensitiveMemberCache`. Install when: diff --git a/docs/policies-discovery-health.md b/docs/policies-discovery-health.md new file mode 100644 index 0000000..9bcbc49 --- /dev/null +++ b/docs/policies-discovery-health.md @@ -0,0 +1,87 @@ +# Policies, Discovery, Health, and Startup Validation + +## Policy engine and profiles + +Use `SensitiveFlowOptions` when an application wants one central place to declare how categories should be handled. + +```csharp +var options = SensitiveFlowPolicyConfiguration.Create(options => +{ + options.UseProfile(SensitiveFlowProfile.Balanced); + + options.Policies.ForCategory(DataCategory.Contact) + .MaskInLogs() + .RedactInJson() + .AuditOnChange(); + + options.Policies.ForSensitiveCategory(SensitiveDataCategory.Health) + .OmitInJson() + .RequireAudit(); +}); +``` + +Built-in profiles: + +- `Development`: useful local defaults with masking and lighter audit requirements. +- `Balanced`: API/log redaction plus audit for common categories. +- `Strict`: omission/redaction and required audit for sensitive categories. +- `AuditOnly`: audit-focused hints without forcing output behavior. + +## Discovery report + +`SensitiveDataDiscovery` scans compiled assemblies for `[PersonalData]`, `[SensitiveData]`, and `[RetentionData]`. + +```csharp +var report = SensitiveDataDiscovery.Scan(typeof(Customer).Assembly); + +File.WriteAllText("sensitiveflow-report.json", report.ToJson()); +File.WriteAllText("sensitiveflow-report.md", report.ToMarkdown()); +``` + +The report includes type, member, annotation kind, category, sensitivity, and retention settings. + +## CLI tool + +`SensitiveFlow.Tool` exposes the same report generator for CI and documentation jobs. + +```bash +dotnet tool install SensitiveFlow.Tool +sensitiveflow scan ./bin/Release/net10.0/MyApp.dll ./artifacts/privacy +``` + +The tool writes: + +- `sensitiveflow-report.json` +- `sensitiveflow-report.md` + +## Health checks + +`SensitiveFlow.HealthChecks` integrates with `Microsoft.Extensions.Diagnostics.HealthChecks`. + +```csharp +builder.Services.AddSensitiveFlowHealthChecks() + .AddAuditStoreCheck() + .AddTokenStoreCheck(); +``` + +`IAuditStore` is checked with a read-only `QueryAsync(take: 1)` probe. `ITokenStore` is resolution-only by default because the contract has no read-only ping; stores can implement `IHealthProbe` for a real non-mutating probe. + +## Startup validation + +`SensitiveFlow.Diagnostics` includes a startup self-test for common misconfiguration. + +```csharp +builder.Services.AddSensitiveFlowValidation(options => +{ + options.RequireAuditStore = true; + options.RequireTokenStore = true; +}); + +var result = app.Services.ValidateSensitiveFlow(); +``` + +Example warnings: + +- `SF-CONFIG-001`: no `IAuditStore` registration found. +- `SF-CONFIG-002`: no durable `ITokenStore` registration found. +- `SF-CONFIG-003`: `IPseudonymizer` registered without `ITokenStore`. diff --git a/docs/testkit.md b/docs/testkit.md index cf432c4..6b26a24 100644 --- a/docs/testkit.md +++ b/docs/testkit.md @@ -91,3 +91,22 @@ Same as `DoesNotContainAny` but accepts an `IEnumerable` for readability var knownValues = new[] { customer.Email, customer.Phone, customer.TaxId }; SensitiveDataAssert.DoesNotLeakKnownValues(body, knownValues); ``` + +## Expanded contract suites + +Beyond `AuditStoreContractTests` and `TokenStoreContractTests`, TestKit now includes: + +- `AuditSnapshotStoreContractTests` +- `PseudonymizerContractTests` +- `MaskerContractTests` +- `AnonymizerContractTests` +- `RetentionExpirationHandlerContractTests` + +## Additional assertions + +```csharp +SensitiveDataAssert.ContainsMaskedEmail(body); +SensitiveDataAssert.DoesNotContainRawValues(body, customer); +SensitiveDataAssert.JsonDoesNotExposeAnnotatedProperties(body, typeof(Customer)); +SensitiveDataAssert.LogsDoNotContainSensitiveValues(logSink.Output, knownValues); +``` diff --git a/samples/Console.Sample/Program.cs b/samples/Console.Sample/Program.cs index 2f760dd..a772924 100644 --- a/samples/Console.Sample/Program.cs +++ b/samples/Console.Sample/Program.cs @@ -20,9 +20,12 @@ using SensitiveFlow.Anonymization.Masking; using SensitiveFlow.Anonymization.Pseudonymizers; using SensitiveFlow.Core.Attributes; +using SensitiveFlow.Core.Discovery; using SensitiveFlow.Core.Enums; using SensitiveFlow.Core.Interfaces; using SensitiveFlow.Core.Models; +using SensitiveFlow.Core.Policies; +using SensitiveFlow.Core.Profiles; using SensitiveFlow.EFCore.Interceptors; using SensitiveFlow.Retention.Services; @@ -174,6 +177,35 @@ await auditStore.AppendAsync(new AuditRecord Console.WriteLine($" Total audit records in store: {all.Count}"); } +Section("8. Policy profile, discovery report, and retention dry-run"); + +var policyOptions = SensitiveFlowPolicyConfiguration.Create(options => +{ + options.UseProfile(SensitiveFlowProfile.Balanced); + options.Policies.ForCategory(DataCategory.Contact) + .MaskInLogs() + .RedactInJson() + .AuditOnChange(); +}); +Console.WriteLine($" Active policy profile: {policyOptions.Profile}"); + +var discovery = SensitiveDataDiscovery.Scan(typeof(Customer).Assembly); +Console.WriteLine($" Discovery entries found: {discovery.Entries.Count}"); +Console.WriteLine(discovery.ToMarkdown()); + +var expiredCustomer = new Customer +{ + DataSubjectId = customer.DataSubjectId, + Name = customer.Name, + Email = customer.Email, + TaxId = customer.TaxId, + Phone = customer.Phone, + CreatedAt = DateTimeOffset.UtcNow.AddYears(-6), +}; +var retentionDryRun = await new RetentionExecutor() + .DryRunAsync([expiredCustomer], entity => ((Customer)entity).CreatedAt); +Console.WriteLine($" Retention dry-run actions: {retentionDryRun.Entries.Count}"); + Log.Information("Console sample finished."); await Log.CloseAndFlushAsync(); diff --git a/samples/MinimalApi.Sample/MinimalApi.Sample.csproj b/samples/MinimalApi.Sample/MinimalApi.Sample.csproj index 28555c5..1d3595d 100644 --- a/samples/MinimalApi.Sample/MinimalApi.Sample.csproj +++ b/samples/MinimalApi.Sample/MinimalApi.Sample.csproj @@ -28,6 +28,7 @@ + diff --git a/samples/MinimalApi.Sample/Program.cs b/samples/MinimalApi.Sample/Program.cs index 476a2f0..c315114 100644 --- a/samples/MinimalApi.Sample/Program.cs +++ b/samples/MinimalApi.Sample/Program.cs @@ -13,12 +13,16 @@ using SensitiveFlow.Audit.Extensions; using SensitiveFlow.AspNetCore.Extensions; using SensitiveFlow.Core.Diagnostics; +using SensitiveFlow.Core.Discovery; using SensitiveFlow.Core.Enums; using SensitiveFlow.Core.Interfaces; using SensitiveFlow.Core.Models; +using SensitiveFlow.Core.Policies; +using SensitiveFlow.Core.Profiles; using SensitiveFlow.Diagnostics.Extensions; using SensitiveFlow.EFCore.Extensions; using SensitiveFlow.EFCore.Interceptors; +using SensitiveFlow.HealthChecks.Extensions; using SensitiveFlow.Json.Configuration; using SensitiveFlow.Json.Enums; using SensitiveFlow.Json.Extensions; @@ -71,9 +75,30 @@ builder.Services.AddSensitiveFlowLogging(); builder.Services.AddSensitiveFlowEFCore(); builder.Services.AddSensitiveFlowAspNetCore(); + builder.Services.AddSensitiveFlowValidation(options => + { + options.RequireAuditStore = true; + options.RequireTokenStore = true; + options.RequireJsonRedaction = true; + options.RequireRetention = true; + }); builder.Services.AddSensitiveFlowJsonRedaction(options => options.DefaultMode = JsonRedactionMode.Mask); builder.Services.AddRetention(); builder.Services.AddRetentionExecutor(); + builder.Services.AddSingleton(SensitiveFlowPolicyConfiguration.Create(options => + { + options.UseProfile(SensitiveFlowProfile.Balanced); + options.Policies.ForCategory(DataCategory.Contact) + .MaskInLogs() + .RedactInJson() + .AuditOnChange(); + options.Policies.ForSensitiveCategory(SensitiveDataCategory.Other) + .OmitInJson() + .RequireAudit(); + })); + builder.Services.AddSensitiveFlowHealthChecks() + .AddAuditStoreCheck() + .AddTokenStoreCheck(); builder.Services.ConfigureHttpJsonOptions(options => options.SerializerOptions.WithSensitiveDataRedaction( new JsonRedactionOptions { DefaultMode = JsonRedactionMode.Mask })); @@ -117,6 +142,7 @@ await scope.ServiceProvider.GetRequiredService() app.UseHttpsRedirection(); app.UseSensitiveFlowAudit(); + app.MapHealthChecks("/health/sensitiveflow"); app.MapPost("/customers", async ( CreateCustomerRequest request, @@ -275,6 +301,49 @@ await auditStore.AppendAsync(new AuditRecord }) .WithName("RunRetention"); + app.MapPost("/retention/dry-run", async ( + SampleDbContext db, + RetentionExecutor retention, + CancellationToken ct) => + { + var customers = await db.Customers.AsNoTracking().ToListAsync(ct); + var report = await retention.DryRunAsync( + customers, + entity => ((Customer)entity).CreatedAt, + ct); + + return Results.Ok(new + { + report.AnonymizedFieldCount, + report.DeletePendingEntityCount, + Entries = report.Entries.Select(e => new + { + e.FieldName, + e.ExpiredAt, + Action = e.Action.ToString(), + }), + }); + }) + .WithName("DryRunRetention"); + + app.MapGet("/sensitiveflow/discovery", () => + { + var report = SensitiveDataDiscovery.Scan(typeof(Customer).Assembly); + return Results.Text(report.ToMarkdown(), "text/markdown"); + }) + .WithName("SensitiveFlowDiscovery"); + + app.MapGet("/sensitiveflow/diagnostics", (IServiceProvider services) => + { + var report = services.ValidateSensitiveFlow(); + return Results.Ok(new + { + report.IsValid, + report.Diagnostics, + }); + }) + .WithName("SensitiveFlowDiagnostics"); + app.Run(); } catch (Exception ex) diff --git a/samples/WebApi.Sample/Controllers/CustomersController.cs b/samples/WebApi.Sample/Controllers/CustomersController.cs index d1d88e3..9996823 100644 --- a/samples/WebApi.Sample/Controllers/CustomersController.cs +++ b/samples/WebApi.Sample/Controllers/CustomersController.cs @@ -184,6 +184,28 @@ public async Task RunRetention(CancellationToken ct) }); } + [HttpPost("/retention/dry-run")] + public async Task DryRunRetention(CancellationToken ct) + { + var customers = await _db.Customers.AsNoTracking().ToListAsync(ct); + var report = await _retention.DryRunAsync( + customers, + entity => ((Customer)entity).CreatedAt, + ct); + + return Ok(new + { + report.AnonymizedFieldCount, + report.DeletePendingEntityCount, + Entries = report.Entries.Select(e => new + { + e.FieldName, + e.ExpiredAt, + Action = e.Action.ToString(), + }), + }); + } + private static CustomerResponse ToResponse(Customer c) => new( c.DataSubjectId, c.Name.MaskName(), diff --git a/samples/WebApi.Sample/Controllers/SensitiveFlowController.cs b/samples/WebApi.Sample/Controllers/SensitiveFlowController.cs new file mode 100644 index 0000000..5c699dc --- /dev/null +++ b/samples/WebApi.Sample/Controllers/SensitiveFlowController.cs @@ -0,0 +1,36 @@ +using Microsoft.AspNetCore.Mvc; +using SensitiveFlow.Core.Discovery; +using SensitiveFlow.Diagnostics.Extensions; +using WebApi.Sample.Infrastructure; + +namespace WebApi.Sample.Controllers; + +[ApiController] +[Route("sensitiveflow")] +public sealed class SensitiveFlowController : ControllerBase +{ + private readonly IServiceProvider _services; + + public SensitiveFlowController(IServiceProvider services) + { + _services = services; + } + + [HttpGet("discovery")] + public ContentResult Discovery() + { + var report = SensitiveDataDiscovery.Scan(typeof(Customer).Assembly); + return Content(report.ToMarkdown(), "text/markdown"); + } + + [HttpGet("diagnostics")] + public IActionResult Diagnostics() + { + var report = _services.ValidateSensitiveFlow(); + return Ok(new + { + report.IsValid, + report.Diagnostics, + }); + } +} diff --git a/samples/WebApi.Sample/Program.cs b/samples/WebApi.Sample/Program.cs index b6dda89..5fc4096 100644 --- a/samples/WebApi.Sample/Program.cs +++ b/samples/WebApi.Sample/Program.cs @@ -10,10 +10,14 @@ using SensitiveFlow.Audit.Extensions; using SensitiveFlow.AspNetCore.Extensions; using SensitiveFlow.Core.Diagnostics; +using SensitiveFlow.Core.Enums; using SensitiveFlow.Core.Interfaces; +using SensitiveFlow.Core.Policies; +using SensitiveFlow.Core.Profiles; using SensitiveFlow.Diagnostics.Extensions; using SensitiveFlow.EFCore.Extensions; using SensitiveFlow.EFCore.Interceptors; +using SensitiveFlow.HealthChecks.Extensions; using SensitiveFlow.Json.Configuration; using SensitiveFlow.Json.Enums; using SensitiveFlow.Json.Extensions; @@ -66,9 +70,30 @@ builder.Services.AddSensitiveFlowLogging(); builder.Services.AddSensitiveFlowEFCore(); builder.Services.AddSensitiveFlowAspNetCore(); + builder.Services.AddSensitiveFlowValidation(options => + { + options.RequireAuditStore = true; + options.RequireTokenStore = true; + options.RequireJsonRedaction = true; + options.RequireRetention = true; + }); builder.Services.AddSensitiveFlowJsonRedaction(options => options.DefaultMode = JsonRedactionMode.Mask); builder.Services.AddRetention(); builder.Services.AddRetentionExecutor(); + builder.Services.AddSingleton(SensitiveFlowPolicyConfiguration.Create(options => + { + options.UseProfile(SensitiveFlowProfile.Strict); + options.Policies.ForCategory(DataCategory.Contact) + .MaskInLogs() + .RedactInJson() + .AuditOnChange(); + options.Policies.ForSensitiveCategory(SensitiveDataCategory.Other) + .OmitInJson() + .RequireAudit(); + })); + builder.Services.AddSensitiveFlowHealthChecks() + .AddAuditStoreCheck() + .AddTokenStoreCheck(); builder.Services.AddOpenTelemetry() .WithTracing(tracing => tracing @@ -114,6 +139,7 @@ await scope.ServiceProvider.GetRequiredService() app.UseHttpsRedirection(); app.UseSensitiveFlowAudit(); app.UseAuthorization(); + app.MapHealthChecks("/health/sensitiveflow"); app.MapControllers(); app.Run(); diff --git a/samples/WebApi.Sample/WebApi.Sample.csproj b/samples/WebApi.Sample/WebApi.Sample.csproj index 28555c5..1d3595d 100644 --- a/samples/WebApi.Sample/WebApi.Sample.csproj +++ b/samples/WebApi.Sample/WebApi.Sample.csproj @@ -28,6 +28,7 @@ + diff --git a/sensitiveflow.slnx b/sensitiveflow.slnx index f0644a3..a9e57c3 100644 --- a/sensitiveflow.slnx +++ b/sensitiveflow.slnx @@ -16,12 +16,14 @@ + + @@ -35,6 +37,7 @@ + diff --git a/src/SensitiveFlow.Analyzers/Analyzers/SensitiveDataLoggingAnalyzer.cs b/src/SensitiveFlow.Analyzers/Analyzers/SensitiveDataLoggingAnalyzer.cs index 2be3cd8..7d771f4 100644 --- a/src/SensitiveFlow.Analyzers/Analyzers/SensitiveDataLoggingAnalyzer.cs +++ b/src/SensitiveFlow.Analyzers/Analyzers/SensitiveDataLoggingAnalyzer.cs @@ -42,6 +42,11 @@ private static void AnalyzeInvocation(OperationAnalysisContext context) continue; } + if (sensitiveMember!.HasAllowSensitiveLoggingAttribute()) + { + continue; + } + context.ReportDiagnostic(Diagnostic.Create( DiagnosticDescriptors.SensitiveDataLoggedDirectly, argument.Syntax.GetLocation(), diff --git a/src/SensitiveFlow.Analyzers/Extensions/SymbolExtensions.cs b/src/SensitiveFlow.Analyzers/Extensions/SymbolExtensions.cs index f93412a..86ff97a 100644 --- a/src/SensitiveFlow.Analyzers/Extensions/SymbolExtensions.cs +++ b/src/SensitiveFlow.Analyzers/Extensions/SymbolExtensions.cs @@ -19,6 +19,32 @@ public static bool HasSensitiveDataAttribute(this ISymbol symbol) return false; } + public static bool HasSensitiveFlowIgnoreAttribute(this ISymbol symbol) + { + foreach (var attribute in symbol.GetAttributes()) + { + if (attribute.AttributeClass?.Name == "SensitiveFlowIgnoreAttribute") + { + return true; + } + } + + return false; + } + + public static bool HasAllowSensitiveLoggingAttribute(this ISymbol symbol) + { + foreach (var attribute in symbol.GetAttributes()) + { + if (attribute.AttributeClass?.Name == "AllowSensitiveLoggingAttribute") + { + return true; + } + } + + return false; + } + public static bool IsSanitizationMethod(this IMethodSymbol method) { var name = method.Name; @@ -42,6 +68,12 @@ public static bool TryFindSensitiveMember(this IOperation operation, out ISymbol if (operation is IPropertyReferenceOperation propertyReference && propertyReference.Property.HasSensitiveDataAttribute()) { + if (propertyReference.Property.HasSensitiveFlowIgnoreAttribute()) + { + member = null; + return false; + } + member = propertyReference.Property; return true; } @@ -49,6 +81,12 @@ public static bool TryFindSensitiveMember(this IOperation operation, out ISymbol if (operation is IFieldReferenceOperation fieldReference && fieldReference.Field.HasSensitiveDataAttribute()) { + if (fieldReference.Field.HasSensitiveFlowIgnoreAttribute()) + { + member = null; + return false; + } + member = fieldReference.Field; return true; } diff --git a/src/SensitiveFlow.Core/Attributes/AllowSensitiveLoggingAttribute.cs b/src/SensitiveFlow.Core/Attributes/AllowSensitiveLoggingAttribute.cs new file mode 100644 index 0000000..cd4816b --- /dev/null +++ b/src/SensitiveFlow.Core/Attributes/AllowSensitiveLoggingAttribute.cs @@ -0,0 +1,31 @@ +namespace SensitiveFlow.Core.Attributes; + +/// +/// Explicitly allows a sensitive member to be logged, suppressing SF0001. +/// A justification is required — this creates an audit trail for why the +/// exception was granted. +/// +/// +/// +/// [PersonalData(Category = DataCategory.Contact)] +/// [AllowSensitiveLogging(Justification = "Email is used as a correlation key in error logs only")] +/// public string Email { get; set; } +/// +/// +[AttributeUsage(AttributeTargets.Property | AttributeTargets.Field | AttributeTargets.Parameter, AllowMultiple = false, Inherited = true)] +public sealed class AllowSensitiveLoggingAttribute : Attribute +{ + /// + /// Initializes a new instance. + /// + /// Why this sensitive value is allowed in logs. This is recorded for audit purposes. + public AllowSensitiveLoggingAttribute(string justification) + { + Justification = justification ?? throw new ArgumentNullException(nameof(justification)); + } + + /// + /// The reason this sensitive value is permitted in log output. + /// + public string Justification { get; } +} diff --git a/src/SensitiveFlow.Core/Attributes/AllowSensitiveReturnAttribute.cs b/src/SensitiveFlow.Core/Attributes/AllowSensitiveReturnAttribute.cs new file mode 100644 index 0000000..230492d --- /dev/null +++ b/src/SensitiveFlow.Core/Attributes/AllowSensitiveReturnAttribute.cs @@ -0,0 +1,18 @@ +namespace SensitiveFlow.Core.Attributes; + +/// +/// Suppresses analyzer warnings for intentionally returning sensitive data. +/// +[AttributeUsage(AttributeTargets.Property | AttributeTargets.Field | AttributeTargets.Method, AllowMultiple = false, Inherited = true)] +public sealed class AllowSensitiveReturnAttribute : Attribute +{ + /// Initializes a new instance with the required justification. + public AllowSensitiveReturnAttribute(string justification) + { + ArgumentException.ThrowIfNullOrWhiteSpace(justification); + Justification = justification; + } + + /// Gets the human-readable justification for allowing the return. + public string Justification { get; } +} diff --git a/src/SensitiveFlow.Core/Attributes/MaskAttribute.cs b/src/SensitiveFlow.Core/Attributes/MaskAttribute.cs new file mode 100644 index 0000000..22c2aa6 --- /dev/null +++ b/src/SensitiveFlow.Core/Attributes/MaskAttribute.cs @@ -0,0 +1,23 @@ +using SensitiveFlow.Core.Enums; + +namespace SensitiveFlow.Core.Attributes; + +/// +/// Requests masking of an annotated member on output surfaces. +/// +[AttributeUsage(AttributeTargets.Property | AttributeTargets.Field, AllowMultiple = false, Inherited = true)] +public sealed class MaskAttribute : Attribute +{ + /// Initializes a new instance. + public MaskAttribute(MaskKind kind = MaskKind.Generic) + { + Kind = kind; + } + + /// Gets the requested mask kind. + public MaskKind Kind { get; } + + /// Gets the output action requested by this attribute. + public OutputRedactionAction Action => OutputRedactionAction.Mask; +} + diff --git a/src/SensitiveFlow.Core/Attributes/OmitAttribute.cs b/src/SensitiveFlow.Core/Attributes/OmitAttribute.cs new file mode 100644 index 0000000..3bbe0c3 --- /dev/null +++ b/src/SensitiveFlow.Core/Attributes/OmitAttribute.cs @@ -0,0 +1,14 @@ +using SensitiveFlow.Core.Enums; + +namespace SensitiveFlow.Core.Attributes; + +/// +/// Requests omission of an annotated member on output surfaces. +/// +[AttributeUsage(AttributeTargets.Property | AttributeTargets.Field, AllowMultiple = false, Inherited = true)] +public sealed class OmitAttribute : Attribute +{ + /// Gets the output action requested by this attribute. + public OutputRedactionAction Action => OutputRedactionAction.Omit; +} + diff --git a/src/SensitiveFlow.Core/Attributes/PersonalDataAttribute.cs b/src/SensitiveFlow.Core/Attributes/PersonalDataAttribute.cs index 0faee06..3ebb460 100644 --- a/src/SensitiveFlow.Core/Attributes/PersonalDataAttribute.cs +++ b/src/SensitiveFlow.Core/Attributes/PersonalDataAttribute.cs @@ -10,6 +10,9 @@ public sealed class PersonalDataAttribute : Attribute { /// Category of personal data. public DataCategory Category { get; set; } = DataCategory.Other; + + /// Risk level used by policy decisions, analyzers, and discovery reports. + public DataSensitivity Sensitivity { get; set; } = DataSensitivity.Medium; } diff --git a/src/SensitiveFlow.Core/Attributes/RedactAttribute.cs b/src/SensitiveFlow.Core/Attributes/RedactAttribute.cs new file mode 100644 index 0000000..87c4095 --- /dev/null +++ b/src/SensitiveFlow.Core/Attributes/RedactAttribute.cs @@ -0,0 +1,14 @@ +using SensitiveFlow.Core.Enums; + +namespace SensitiveFlow.Core.Attributes; + +/// +/// Requests redaction of an annotated member on output surfaces. +/// +[AttributeUsage(AttributeTargets.Property | AttributeTargets.Field, AllowMultiple = false, Inherited = true)] +public sealed class RedactAttribute : Attribute +{ + /// Gets the output action requested by this attribute. + public OutputRedactionAction Action => OutputRedactionAction.Redact; +} + diff --git a/src/SensitiveFlow.Core/Attributes/RedactionAttribute.cs b/src/SensitiveFlow.Core/Attributes/RedactionAttribute.cs new file mode 100644 index 0000000..5a3d28d --- /dev/null +++ b/src/SensitiveFlow.Core/Attributes/RedactionAttribute.cs @@ -0,0 +1,36 @@ +using SensitiveFlow.Core.Enums; + +namespace SensitiveFlow.Core.Attributes; + +/// +/// Declares contextual redaction behavior for a member. +/// +[AttributeUsage(AttributeTargets.Property | AttributeTargets.Field, AllowMultiple = false, Inherited = true)] +public sealed class RedactionAttribute : Attribute +{ + /// Gets or sets the API response action. + public OutputRedactionAction ApiResponse { get; set; } = OutputRedactionAction.None; + + /// Gets or sets the log action. + public OutputRedactionAction Logs { get; set; } = OutputRedactionAction.None; + + /// Gets or sets the audit action. + public OutputRedactionAction Audit { get; set; } = OutputRedactionAction.None; + + /// Gets or sets the export action. + public OutputRedactionAction Export { get; set; } = OutputRedactionAction.None; + + /// Resolves the action for the given context. + public OutputRedactionAction ForContext(RedactionContext context) + { + return context switch + { + RedactionContext.ApiResponse => ApiResponse, + RedactionContext.Log => Logs, + RedactionContext.Audit => Audit, + RedactionContext.Export => Export, + _ => OutputRedactionAction.None, + }; + } +} + diff --git a/src/SensitiveFlow.Core/Attributes/SensitiveDataAttribute.cs b/src/SensitiveFlow.Core/Attributes/SensitiveDataAttribute.cs index 0e56566..77e788f 100644 --- a/src/SensitiveFlow.Core/Attributes/SensitiveDataAttribute.cs +++ b/src/SensitiveFlow.Core/Attributes/SensitiveDataAttribute.cs @@ -14,6 +14,9 @@ public sealed class SensitiveDataAttribute : Attribute /// sensitive categories carry distinct obligations. /// public SensitiveDataCategory Category { get; set; } = SensitiveDataCategory.Other; + + /// Risk level used by policy decisions, analyzers, and discovery reports. + public DataSensitivity Sensitivity { get; set; } = DataSensitivity.High; } diff --git a/src/SensitiveFlow.Core/Attributes/SensitiveFlowIgnoreAttribute.cs b/src/SensitiveFlow.Core/Attributes/SensitiveFlowIgnoreAttribute.cs new file mode 100644 index 0000000..b265c7a --- /dev/null +++ b/src/SensitiveFlow.Core/Attributes/SensitiveFlowIgnoreAttribute.cs @@ -0,0 +1,16 @@ +namespace SensitiveFlow.Core.Attributes; + +/// +/// Suppresses SensitiveFlow analyzer diagnostics on the target member. +/// Use sparingly and only when you are certain the data is not sensitive +/// despite matching a pattern that would normally trigger a diagnostic. +/// +/// +/// This attribute tells analyzers to skip the decorated member entirely. +/// Prefer when you need to +/// log a sensitive value intentionally — it requires a justification. +/// +[AttributeUsage(AttributeTargets.Property | AttributeTargets.Field | AttributeTargets.Parameter, AllowMultiple = false, Inherited = true)] +public sealed class SensitiveFlowIgnoreAttribute : Attribute +{ +} diff --git a/src/SensitiveFlow.Core/Correlation/AuditCorrelationSnapshot.cs b/src/SensitiveFlow.Core/Correlation/AuditCorrelationSnapshot.cs new file mode 100644 index 0000000..c340cb2 --- /dev/null +++ b/src/SensitiveFlow.Core/Correlation/AuditCorrelationSnapshot.cs @@ -0,0 +1,16 @@ +namespace SensitiveFlow.Core.Correlation; + +/// +/// Immutable correlation snapshot for audit and diagnostics flows. +/// +public sealed record AuditCorrelationSnapshot : IAuditCorrelationContext +{ + /// + public string? CorrelationId { get; init; } + + /// + public string? RequestId { get; init; } + + /// + public string? TraceId { get; init; } +} diff --git a/src/SensitiveFlow.Core/Correlation/IAuditCorrelationContext.cs b/src/SensitiveFlow.Core/Correlation/IAuditCorrelationContext.cs new file mode 100644 index 0000000..5113189 --- /dev/null +++ b/src/SensitiveFlow.Core/Correlation/IAuditCorrelationContext.cs @@ -0,0 +1,17 @@ +namespace SensitiveFlow.Core.Correlation; + +/// +/// Supplies correlation identifiers for audit and diagnostics records. +/// +public interface IAuditCorrelationContext +{ + /// Gets the current correlation identifier. + string? CorrelationId { get; } + + /// Gets the current request identifier. + string? RequestId { get; } + + /// Gets the current trace identifier. + string? TraceId { get; } +} + diff --git a/src/SensitiveFlow.Core/Correlation/SensitiveFlowCorrelation.cs b/src/SensitiveFlow.Core/Correlation/SensitiveFlowCorrelation.cs new file mode 100644 index 0000000..364dd41 --- /dev/null +++ b/src/SensitiveFlow.Core/Correlation/SensitiveFlowCorrelation.cs @@ -0,0 +1,19 @@ +using System.Threading; + +namespace SensitiveFlow.Core.Correlation; + +/// +/// Async-local correlation holder for non-ASP.NET code paths. +/// +public static class SensitiveFlowCorrelation +{ + private static readonly AsyncLocal CurrentSnapshot = new(); + + /// Gets the current correlation snapshot. + public static AuditCorrelationSnapshot? Current + { + get => CurrentSnapshot.Value; + set => CurrentSnapshot.Value = value; + } +} + diff --git a/src/SensitiveFlow.Core/Discovery/SensitiveDataDiscovery.cs b/src/SensitiveFlow.Core/Discovery/SensitiveDataDiscovery.cs new file mode 100644 index 0000000..4775ac4 --- /dev/null +++ b/src/SensitiveFlow.Core/Discovery/SensitiveDataDiscovery.cs @@ -0,0 +1,54 @@ +using System.Reflection; +using SensitiveFlow.Core.Attributes; + +namespace SensitiveFlow.Core.Discovery; + +/// +/// Scans assemblies for SensitiveFlow annotations and produces in-memory, JSON, or Markdown reports. +/// +public static class SensitiveDataDiscovery +{ + /// Scans one assembly for annotated fields and properties. + public static SensitiveDataDiscoveryReport Scan(Assembly assembly) + { + ArgumentNullException.ThrowIfNull(assembly); + + var entries = new List(); + foreach (var type in assembly.GetTypes().Where(static t => !t.IsGenericTypeDefinition)) + { + foreach (var member in type.GetMembers(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)) + { + if (member.MemberType is not (MemberTypes.Property or MemberTypes.Field)) + { + continue; + } + + var personal = member.GetCustomAttribute(inherit: true); + var sensitive = member.GetCustomAttribute(inherit: true); + var retention = member.GetCustomAttribute(inherit: true); + + if (personal is null && sensitive is null && retention is null) + { + continue; + } + + entries.Add(new SensitiveDataDiscoveryEntry + { + TypeName = type.Name, + MemberName = member.Name, + Annotation = sensitive is not null + ? nameof(SensitiveDataAttribute) + : personal is not null ? nameof(PersonalDataAttribute) : nameof(RetentionDataAttribute), + Category = personal?.Category, + SensitiveCategory = sensitive?.Category, + Sensitivity = sensitive?.Sensitivity ?? personal?.Sensitivity ?? Core.Enums.DataSensitivity.Medium, + RetentionYears = retention?.Years, + RetentionMonths = retention?.Months, + RetentionPolicy = retention?.Policy, + }); + } + } + + return new SensitiveDataDiscoveryReport(entries); + } +} diff --git a/src/SensitiveFlow.Core/Discovery/SensitiveDataDiscoveryEntry.cs b/src/SensitiveFlow.Core/Discovery/SensitiveDataDiscoveryEntry.cs new file mode 100644 index 0000000..1294d6a --- /dev/null +++ b/src/SensitiveFlow.Core/Discovery/SensitiveDataDiscoveryEntry.cs @@ -0,0 +1,37 @@ +using SensitiveFlow.Core.Enums; + +namespace SensitiveFlow.Core.Discovery; + +/// +/// One annotated member discovered in an assembly scan. +/// +public sealed record SensitiveDataDiscoveryEntry +{ + /// Gets the containing type name. + public required string TypeName { get; init; } + + /// Gets the member name. + public required string MemberName { get; init; } + + /// Gets the annotation kind. + public required string Annotation { get; init; } + + /// Gets the regular personal data category, when present. + public DataCategory? Category { get; init; } + + /// Gets the sensitive personal data category, when present. + public SensitiveDataCategory? SensitiveCategory { get; init; } + + /// Gets the sensitivity level. + public DataSensitivity Sensitivity { get; init; } + + /// Gets the retention period in years, when present. + public int? RetentionYears { get; init; } + + /// Gets the retention period in months, when present. + public int? RetentionMonths { get; init; } + + /// Gets the retention policy, when present. + public RetentionPolicy? RetentionPolicy { get; init; } +} + diff --git a/src/SensitiveFlow.Core/Discovery/SensitiveDataDiscoveryReport.cs b/src/SensitiveFlow.Core/Discovery/SensitiveDataDiscoveryReport.cs new file mode 100644 index 0000000..fdbe83e --- /dev/null +++ b/src/SensitiveFlow.Core/Discovery/SensitiveDataDiscoveryReport.cs @@ -0,0 +1,57 @@ +using System.Text; +using System.Text.Json; + +namespace SensitiveFlow.Core.Discovery; + +/// +/// Report produced by scanning assemblies for SensitiveFlow annotations. +/// +public sealed class SensitiveDataDiscoveryReport +{ + /// Initializes a new report. + public SensitiveDataDiscoveryReport(IEnumerable entries) + { + ArgumentNullException.ThrowIfNull(entries); + Entries = entries.OrderBy(static e => e.TypeName, StringComparer.Ordinal) + .ThenBy(static e => e.MemberName, StringComparer.Ordinal) + .ToArray(); + } + + /// Gets discovered entries. + public IReadOnlyList Entries { get; } + + /// Serializes the report to indented JSON. + public string ToJson() + { + return JsonSerializer.Serialize(Entries, new JsonSerializerOptions { WriteIndented = true }); + } + + /// Renders the report as a Markdown table. + public string ToMarkdown() + { + var sb = new StringBuilder(); + sb.AppendLine("| Member | Annotation | Category | Sensitivity | Retention |"); + sb.AppendLine("| --- | --- | --- | --- | --- |"); + + foreach (var entry in Entries) + { + var category = entry.Category?.ToString() + ?? entry.SensitiveCategory?.ToString() + ?? "Other"; + var retention = entry.RetentionPolicy is null + ? string.Empty + : $"{entry.RetentionYears.GetValueOrDefault()}y {entry.RetentionMonths.GetValueOrDefault()}m / {entry.RetentionPolicy}"; + + sb.AppendLine(CultureInvariant( + $"| {entry.TypeName}.{entry.MemberName} | {entry.Annotation} | {category} | {entry.Sensitivity} | {retention} |")); + } + + return sb.ToString(); + } + + private static string CultureInvariant(FormattableString value) + { + return FormattableString.Invariant(value); + } +} + diff --git a/src/SensitiveFlow.Core/Enums/DataSensitivity.cs b/src/SensitiveFlow.Core/Enums/DataSensitivity.cs new file mode 100644 index 0000000..c98544e --- /dev/null +++ b/src/SensitiveFlow.Core/Enums/DataSensitivity.cs @@ -0,0 +1,20 @@ +namespace SensitiveFlow.Core.Enums; + +/// +/// Risk level assigned to personal or sensitive data. +/// +public enum DataSensitivity +{ + /// Low risk if exposed, usually because the value is already public or weakly identifying. + Low = 0, + + /// Moderate risk if exposed. + Medium, + + /// High risk if exposed. + High, + + /// Critical risk if exposed, requiring the strictest handling available. + Critical, +} + diff --git a/src/SensitiveFlow.Core/Enums/MaskKind.cs b/src/SensitiveFlow.Core/Enums/MaskKind.cs new file mode 100644 index 0000000..6e0293d --- /dev/null +++ b/src/SensitiveFlow.Core/Enums/MaskKind.cs @@ -0,0 +1,20 @@ +namespace SensitiveFlow.Core.Enums; + +/// +/// Built-in masking shape requested by output behavior attributes and assertions. +/// +public enum MaskKind +{ + /// Generic mask preserving only minimal structure. + Generic = 0, + + /// Email-aware mask. + Email, + + /// Phone-aware mask. + Phone, + + /// Name-aware mask. + Name, +} + diff --git a/src/SensitiveFlow.Core/Enums/OutputRedactionAction.cs b/src/SensitiveFlow.Core/Enums/OutputRedactionAction.cs new file mode 100644 index 0000000..de9b7f5 --- /dev/null +++ b/src/SensitiveFlow.Core/Enums/OutputRedactionAction.cs @@ -0,0 +1,23 @@ +namespace SensitiveFlow.Core.Enums; + +/// +/// Output handling action for annotated sensitive values. +/// +public enum OutputRedactionAction +{ + /// No explicit output handling was requested. + None = 0, + + /// Replace the value with a fixed placeholder. + Redact, + + /// Mask the value while preserving limited shape. + Mask, + + /// Omit the value from the output surface. + Omit, + + /// Pseudonymize the value before output. + Pseudonymize, +} + diff --git a/src/SensitiveFlow.Core/Enums/RedactionContext.cs b/src/SensitiveFlow.Core/Enums/RedactionContext.cs new file mode 100644 index 0000000..a4e4d03 --- /dev/null +++ b/src/SensitiveFlow.Core/Enums/RedactionContext.cs @@ -0,0 +1,20 @@ +namespace SensitiveFlow.Core.Enums; + +/// +/// Output context for contextual redaction decisions. +/// +public enum RedactionContext +{ + /// Generic API response context. + ApiResponse = 0, + + /// Application logging context. + Log, + + /// Audit record context. + Audit, + + /// Data export context. + Export, +} + diff --git a/src/SensitiveFlow.Core/Exceptions/SensitiveFlowConfigurationException.cs b/src/SensitiveFlow.Core/Exceptions/SensitiveFlowConfigurationException.cs new file mode 100644 index 0000000..efed62e --- /dev/null +++ b/src/SensitiveFlow.Core/Exceptions/SensitiveFlowConfigurationException.cs @@ -0,0 +1,14 @@ +namespace SensitiveFlow.Core.Exceptions; + +/// +/// Exception thrown when SensitiveFlow is configured inconsistently. +/// +public sealed class SensitiveFlowConfigurationException : SensitiveFlowException +{ + /// Initializes a new instance. + public SensitiveFlowConfigurationException(string message, string code, Exception? innerException = null) + : base(message, code, innerException) + { + } +} + diff --git a/src/SensitiveFlow.Core/Exceptions/SensitiveFlowException.cs b/src/SensitiveFlow.Core/Exceptions/SensitiveFlowException.cs new file mode 100644 index 0000000..1f85037 --- /dev/null +++ b/src/SensitiveFlow.Core/Exceptions/SensitiveFlowException.cs @@ -0,0 +1,19 @@ +namespace SensitiveFlow.Core.Exceptions; + +/// +/// Base exception type for SensitiveFlow failures with a safe machine-readable code. +/// +public class SensitiveFlowException : Exception +{ + /// Initializes a new instance. + public SensitiveFlowException(string message, string code, Exception? innerException = null) + : base(message, innerException) + { + ArgumentException.ThrowIfNullOrWhiteSpace(code); + Code = code; + } + + /// Gets the safe machine-readable code. + public string Code { get; } +} + diff --git a/src/SensitiveFlow.Core/Exceptions/SensitiveFlowExceptionSanitizer.cs b/src/SensitiveFlow.Core/Exceptions/SensitiveFlowExceptionSanitizer.cs new file mode 100644 index 0000000..7f8feac --- /dev/null +++ b/src/SensitiveFlow.Core/Exceptions/SensitiveFlowExceptionSanitizer.cs @@ -0,0 +1,42 @@ +using System.Text.RegularExpressions; + +namespace SensitiveFlow.Core.Exceptions; + +/// +/// Produces safe exception messages by removing common high-risk raw values. +/// +public static partial class SensitiveFlowExceptionSanitizer +{ + /// Creates a sanitized exception view suitable for logs and diagnostics. + public static SensitiveFlowSanitizedException Sanitize(Exception exception) + { + ArgumentNullException.ThrowIfNull(exception); + + var code = exception is SensitiveFlowException sensitiveFlowException + ? sensitiveFlowException.Code + : null; + + return new SensitiveFlowSanitizedException + { + Type = exception.GetType().Name, + Code = code, + Message = SanitizeMessage(exception.Message), + }; + } + + /// Sanitizes a raw exception message. + public static string SanitizeMessage(string message) + { + ArgumentNullException.ThrowIfNull(message); + var sanitized = EmailRegex().Replace(message, "[email]"); + sanitized = LongDigitRegex().Replace(sanitized, "[number]"); + return sanitized; + } + + [GeneratedRegex(@"[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] + private static partial Regex EmailRegex(); + + [GeneratedRegex(@"\b\d[\d.\-\/\s]{5,}\d\b", RegexOptions.CultureInvariant)] + private static partial Regex LongDigitRegex(); +} + diff --git a/src/SensitiveFlow.Core/Exceptions/SensitiveFlowSanitizedException.cs b/src/SensitiveFlow.Core/Exceptions/SensitiveFlowSanitizedException.cs new file mode 100644 index 0000000..7b4c218 --- /dev/null +++ b/src/SensitiveFlow.Core/Exceptions/SensitiveFlowSanitizedException.cs @@ -0,0 +1,17 @@ +namespace SensitiveFlow.Core.Exceptions; + +/// +/// Privacy-safe exception view for logs and diagnostics. +/// +public sealed record SensitiveFlowSanitizedException +{ + /// Gets the exception type name. + public required string Type { get; init; } + + /// Gets a safe error code when available. + public string? Code { get; init; } + + /// Gets the sanitized message. + public required string Message { get; init; } +} + diff --git a/src/SensitiveFlow.Core/Export/CsvDataExportFormatter.cs b/src/SensitiveFlow.Core/Export/CsvDataExportFormatter.cs new file mode 100644 index 0000000..c407b7a --- /dev/null +++ b/src/SensitiveFlow.Core/Export/CsvDataExportFormatter.cs @@ -0,0 +1,44 @@ +using System.Text; + +namespace SensitiveFlow.Core.Export; + +/// +/// CSV formatter with formula-injection protection for spreadsheet consumers. +/// +public sealed class CsvDataExportFormatter : IDataExportFormatter +{ + /// + public string Format(IEnumerable> rows) + { + ArgumentNullException.ThrowIfNull(rows); + var materialized = rows.ToArray(); + if (materialized.Length == 0) + { + return string.Empty; + } + + var headers = materialized.SelectMany(static row => row.Keys) + .Distinct(StringComparer.Ordinal) + .ToArray(); + + var sb = new StringBuilder(); + sb.AppendLine(string.Join(",", headers.Select(Escape))); + foreach (var row in materialized) + { + sb.AppendLine(string.Join(",", headers.Select(h => Escape(row.TryGetValue(h, out var value) ? value?.ToString() : string.Empty)))); + } + + return sb.ToString(); + } + + private static string Escape(string? value) + { + var safe = value ?? string.Empty; + if (safe.Length > 0 && "=+-@".Contains(safe[0], StringComparison.Ordinal)) + { + safe = "'" + safe; + } + + return "\"" + safe.Replace("\"", "\"\"", StringComparison.Ordinal) + "\""; + } +} diff --git a/src/SensitiveFlow.Core/Export/IDataExportFormatter.cs b/src/SensitiveFlow.Core/Export/IDataExportFormatter.cs new file mode 100644 index 0000000..e090386 --- /dev/null +++ b/src/SensitiveFlow.Core/Export/IDataExportFormatter.cs @@ -0,0 +1,11 @@ +namespace SensitiveFlow.Core.Export; + +/// +/// Formats data export rows into a transport representation. +/// +public interface IDataExportFormatter +{ + /// Formats the given rows. + string Format(IEnumerable> rows); +} + diff --git a/src/SensitiveFlow.Core/Export/JsonDataExportFormatter.cs b/src/SensitiveFlow.Core/Export/JsonDataExportFormatter.cs new file mode 100644 index 0000000..0b75cdc --- /dev/null +++ b/src/SensitiveFlow.Core/Export/JsonDataExportFormatter.cs @@ -0,0 +1,17 @@ +using System.Text.Json; + +namespace SensitiveFlow.Core.Export; + +/// +/// JSON formatter for data export rows. +/// +public sealed class JsonDataExportFormatter : IDataExportFormatter +{ + /// + public string Format(IEnumerable> rows) + { + ArgumentNullException.ThrowIfNull(rows); + return JsonSerializer.Serialize(rows, new JsonSerializerOptions { WriteIndented = true }); + } +} + diff --git a/src/SensitiveFlow.Core/Interfaces/IAuditOutbox.cs b/src/SensitiveFlow.Core/Interfaces/IAuditOutbox.cs new file mode 100644 index 0000000..1377d65 --- /dev/null +++ b/src/SensitiveFlow.Core/Interfaces/IAuditOutbox.cs @@ -0,0 +1,13 @@ +using SensitiveFlow.Core.Models; + +namespace SensitiveFlow.Core.Interfaces; + +/// +/// Application-owned outbox abstraction for durable audit delivery. +/// +public interface IAuditOutbox +{ + /// Enqueues an audit record for later delivery. + Task EnqueueAsync(AuditRecord record, CancellationToken cancellationToken = default); +} + diff --git a/src/SensitiveFlow.Core/Interfaces/IAuditOutboxSerializer.cs b/src/SensitiveFlow.Core/Interfaces/IAuditOutboxSerializer.cs new file mode 100644 index 0000000..bb05c43 --- /dev/null +++ b/src/SensitiveFlow.Core/Interfaces/IAuditOutboxSerializer.cs @@ -0,0 +1,16 @@ +using SensitiveFlow.Core.Models; + +namespace SensitiveFlow.Core.Interfaces; + +/// +/// Serializes audit outbox payloads without prescribing the outbox storage backend. +/// +public interface IAuditOutboxSerializer +{ + /// Serializes an audit record. + string Serialize(AuditRecord record); + + /// Deserializes an audit record. + AuditRecord Deserialize(string payload); +} + diff --git a/src/SensitiveFlow.Core/Interfaces/IDataSubjectEraser.cs b/src/SensitiveFlow.Core/Interfaces/IDataSubjectEraser.cs new file mode 100644 index 0000000..19ff31f --- /dev/null +++ b/src/SensitiveFlow.Core/Interfaces/IDataSubjectEraser.cs @@ -0,0 +1,12 @@ +namespace SensitiveFlow.Core.Interfaces; + +/// +/// Erases or anonymizes data-subject data in a specific source type. +/// +/// Source type to erase. +public interface IDataSubjectEraser +{ + /// Erases or anonymizes data for the given source object. + Task EraseAsync(T source, CancellationToken cancellationToken = default); +} + diff --git a/src/SensitiveFlow.Core/Interfaces/IDataSubjectExporter.cs b/src/SensitiveFlow.Core/Interfaces/IDataSubjectExporter.cs new file mode 100644 index 0000000..74f4d85 --- /dev/null +++ b/src/SensitiveFlow.Core/Interfaces/IDataSubjectExporter.cs @@ -0,0 +1,12 @@ +namespace SensitiveFlow.Core.Interfaces; + +/// +/// Exports data-subject data from a specific source type. +/// +/// Source type to export. +public interface IDataSubjectExporter +{ + /// Exports data from the given source object. + Task> ExportAsync(T source, CancellationToken cancellationToken = default); +} + diff --git a/src/SensitiveFlow.Core/Interfaces/IDataSubjectRequestHandler.cs b/src/SensitiveFlow.Core/Interfaces/IDataSubjectRequestHandler.cs new file mode 100644 index 0000000..c415ef8 --- /dev/null +++ b/src/SensitiveFlow.Core/Interfaces/IDataSubjectRequestHandler.cs @@ -0,0 +1,11 @@ +namespace SensitiveFlow.Core.Interfaces; + +/// +/// Handles a data-subject request coordinated by the application. +/// +public interface IDataSubjectRequestHandler +{ + /// Handles the request. + Task HandleAsync(string dataSubjectId, string requestType, CancellationToken cancellationToken = default); +} + diff --git a/src/SensitiveFlow.Core/Interfaces/IHealthProbe.cs b/src/SensitiveFlow.Core/Interfaces/IHealthProbe.cs new file mode 100644 index 0000000..5f87a8f --- /dev/null +++ b/src/SensitiveFlow.Core/Interfaces/IHealthProbe.cs @@ -0,0 +1,11 @@ +namespace SensitiveFlow.Core.Interfaces; + +/// +/// Optional non-mutating health probe for SensitiveFlow infrastructure implementations. +/// +public interface IHealthProbe +{ + /// Verifies the implementation can respond to a lightweight probe. + Task ProbeAsync(CancellationToken cancellationToken = default); +} + diff --git a/src/SensitiveFlow.Core/Policies/SensitiveFlowPolicyAction.cs b/src/SensitiveFlow.Core/Policies/SensitiveFlowPolicyAction.cs new file mode 100644 index 0000000..c00d9a6 --- /dev/null +++ b/src/SensitiveFlow.Core/Policies/SensitiveFlowPolicyAction.cs @@ -0,0 +1,27 @@ +namespace SensitiveFlow.Core.Policies; + +/// +/// Handling actions that can be requested by a SensitiveFlow policy. +/// +[Flags] +public enum SensitiveFlowPolicyAction +{ + /// No behavior is requested. + None = 0, + + /// Mask values before writing them to logs. + MaskInLogs = 1 << 0, + + /// Redact values before serializing them as JSON. + RedactInJson = 1 << 1, + + /// Omit values from JSON output. + OmitInJson = 1 << 2, + + /// Record audit entries when values change. + AuditOnChange = 1 << 3, + + /// Require audit support before processing this category. + RequireAudit = 1 << 4, +} + diff --git a/src/SensitiveFlow.Core/Policies/SensitiveFlowPolicyConfiguration.cs b/src/SensitiveFlow.Core/Policies/SensitiveFlowPolicyConfiguration.cs new file mode 100644 index 0000000..066c939 --- /dev/null +++ b/src/SensitiveFlow.Core/Policies/SensitiveFlowPolicyConfiguration.cs @@ -0,0 +1,17 @@ +using SensitiveFlow.Core.Profiles; + +namespace SensitiveFlow.Core.Policies; + +/// +/// Helper methods for constructing SensitiveFlow policy options without a DI dependency. +/// +public static class SensitiveFlowPolicyConfiguration +{ + /// Creates and configures a new options instance. + public static SensitiveFlowOptions Create(Action? configure = null) + { + var options = new SensitiveFlowOptions(); + configure?.Invoke(options); + return options; + } +} diff --git a/src/SensitiveFlow.Core/Policies/SensitiveFlowPolicyRegistry.cs b/src/SensitiveFlow.Core/Policies/SensitiveFlowPolicyRegistry.cs new file mode 100644 index 0000000..87d64c6 --- /dev/null +++ b/src/SensitiveFlow.Core/Policies/SensitiveFlowPolicyRegistry.cs @@ -0,0 +1,63 @@ +using SensitiveFlow.Core.Enums; + +namespace SensitiveFlow.Core.Policies; + +/// +/// Registry of category-level policy rules. +/// +public sealed class SensitiveFlowPolicyRegistry +{ + private readonly Dictionary _personal = []; + private readonly Dictionary _sensitive = []; + + /// Starts or updates a rule for a personal data category. + public SensitiveFlowPolicyRuleBuilder ForCategory(DataCategory category) + { + if (!_personal.TryGetValue(category, out var builder)) + { + builder = new SensitiveFlowPolicyRuleBuilder(new SensitiveFlowPolicyRule + { + IsSensitiveCategory = false, + Category = category, + Actions = SensitiveFlowPolicyAction.None, + }); + _personal.Add(category, builder); + } + + return builder; + } + + /// Starts or updates a rule for a sensitive data category. + public SensitiveFlowPolicyRuleBuilder ForSensitiveCategory(SensitiveDataCategory category) + { + if (!_sensitive.TryGetValue(category, out var builder)) + { + builder = new SensitiveFlowPolicyRuleBuilder(new SensitiveFlowPolicyRule + { + IsSensitiveCategory = true, + SensitiveCategory = category, + Actions = SensitiveFlowPolicyAction.None, + }); + _sensitive.Add(category, builder); + } + + return builder; + } + + /// Gets all configured policy rules. + public IReadOnlyList Rules => + _personal.Values.Concat(_sensitive.Values).Select(static b => b.Rule).ToArray(); + + /// Finds a rule for a personal data category. + public SensitiveFlowPolicyRule? Find(DataCategory category) + { + return _personal.TryGetValue(category, out var builder) ? builder.Rule : null; + } + + /// Finds a rule for a sensitive data category. + public SensitiveFlowPolicyRule? Find(SensitiveDataCategory category) + { + return _sensitive.TryGetValue(category, out var builder) ? builder.Rule : null; + } +} + diff --git a/src/SensitiveFlow.Core/Policies/SensitiveFlowPolicyRule.cs b/src/SensitiveFlow.Core/Policies/SensitiveFlowPolicyRule.cs new file mode 100644 index 0000000..058869d --- /dev/null +++ b/src/SensitiveFlow.Core/Policies/SensitiveFlowPolicyRule.cs @@ -0,0 +1,22 @@ +using SensitiveFlow.Core.Enums; + +namespace SensitiveFlow.Core.Policies; + +/// +/// Immutable policy rule for one personal or sensitive data category. +/// +public sealed record SensitiveFlowPolicyRule +{ + /// Gets whether the rule targets sensitive personal data. + public required bool IsSensitiveCategory { get; init; } + + /// Gets the regular personal data category, when applicable. + public DataCategory? Category { get; init; } + + /// Gets the sensitive data category, when applicable. + public SensitiveDataCategory? SensitiveCategory { get; init; } + + /// Gets the configured actions. + public SensitiveFlowPolicyAction Actions { get; init; } +} + diff --git a/src/SensitiveFlow.Core/Policies/SensitiveFlowPolicyRuleBuilder.cs b/src/SensitiveFlow.Core/Policies/SensitiveFlowPolicyRuleBuilder.cs new file mode 100644 index 0000000..df46277 --- /dev/null +++ b/src/SensitiveFlow.Core/Policies/SensitiveFlowPolicyRuleBuilder.cs @@ -0,0 +1,55 @@ +namespace SensitiveFlow.Core.Policies; + +/// +/// Fluent builder for a single SensitiveFlow policy rule. +/// +public sealed class SensitiveFlowPolicyRuleBuilder +{ + private SensitiveFlowPolicyAction _actions; + + internal SensitiveFlowPolicyRuleBuilder(SensitiveFlowPolicyRule seed) + { + Rule = seed; + _actions = seed.Actions; + } + + internal SensitiveFlowPolicyRule Rule { get; private set; } + + /// Requests log masking for the category. + public SensitiveFlowPolicyRuleBuilder MaskInLogs() + { + return Add(SensitiveFlowPolicyAction.MaskInLogs); + } + + /// Requests JSON redaction for the category. + public SensitiveFlowPolicyRuleBuilder RedactInJson() + { + return Add(SensitiveFlowPolicyAction.RedactInJson); + } + + /// Requests JSON omission for the category. + public SensitiveFlowPolicyRuleBuilder OmitInJson() + { + return Add(SensitiveFlowPolicyAction.OmitInJson); + } + + /// Requests audit entries when values in the category change. + public SensitiveFlowPolicyRuleBuilder AuditOnChange() + { + return Add(SensitiveFlowPolicyAction.AuditOnChange); + } + + /// Requires audit support for this category. + public SensitiveFlowPolicyRuleBuilder RequireAudit() + { + return Add(SensitiveFlowPolicyAction.RequireAudit); + } + + private SensitiveFlowPolicyRuleBuilder Add(SensitiveFlowPolicyAction action) + { + _actions |= action; + Rule = Rule with { Actions = _actions }; + return this; + } +} + diff --git a/src/SensitiveFlow.Core/Profiles/SensitiveFlowOptions.cs b/src/SensitiveFlow.Core/Profiles/SensitiveFlowOptions.cs new file mode 100644 index 0000000..2ecd755 --- /dev/null +++ b/src/SensitiveFlow.Core/Profiles/SensitiveFlowOptions.cs @@ -0,0 +1,53 @@ +using SensitiveFlow.Core.Enums; +using SensitiveFlow.Core.Policies; + +namespace SensitiveFlow.Core.Profiles; + +/// +/// Shared SensitiveFlow options used by modules that want central policy/profile decisions. +/// +public sealed class SensitiveFlowOptions +{ + /// Gets the active profile. + public SensitiveFlowProfile Profile { get; private set; } = SensitiveFlowProfile.Balanced; + + /// Gets the policy registry. + public SensitiveFlowPolicyRegistry Policies { get; } = new(); + + /// Applies one of the built-in profiles. + public SensitiveFlowOptions UseProfile(SensitiveFlowProfile profile) + { + Profile = profile; + + switch (profile) + { + case SensitiveFlowProfile.Development: + Policies.ForCategory(DataCategory.Contact).MaskInLogs().RedactInJson(); + Policies.ForCategory(DataCategory.Identification).MaskInLogs(); + break; + case SensitiveFlowProfile.Balanced: + Policies.ForCategory(DataCategory.Contact).MaskInLogs().RedactInJson().AuditOnChange(); + Policies.ForCategory(DataCategory.Financial).MaskInLogs().OmitInJson().RequireAudit(); + Policies.ForSensitiveCategory(SensitiveDataCategory.Other).OmitInJson().RequireAudit(); + break; + case SensitiveFlowProfile.Strict: + Policies.ForCategory(DataCategory.Contact).MaskInLogs().OmitInJson().RequireAudit(); + Policies.ForCategory(DataCategory.Identification).MaskInLogs().RedactInJson().RequireAudit(); + Policies.ForCategory(DataCategory.Financial).MaskInLogs().OmitInJson().RequireAudit(); + foreach (var category in Enum.GetValues()) + { + Policies.ForSensitiveCategory(category).OmitInJson().RequireAudit(); + } + break; + case SensitiveFlowProfile.AuditOnly: + Policies.ForCategory(DataCategory.Other).AuditOnChange(); + Policies.ForSensitiveCategory(SensitiveDataCategory.Other).RequireAudit(); + break; + default: + throw new ArgumentOutOfRangeException(nameof(profile), profile, "Unknown SensitiveFlow profile."); + } + + return this; + } +} + diff --git a/src/SensitiveFlow.Core/Profiles/SensitiveFlowProfile.cs b/src/SensitiveFlow.Core/Profiles/SensitiveFlowProfile.cs new file mode 100644 index 0000000..51d62b2 --- /dev/null +++ b/src/SensitiveFlow.Core/Profiles/SensitiveFlowProfile.cs @@ -0,0 +1,19 @@ +namespace SensitiveFlow.Core.Profiles; + +/// +/// Built-in handling profiles that provide a quick starting point for SensitiveFlow options. +/// +public enum SensitiveFlowProfile +{ + /// Developer-friendly defaults with masking and lightweight audit hints. + Development = 0, + + /// Balanced defaults suitable for most API and logging surfaces. + Balanced, + + /// Strict defaults that prefer omission/redaction and required audit. + Strict, + + /// Audit-focused defaults that do not force JSON/log redaction decisions. + AuditOnly, +} diff --git a/src/SensitiveFlow.Diagnostics/Extensions/SensitiveFlowValidationServiceCollectionExtensions.cs b/src/SensitiveFlow.Diagnostics/Extensions/SensitiveFlowValidationServiceCollectionExtensions.cs new file mode 100644 index 0000000..a44c4ce --- /dev/null +++ b/src/SensitiveFlow.Diagnostics/Extensions/SensitiveFlowValidationServiceCollectionExtensions.cs @@ -0,0 +1,39 @@ +using Microsoft.Extensions.DependencyInjection; +using SensitiveFlow.Diagnostics.Validation; + +namespace SensitiveFlow.Diagnostics.Extensions; + +/// +/// Service registration helpers for SensitiveFlow configuration validation. +/// +public static class SensitiveFlowValidationServiceCollectionExtensions +{ + /// Registers startup validation options and validator services. + public static IServiceCollection AddSensitiveFlowValidation( + this IServiceCollection services, + Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + var options = new SensitiveFlowValidationOptions(); + configure?.Invoke(options); + services.AddSingleton(options); + services.AddSingleton(); + return services; + } + + /// Runs SensitiveFlow startup validation from a built service provider. + public static SensitiveFlowConfigurationReport ValidateSensitiveFlow(this IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + var validator = services.GetService() + ?? new SensitiveFlowConfigurationValidator(new SensitiveFlowValidationOptions + { + RequireAuditStore = true, + RequireTokenStore = false, + RequireJsonRedaction = false, + RequireRetention = false, + }); + + return validator.Validate(services); + } +} diff --git a/src/SensitiveFlow.Diagnostics/SensitiveFlow.Diagnostics.csproj b/src/SensitiveFlow.Diagnostics/SensitiveFlow.Diagnostics.csproj index a02fcb0..e2a2c99 100644 --- a/src/SensitiveFlow.Diagnostics/SensitiveFlow.Diagnostics.csproj +++ b/src/SensitiveFlow.Diagnostics/SensitiveFlow.Diagnostics.csproj @@ -6,6 +6,7 @@ + diff --git a/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationDiagnostic.cs b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationDiagnostic.cs new file mode 100644 index 0000000..da854ae --- /dev/null +++ b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationDiagnostic.cs @@ -0,0 +1,17 @@ +namespace SensitiveFlow.Diagnostics.Validation; + +/// +/// One SensitiveFlow configuration diagnostic. +/// +public sealed record SensitiveFlowConfigurationDiagnostic +{ + /// Gets the diagnostic code. + public required string Code { get; init; } + + /// Gets the diagnostic message. + public required string Message { get; init; } + + /// Gets the diagnostic severity. + public SensitiveFlowDiagnosticSeverity Severity { get; init; } = SensitiveFlowDiagnosticSeverity.Warning; +} + diff --git a/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationReport.cs b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationReport.cs new file mode 100644 index 0000000..6a60a7c --- /dev/null +++ b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationReport.cs @@ -0,0 +1,21 @@ +namespace SensitiveFlow.Diagnostics.Validation; + +/// +/// Result of SensitiveFlow startup configuration validation. +/// +public sealed class SensitiveFlowConfigurationReport +{ + /// Initializes a new report. + public SensitiveFlowConfigurationReport(IEnumerable diagnostics) + { + ArgumentNullException.ThrowIfNull(diagnostics); + Diagnostics = diagnostics.ToArray(); + } + + /// Gets the diagnostics. + public IReadOnlyList Diagnostics { get; } + + /// Gets whether any error diagnostics were produced. + public bool IsValid => !Diagnostics.Any(static d => d.Severity == SensitiveFlowDiagnosticSeverity.Error); +} + diff --git a/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs new file mode 100644 index 0000000..a0b844b --- /dev/null +++ b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs @@ -0,0 +1,66 @@ +using Microsoft.Extensions.DependencyInjection; +using SensitiveFlow.Core.Interfaces; + +namespace SensitiveFlow.Diagnostics.Validation; + +/// +/// Validates SensitiveFlow service registrations at startup. +/// +public sealed class SensitiveFlowConfigurationValidator +{ + private readonly SensitiveFlowValidationOptions _options; + + /// Initializes a new instance. + public SensitiveFlowConfigurationValidator(SensitiveFlowValidationOptions options) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + } + + /// Validates a built service provider. + public SensitiveFlowConfigurationReport Validate(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + + var diagnostics = new List(); + using var scope = services.CreateScope(); + var provider = scope.ServiceProvider; + + if (_options.RequireAuditStore && provider.GetService() is null) + { + diagnostics.Add(Warning("SF-CONFIG-001", "No IAuditStore registration was found.")); + } + + if (_options.RequireTokenStore && provider.GetService() is null) + { + diagnostics.Add(Warning("SF-CONFIG-002", "No durable ITokenStore registration was found.")); + } + + if (provider.GetService() is not null && provider.GetService() is null) + { + diagnostics.Add(Warning("SF-CONFIG-003", "IPseudonymizer is registered without an ITokenStore; reversible pseudonymization may not be durable.")); + } + + if (_options.RequireJsonRedaction && services.GetService(typeof(Microsoft.Extensions.Options.IOptions<>).MakeGenericType(Type.GetType("SensitiveFlow.Json.Configuration.JsonRedactionOptions, SensitiveFlow.Json") ?? typeof(object))) is null) + { + diagnostics.Add(Warning("SF-CONFIG-004", "JSON redaction is required but JsonRedactionOptions were not found.")); + } + + if (_options.RequireRetention && !services.GetServices().Any(static s => s.GetType().FullName?.Contains("Retention", StringComparison.Ordinal) == true)) + { + diagnostics.Add(Warning("SF-CONFIG-005", "Retention validation was requested, but no retention services were detected.")); + } + + return new SensitiveFlowConfigurationReport(diagnostics); + } + + private static SensitiveFlowConfigurationDiagnostic Warning(string code, string message) + { + return new SensitiveFlowConfigurationDiagnostic + { + Code = code, + Message = message, + Severity = SensitiveFlowDiagnosticSeverity.Warning, + }; + } +} + diff --git a/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowDiagnosticSeverity.cs b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowDiagnosticSeverity.cs new file mode 100644 index 0000000..2639f49 --- /dev/null +++ b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowDiagnosticSeverity.cs @@ -0,0 +1,17 @@ +namespace SensitiveFlow.Diagnostics.Validation; + +/// +/// Severity for configuration diagnostics. +/// +public enum SensitiveFlowDiagnosticSeverity +{ + /// Informational diagnostic. + Info = 0, + + /// Warning diagnostic. + Warning, + + /// Error diagnostic. + Error, +} + diff --git a/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowValidationOptions.cs b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowValidationOptions.cs new file mode 100644 index 0000000..b2d62c1 --- /dev/null +++ b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowValidationOptions.cs @@ -0,0 +1,20 @@ +namespace SensitiveFlow.Diagnostics.Validation; + +/// +/// Options controlling startup validation expectations. +/// +public sealed class SensitiveFlowValidationOptions +{ + /// Gets or sets whether a durable audit store is required. + public bool RequireAuditStore { get; set; } + + /// Gets or sets whether a token store is required. + public bool RequireTokenStore { get; set; } + + /// Gets or sets whether JSON redaction configuration is required. + public bool RequireJsonRedaction { get; set; } + + /// Gets or sets whether retention services are required. + public bool RequireRetention { get; set; } +} + diff --git a/src/SensitiveFlow.HealthChecks/Checks/AuditStoreHealthCheck.cs b/src/SensitiveFlow.HealthChecks/Checks/AuditStoreHealthCheck.cs new file mode 100644 index 0000000..0da5459 --- /dev/null +++ b/src/SensitiveFlow.HealthChecks/Checks/AuditStoreHealthCheck.cs @@ -0,0 +1,43 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using SensitiveFlow.Core.Interfaces; + +namespace SensitiveFlow.HealthChecks.Checks; + +/// +/// Health check for . +/// +public sealed class AuditStoreHealthCheck : IHealthCheck +{ + private readonly IAuditStore _store; + + /// Initializes a new instance. + public AuditStoreHealthCheck(IAuditStore store) + { + _store = store ?? throw new ArgumentNullException(nameof(store)); + } + + /// + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + try + { + if (_store is IHealthProbe probe) + { + await probe.ProbeAsync(cancellationToken); + } + else + { + await _store.QueryAsync(take: 1, cancellationToken: cancellationToken); + } + + return HealthCheckResult.Healthy("Audit store responded."); + } + catch (Exception ex) + { + return HealthCheckResult.Unhealthy("Audit store probe failed.", ex); + } + } +} + diff --git a/src/SensitiveFlow.HealthChecks/Checks/TokenStoreHealthCheck.cs b/src/SensitiveFlow.HealthChecks/Checks/TokenStoreHealthCheck.cs new file mode 100644 index 0000000..fb12313 --- /dev/null +++ b/src/SensitiveFlow.HealthChecks/Checks/TokenStoreHealthCheck.cs @@ -0,0 +1,39 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using SensitiveFlow.Core.Interfaces; + +namespace SensitiveFlow.HealthChecks.Checks; + +/// +/// Health check for . +/// +public sealed class TokenStoreHealthCheck : IHealthCheck +{ + private readonly ITokenStore _store; + + /// Initializes a new instance. + public TokenStoreHealthCheck(ITokenStore store) + { + _store = store ?? throw new ArgumentNullException(nameof(store)); + } + + /// + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + try + { + if (_store is IHealthProbe probe) + { + await probe.ProbeAsync(cancellationToken); + } + + return HealthCheckResult.Healthy("Token store resolved."); + } + catch (Exception ex) + { + return HealthCheckResult.Unhealthy("Token store probe failed.", ex); + } + } +} + diff --git a/src/SensitiveFlow.HealthChecks/Extensions/SensitiveFlowHealthCheckBuilderExtensions.cs b/src/SensitiveFlow.HealthChecks/Extensions/SensitiveFlowHealthCheckBuilderExtensions.cs new file mode 100644 index 0000000..db73f3c --- /dev/null +++ b/src/SensitiveFlow.HealthChecks/Extensions/SensitiveFlowHealthCheckBuilderExtensions.cs @@ -0,0 +1,40 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using SensitiveFlow.HealthChecks.Checks; + +namespace SensitiveFlow.HealthChecks.Extensions; + +/// +/// Health check builder extensions for SensitiveFlow infrastructure. +/// +public static class SensitiveFlowHealthCheckBuilderExtensions +{ + /// Returns the health check builder for fluent SensitiveFlow registration. + public static IHealthChecksBuilder AddSensitiveFlowHealthChecks(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + return services.AddHealthChecks(); + } + + /// Adds an audit store health check. + public static IHealthChecksBuilder AddAuditStoreCheck( + this IHealthChecksBuilder builder, + string name = "sensitiveflow-audit-store", + HealthStatus? failureStatus = null, + IEnumerable? tags = null) + { + ArgumentNullException.ThrowIfNull(builder); + return builder.AddCheck(name, failureStatus, tags ?? Array.Empty()); + } + + /// Adds a token store health check. + public static IHealthChecksBuilder AddTokenStoreCheck( + this IHealthChecksBuilder builder, + string name = "sensitiveflow-token-store", + HealthStatus? failureStatus = null, + IEnumerable? tags = null) + { + ArgumentNullException.ThrowIfNull(builder); + return builder.AddCheck(name, failureStatus, tags ?? Array.Empty()); + } +} diff --git a/src/SensitiveFlow.HealthChecks/SensitiveFlow.HealthChecks.csproj b/src/SensitiveFlow.HealthChecks/SensitiveFlow.HealthChecks.csproj new file mode 100644 index 0000000..d40b0e1 --- /dev/null +++ b/src/SensitiveFlow.HealthChecks/SensitiveFlow.HealthChecks.csproj @@ -0,0 +1,13 @@ + + + + Health check integrations for SensitiveFlow audit and token infrastructure. + + + + + + + + + diff --git a/src/SensitiveFlow.Retention/Services/RetentionEvaluator.cs b/src/SensitiveFlow.Retention/Services/RetentionEvaluator.cs index f60636b..8585f0a 100644 --- a/src/SensitiveFlow.Retention/Services/RetentionEvaluator.cs +++ b/src/SensitiveFlow.Retention/Services/RetentionEvaluator.cs @@ -26,13 +26,21 @@ public sealed class RetentionEvaluator ]; private readonly IEnumerable _handlers; + private readonly TimeProvider _timeProvider; /// /// Initializes a new instance of . /// public RetentionEvaluator(IEnumerable handlers) + : this(handlers, TimeProvider.System) { } + + /// + /// Initializes a new instance of with a custom . + /// + public RetentionEvaluator(IEnumerable handlers, TimeProvider timeProvider) { _handlers = handlers; + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); } /// @@ -75,7 +83,7 @@ private async Task EvaluateRecursiveAsync(object entity, DateTimeOffset referenc foreach (var pair in retentionProperties) { var expiration = pair.Attribute.GetExpirationDate(referenceDate); - if (DateTimeOffset.UtcNow <= expiration) + if (_timeProvider.GetUtcNow() <= expiration) { continue; } diff --git a/src/SensitiveFlow.Retention/Services/RetentionExecutor.cs b/src/SensitiveFlow.Retention/Services/RetentionExecutor.cs index ceeecc8..9866b52 100644 --- a/src/SensitiveFlow.Retention/Services/RetentionExecutor.cs +++ b/src/SensitiveFlow.Retention/Services/RetentionExecutor.cs @@ -41,15 +41,21 @@ public sealed class RetentionExecutor ]; private readonly RetentionExecutorOptions _options; + private readonly TimeProvider _timeProvider; - /// Initializes a new instance with default options. - public RetentionExecutor() : this(new RetentionExecutorOptions()) { } + /// Initializes a new instance with default options and system clock. + public RetentionExecutor() : this(new RetentionExecutorOptions(), TimeProvider.System) { } /// Initializes a new instance. public RetentionExecutor(RetentionExecutorOptions options) + : this(options, TimeProvider.System) { } + + /// Initializes a new instance with a custom . + public RetentionExecutor(RetentionExecutorOptions options, TimeProvider timeProvider) { ArgumentNullException.ThrowIfNull(options); _options = options; + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); } /// @@ -65,12 +71,36 @@ public Task ExecuteAsync( IEnumerable entities, Func referenceDateSelector, CancellationToken cancellationToken = default) + { + return ExecuteCoreAsync(entities, referenceDateSelector, mutate: true, cancellationToken); + } + + /// + /// Evaluates retention policies and returns the actions that would be taken without mutating entities. + /// + /// Entities to evaluate. + /// Returns the retention start date for each entity. + /// Cancellation token. + /// An aggregated report describing pending actions. + public Task DryRunAsync( + IEnumerable entities, + Func referenceDateSelector, + CancellationToken cancellationToken = default) + { + return ExecuteCoreAsync(entities, referenceDateSelector, mutate: false, cancellationToken); + } + + private Task ExecuteCoreAsync( + IEnumerable entities, + Func referenceDateSelector, + bool mutate, + CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(entities); ArgumentNullException.ThrowIfNull(referenceDateSelector); var report = new RetentionExecutionReport(); - var now = DateTimeOffset.UtcNow; + var now = _timeProvider.GetUtcNow(); foreach (var entity in entities) { @@ -81,13 +111,18 @@ public Task ExecuteAsync( } var referenceDate = referenceDateSelector(entity); - ProcessEntityRecursive(entity, referenceDate, now, report); + ProcessEntityRecursive(entity, referenceDate, now, report, mutate); } return Task.FromResult(report); } - private void ProcessEntityRecursive(object entity, DateTimeOffset referenceDate, DateTimeOffset now, RetentionExecutionReport report) + private void ProcessEntityRecursive( + object entity, + DateTimeOffset referenceDate, + DateTimeOffset now, + RetentionExecutionReport report, + bool mutate) { var retentionProperties = SensitiveMemberCache.GetRetentionProperties(entity.GetType()); @@ -101,7 +136,7 @@ private void ProcessEntityRecursive(object entity, DateTimeOffset referenceDate, var action = pair.Attribute.Policy switch { - RetentionPolicy.AnonymizeOnExpiration => Anonymize(entity, pair.Property), + RetentionPolicy.AnonymizeOnExpiration => mutate ? Anonymize(entity, pair.Property) : RetentionAction.Anonymized, RetentionPolicy.DeleteOnExpiration => RetentionAction.DeletePending, RetentionPolicy.BlockOnExpiration => RetentionAction.Blocked, RetentionPolicy.NotifyOwner => RetentionAction.NotifyPending, @@ -120,7 +155,7 @@ private void ProcessEntityRecursive(object entity, DateTimeOffset referenceDate, continue; } - ProcessEntityRecursive(value, referenceDate, now, report); + ProcessEntityRecursive(value, referenceDate, now, report, mutate); } } diff --git a/src/SensitiveFlow.TestKit/AnonymizerContractTests.cs b/src/SensitiveFlow.TestKit/AnonymizerContractTests.cs new file mode 100644 index 0000000..992664c --- /dev/null +++ b/src/SensitiveFlow.TestKit/AnonymizerContractTests.cs @@ -0,0 +1,28 @@ +using FluentAssertions; +using SensitiveFlow.Core.Interfaces; +using Xunit; + +namespace SensitiveFlow.TestKit; + +/// +/// Conformance suite for implementations. +/// +public abstract class AnonymizerContractTests +{ + /// Creates an anonymizer instance for the test. + protected abstract IAnonymizer CreateAnonymizer(); + + /// Verifies that supported values are changed. + [Fact] + public void Anonymize_SupportedValue_DoesNotReturnRawValue() + { + var anonymizer = CreateAnonymizer(); + const string value = "12345678909"; + + if (anonymizer.CanAnonymize(value)) + { + anonymizer.Anonymize(value).Should().NotBe(value); + } + } +} + diff --git a/src/SensitiveFlow.TestKit/Assertions/SensitiveDataAssert.cs b/src/SensitiveFlow.TestKit/Assertions/SensitiveDataAssert.cs index 1967241..b420657 100644 --- a/src/SensitiveFlow.TestKit/Assertions/SensitiveDataAssert.cs +++ b/src/SensitiveFlow.TestKit/Assertions/SensitiveDataAssert.cs @@ -1,4 +1,5 @@ using SensitiveFlow.Core.Reflection; +using System.Text.Json; using Xunit.Sdk; namespace SensitiveFlow.TestKit.Assertions; @@ -125,4 +126,64 @@ public static void DoesNotLeakKnownValues(string payload, IEnumerable kn ArgumentNullException.ThrowIfNull(knownValues); DoesNotContainAny(payload, knownValues.ToArray()); } + + /// + /// Fails the test if does not contain an email-shaped masked value. + /// + public static void ContainsMaskedEmail(string payload) + { + ArgumentNullException.ThrowIfNull(payload); + + if (!System.Text.RegularExpressions.Regex.IsMatch( + payload, + @"[A-Za-z0-9]\*+@[^@\s]+\.[^@\s]+", + System.Text.RegularExpressions.RegexOptions.CultureInvariant)) + { + throw new XunitException("Payload did not contain a masked email value."); + } + } + + /// + /// Fails the test if any annotated string value from appears verbatim in . + /// + public static void DoesNotContainRawValues(string payload, object entity) + { + DoesNotLeak(payload, entity); + } + + /// + /// Fails the test if JSON exposes a property annotated as sensitive on . + /// + public static void JsonDoesNotExposeAnnotatedProperties(string json, Type annotatedType) + { + ArgumentNullException.ThrowIfNull(json); + ArgumentNullException.ThrowIfNull(annotatedType); + + using var document = JsonDocument.Parse(json); + if (document.RootElement.ValueKind != JsonValueKind.Object) + { + return; + } + + var exposed = SensitiveMemberCache.GetSensitiveProperties(annotatedType) + .Select(static p => p.Name) + .Where(name => document.RootElement.TryGetProperty(name, out _)) + .ToArray(); + + if (exposed.Length > 0) + { + throw new XunitException( + "JSON exposed annotated sensitive properties:" + + Environment.NewLine + + string.Join(Environment.NewLine, exposed.Select(static p => " - " + p))); + } + } + + /// + /// Fails the test if any known sensitive value appears in log output. + /// + public static void LogsDoNotContainSensitiveValues(string logs, IEnumerable values) + { + DoesNotLeakKnownValues(logs, values); + } } diff --git a/src/SensitiveFlow.TestKit/AuditSnapshotStoreContractTests.cs b/src/SensitiveFlow.TestKit/AuditSnapshotStoreContractTests.cs new file mode 100644 index 0000000..08bb8d7 --- /dev/null +++ b/src/SensitiveFlow.TestKit/AuditSnapshotStoreContractTests.cs @@ -0,0 +1,68 @@ +using FluentAssertions; +using SensitiveFlow.Core.Interfaces; +using SensitiveFlow.Core.Models; +using Xunit; + +namespace SensitiveFlow.TestKit; + +/// +/// Conformance suite for implementations. +/// +public abstract class AuditSnapshotStoreContractTests +{ + /// Creates a fresh store instance for the test. + protected abstract Task CreateStoreAsync(); + + /// Cleanup hook for stores that need teardown. + protected virtual Task DisposeStoreAsync(IAuditSnapshotStore store) => Task.CompletedTask; + + /// Verifies append and aggregate query behavior. + [Fact] + public async Task AppendAsync_ThenQueryByAggregate_ReturnsSnapshot() + { + var store = await CreateStoreAsync(); + try + { + var snapshot = SampleSnapshot(); + await store.AppendAsync(snapshot); + + var page = await store.QueryByAggregateAsync("Customer", "42"); + page.Should().ContainSingle(s => s.Id == snapshot.Id); + } + finally + { + await DisposeStoreAsync(store); + } + } + + /// Verifies data-subject filtering behavior. + [Fact] + public async Task QueryByDataSubject_FiltersBySubject() + { + var store = await CreateStoreAsync(); + try + { + await store.AppendAsync(SampleSnapshot(subject: "alice")); + await store.AppendAsync(SampleSnapshot(subject: "bob", aggregateId: "43")); + + var alice = await store.QueryByDataSubjectAsync("alice"); + alice.Should().OnlyContain(s => s.DataSubjectId == "alice"); + } + finally + { + await DisposeStoreAsync(store); + } + } + + private static AuditSnapshot SampleSnapshot(string subject = "alice", string aggregateId = "42") + { + return new AuditSnapshot + { + DataSubjectId = subject, + Aggregate = "Customer", + AggregateId = aggregateId, + BeforeJson = "{\"Email\":\"old\"}", + AfterJson = "{\"Email\":\"[REDACTED]\"}", + }; + } +} diff --git a/src/SensitiveFlow.TestKit/MaskerContractTests.cs b/src/SensitiveFlow.TestKit/MaskerContractTests.cs new file mode 100644 index 0000000..16e2b2d --- /dev/null +++ b/src/SensitiveFlow.TestKit/MaskerContractTests.cs @@ -0,0 +1,28 @@ +using FluentAssertions; +using SensitiveFlow.Core.Interfaces; +using Xunit; + +namespace SensitiveFlow.TestKit; + +/// +/// Conformance suite for implementations. +/// +public abstract class MaskerContractTests +{ + /// Creates a masker instance for the test. + protected abstract IMasker CreateMasker(); + + /// Verifies that supported values are changed. + [Fact] + public void Mask_SupportedValue_DoesNotReturnRawValue() + { + var masker = CreateMasker(); + const string value = "alice@example.com"; + + if (masker.CanMask(value)) + { + masker.Mask(value).Should().NotBe(value); + } + } +} + diff --git a/src/SensitiveFlow.TestKit/PseudonymizerContractTests.cs b/src/SensitiveFlow.TestKit/PseudonymizerContractTests.cs new file mode 100644 index 0000000..d41652d --- /dev/null +++ b/src/SensitiveFlow.TestKit/PseudonymizerContractTests.cs @@ -0,0 +1,36 @@ +using FluentAssertions; +using SensitiveFlow.Core.Interfaces; +using Xunit; + +namespace SensitiveFlow.TestKit; + +/// +/// Conformance suite for implementations. +/// +public abstract class PseudonymizerContractTests +{ + /// Creates a pseudonymizer instance for the test. + protected abstract Task CreatePseudonymizerAsync(); + + /// Verifies stable reversible pseudonymization. + [Fact] + public async Task PseudonymizeAsync_ReturnsStableReversibleValue() + { + var pseudonymizer = await CreatePseudonymizerAsync(); + var first = await pseudonymizer.PseudonymizeAsync("alice@example.com"); + var second = await pseudonymizer.PseudonymizeAsync("alice@example.com"); + + first.Should().Be(second); + var original = await pseudonymizer.ReverseAsync(first); + original.Should().Be("alice@example.com"); + } + + /// Verifies capability checks for valid values. + [Fact] + public async Task CanPseudonymize_ReturnsTrueForNonEmptyValue() + { + var pseudonymizer = await CreatePseudonymizerAsync(); + pseudonymizer.CanPseudonymize("value").Should().BeTrue(); + } +} + diff --git a/src/SensitiveFlow.TestKit/RetentionExpirationHandlerContractTests.cs b/src/SensitiveFlow.TestKit/RetentionExpirationHandlerContractTests.cs new file mode 100644 index 0000000..664723c --- /dev/null +++ b/src/SensitiveFlow.TestKit/RetentionExpirationHandlerContractTests.cs @@ -0,0 +1,25 @@ +using FluentAssertions; +using SensitiveFlow.Retention.Contracts; +using Xunit; + +namespace SensitiveFlow.TestKit; + +/// +/// Conformance suite for implementations. +/// +public abstract class RetentionExpirationHandlerContractTests +{ + /// Creates a handler instance for the test. + protected abstract IRetentionExpirationHandler CreateHandler(); + + /// Verifies the handler accepts a minimal expired-field notification. + [Fact] + public async Task HandleAsync_AcceptsExpiredField() + { + var handler = CreateHandler(); + + Func act = () => handler.HandleAsync(new { Id = "customer-1" }, "Email", DateTimeOffset.UtcNow.AddDays(-1)); + + await act.Should().NotThrowAsync(); + } +} diff --git a/src/SensitiveFlow.TestKit/SensitiveFlow.TestKit.csproj b/src/SensitiveFlow.TestKit/SensitiveFlow.TestKit.csproj index 8d85cda..3f88d77 100644 --- a/src/SensitiveFlow.TestKit/SensitiveFlow.TestKit.csproj +++ b/src/SensitiveFlow.TestKit/SensitiveFlow.TestKit.csproj @@ -11,6 +11,7 @@ + diff --git a/src/SensitiveFlow.Tool/Program.cs b/src/SensitiveFlow.Tool/Program.cs new file mode 100644 index 0000000..ce89ffd --- /dev/null +++ b/src/SensitiveFlow.Tool/Program.cs @@ -0,0 +1,3 @@ +using SensitiveFlow.Tool; + +return SensitiveFlowToolRunner.Run(args, Console.Out, Console.Error); diff --git a/src/SensitiveFlow.Tool/SensitiveFlow.Tool.csproj b/src/SensitiveFlow.Tool/SensitiveFlow.Tool.csproj new file mode 100644 index 0000000..2603b77 --- /dev/null +++ b/src/SensitiveFlow.Tool/SensitiveFlow.Tool.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + Exe + true + sensitiveflow + Command-line reporting tool for SensitiveFlow annotated models. + + + + + + + + diff --git a/src/SensitiveFlow.Tool/SensitiveFlowToolRunner.cs b/src/SensitiveFlow.Tool/SensitiveFlowToolRunner.cs new file mode 100644 index 0000000..b6baf48 --- /dev/null +++ b/src/SensitiveFlow.Tool/SensitiveFlowToolRunner.cs @@ -0,0 +1,49 @@ +using System.Reflection; +using SensitiveFlow.Core.Discovery; + +namespace SensitiveFlow.Tool; + +/// +/// Command runner for the SensitiveFlow CLI. +/// +public static class SensitiveFlowToolRunner +{ + /// Runs the CLI command. + public static int Run(string[] args, TextWriter output, TextWriter error) + { + ArgumentNullException.ThrowIfNull(args); + ArgumentNullException.ThrowIfNull(output); + ArgumentNullException.ThrowIfNull(error); + + if (args.Length < 2 || !string.Equals(args[0], "scan", StringComparison.OrdinalIgnoreCase)) + { + error.WriteLine("Usage: sensitiveflow scan [output-directory]"); + return 2; + } + + var assemblyPath = Path.GetFullPath(args[1]); + if (!File.Exists(assemblyPath)) + { + error.WriteLine($"Assembly not found: {assemblyPath}"); + return 3; + } + + var outputDirectory = args.Length >= 3 + ? Path.GetFullPath(args[2]) + : Directory.GetCurrentDirectory(); + Directory.CreateDirectory(outputDirectory); + + var assembly = Assembly.LoadFrom(assemblyPath); + var report = SensitiveDataDiscovery.Scan(assembly); + + var jsonPath = Path.Combine(outputDirectory, "sensitiveflow-report.json"); + var markdownPath = Path.Combine(outputDirectory, "sensitiveflow-report.md"); + + File.WriteAllText(jsonPath, report.ToJson()); + File.WriteAllText(markdownPath, report.ToMarkdown()); + + output.WriteLine($"SensitiveFlow report written: {jsonPath}"); + output.WriteLine($"SensitiveFlow report written: {markdownPath}"); + return 0; + } +} diff --git a/tests/SensitiveFlow.Core.Tests/CoreContractTests.cs b/tests/SensitiveFlow.Core.Tests/CoreContractTests.cs index 991b983..06b6476 100644 --- a/tests/SensitiveFlow.Core.Tests/CoreContractTests.cs +++ b/tests/SensitiveFlow.Core.Tests/CoreContractTests.cs @@ -1,10 +1,14 @@ using FluentAssertions; using NSubstitute; using SensitiveFlow.Core.Attributes; +using SensitiveFlow.Core.Discovery; using SensitiveFlow.Core.Enums; using SensitiveFlow.Core.Exceptions; +using SensitiveFlow.Core.Export; using SensitiveFlow.Core.Interfaces; using SensitiveFlow.Core.Models; +using SensitiveFlow.Core.Policies; +using SensitiveFlow.Core.Profiles; using Xunit; namespace SensitiveFlow.Core.Tests; @@ -23,6 +27,7 @@ public void SensitiveDataAttribute_DefaultsToOtherCategory() { var attribute = new SensitiveDataAttribute(); attribute.Category.Should().Be(SensitiveDataCategory.Other); + attribute.Sensitivity.Should().Be(DataSensitivity.High); } [Fact] @@ -209,4 +214,61 @@ public void IAuditContext_Contract_ExposesActorAndIpToken() ctx.ActorId.Should().Be("actor-1"); ctx.IpAddressToken.Should().Be("token-x"); } + + [Fact] + public void SensitiveFlowOptions_StrictProfile_RequiresAuditForSensitiveCategories() + { + var options = new SensitiveFlowOptions().UseProfile(SensitiveFlowProfile.Strict); + + var rule = options.Policies.Find(SensitiveDataCategory.Health); + + rule.Should().NotBeNull(); + (rule!.Actions & SensitiveFlowPolicyAction.OmitInJson).Should().Be(SensitiveFlowPolicyAction.OmitInJson); + (rule.Actions & SensitiveFlowPolicyAction.RequireAudit).Should().Be(SensitiveFlowPolicyAction.RequireAudit); + } + + [Fact] + public void SensitiveDataDiscovery_Scan_ReturnsAnnotatedMembers() + { + var report = SensitiveDataDiscovery.Scan(typeof(CoreContractTests).Assembly); + + report.Entries.Should().Contain(e => e.TypeName == nameof(DiscoveryCustomer) + && e.MemberName == nameof(DiscoveryCustomer.Email) + && e.Category == DataCategory.Contact); + report.ToMarkdown().Should().Contain("DiscoveryCustomer.Email"); + report.ToJson().Should().Contain("DiscoveryCustomer"); + } + + [Fact] + public void SensitiveFlowExceptionSanitizer_RemovesCommonRawValues() + { + var exception = new SensitiveFlowConfigurationException( + "Failed for alice@example.com and 123.456.789-09.", + "SF_CONFIG_001"); + + var sanitized = SensitiveFlowExceptionSanitizer.Sanitize(exception); + + sanitized.Code.Should().Be("SF_CONFIG_001"); + sanitized.Message.Should().NotContain("alice@example.com"); + sanitized.Message.Should().NotContain("123.456.789-09"); + } + + [Fact] + public void CsvDataExportFormatter_PrefixesFormulaValues() + { + var formatter = new CsvDataExportFormatter(); + + var csv = formatter.Format([ + new Dictionary { ["Name"] = "=cmd" }, + ]); + + csv.Should().Contain("\"'=cmd\""); + } + + private sealed class DiscoveryCustomer + { + [PersonalData(Category = DataCategory.Contact)] + [RetentionData(Years = 5, Policy = RetentionPolicy.AnonymizeOnExpiration)] + public string Email { get; set; } = string.Empty; + } } diff --git a/tests/SensitiveFlow.Core.Tests/SensitiveFlow.Core.Tests.csproj b/tests/SensitiveFlow.Core.Tests/SensitiveFlow.Core.Tests.csproj index 1aa10db..8ba3e43 100644 --- a/tests/SensitiveFlow.Core.Tests/SensitiveFlow.Core.Tests.csproj +++ b/tests/SensitiveFlow.Core.Tests/SensitiveFlow.Core.Tests.csproj @@ -7,5 +7,6 @@ + - \ No newline at end of file + diff --git a/tests/SensitiveFlow.Core.Tests/SensitiveFlowToolRunnerTests.cs b/tests/SensitiveFlow.Core.Tests/SensitiveFlowToolRunnerTests.cs new file mode 100644 index 0000000..6122c96 --- /dev/null +++ b/tests/SensitiveFlow.Core.Tests/SensitiveFlowToolRunnerTests.cs @@ -0,0 +1,37 @@ +#if NET10_0_OR_GREATER +using FluentAssertions; +using SensitiveFlow.Tool; + +namespace SensitiveFlow.Core.Tests; + +public sealed class SensitiveFlowToolRunnerTests +{ + [Fact] + public void Run_Scan_WritesJsonAndMarkdownReports() + { + var outputDirectory = Path.Combine(Path.GetTempPath(), "sensitiveflow-tool-tests", Guid.NewGuid().ToString("N")); + var assemblyPath = typeof(SensitiveFlowToolRunnerTests).Assembly.Location; + using var output = new StringWriter(); + using var error = new StringWriter(); + + var exitCode = SensitiveFlowToolRunner.Run(["scan", assemblyPath, outputDirectory], output, error); + + exitCode.Should().Be(0); + File.Exists(Path.Combine(outputDirectory, "sensitiveflow-report.json")).Should().BeTrue(); + File.Exists(Path.Combine(outputDirectory, "sensitiveflow-report.md")).Should().BeTrue(); + error.ToString().Should().BeEmpty(); + } + + [Fact] + public void Run_WithoutScanCommand_ReturnsUsageError() + { + using var output = new StringWriter(); + using var error = new StringWriter(); + + var exitCode = SensitiveFlowToolRunner.Run([], output, error); + + exitCode.Should().Be(2); + error.ToString().Should().Contain("Usage:"); + } +} +#endif diff --git a/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs b/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs new file mode 100644 index 0000000..f1695ed --- /dev/null +++ b/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs @@ -0,0 +1,52 @@ +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using SensitiveFlow.Core.Interfaces; +using SensitiveFlow.Core.Models; +using SensitiveFlow.Diagnostics.Extensions; +using SensitiveFlow.Diagnostics.Validation; + +namespace SensitiveFlow.Diagnostics.Tests; + +public sealed class SensitiveFlowValidationTests +{ + [Fact] + public void ValidateSensitiveFlow_WhenAuditStoreRequiredAndMissing_ReportsWarning() + { + var services = new ServiceCollection(); + services.AddSensitiveFlowValidation(o => o.RequireAuditStore = true); + + var report = services.BuildServiceProvider().ValidateSensitiveFlow(); + + report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-001"); + } + + [Fact] + public void ValidateSensitiveFlow_WhenRequiredStoresExist_ReturnsNoStoreWarnings() + { + var services = new ServiceCollection(); + services.AddSingleton(); + services.AddSensitiveFlowValidation(o => o.RequireAuditStore = true); + + var report = services.BuildServiceProvider().ValidateSensitiveFlow(); + + report.Diagnostics.Should().NotContain(d => d.Code == "SF-CONFIG-001"); + } + + private sealed class HealthyAuditStore : IAuditStore + { + public Task AppendAsync(AuditRecord record, CancellationToken cancellationToken = default) + { + return Task.CompletedTask; + } + + public Task> QueryAsync(DateTimeOffset? from = null, DateTimeOffset? to = null, int skip = 0, int take = 100, CancellationToken cancellationToken = default) + { + return Task.FromResult>([]); + } + + public Task> QueryByDataSubjectAsync(string dataSubjectId, DateTimeOffset? from = null, DateTimeOffset? to = null, int skip = 0, int take = 100, CancellationToken cancellationToken = default) + { + return Task.FromResult>([]); + } + } +} diff --git a/tests/SensitiveFlow.HealthChecks.Tests/SensitiveFlow.HealthChecks.Tests.csproj b/tests/SensitiveFlow.HealthChecks.Tests/SensitiveFlow.HealthChecks.Tests.csproj new file mode 100644 index 0000000..3187b05 --- /dev/null +++ b/tests/SensitiveFlow.HealthChecks.Tests/SensitiveFlow.HealthChecks.Tests.csproj @@ -0,0 +1,13 @@ + + + false + false + + + + + + + + + diff --git a/tests/SensitiveFlow.HealthChecks.Tests/SensitiveFlowHealthChecksTests.cs b/tests/SensitiveFlow.HealthChecks.Tests/SensitiveFlowHealthChecksTests.cs new file mode 100644 index 0000000..47a8086 --- /dev/null +++ b/tests/SensitiveFlow.HealthChecks.Tests/SensitiveFlowHealthChecksTests.cs @@ -0,0 +1,79 @@ +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; +using SensitiveFlow.Core.Interfaces; +using SensitiveFlow.Core.Models; +using SensitiveFlow.HealthChecks.Extensions; + +namespace SensitiveFlow.HealthChecks.Tests; + +public sealed class SensitiveFlowHealthChecksTests +{ + [Fact] + public void AddSensitiveFlowHealthChecks_RegistersChecks() + { + var services = new ServiceCollection(); + services.AddSingleton(); + services.AddSingleton(); + services.AddLogging(); + + services.AddSensitiveFlowHealthChecks() + .AddAuditStoreCheck() + .AddTokenStoreCheck(); + + var provider = services.BuildServiceProvider(); + var checks = provider.GetRequiredService>().Value.Registrations; + + checks.Should().Contain(r => r.Name == "sensitiveflow-audit-store"); + checks.Should().Contain(r => r.Name == "sensitiveflow-token-store"); + } + + [Fact] + public async Task AddSensitiveFlowHealthChecks_RegisteredChecksRunHealthy() + { + var services = new ServiceCollection(); + services.AddSingleton(); + services.AddSingleton(); + services.AddLogging(); + services.AddSensitiveFlowHealthChecks() + .AddAuditStoreCheck() + .AddTokenStoreCheck(); + + var provider = services.BuildServiceProvider(); + var result = await provider.GetRequiredService().CheckHealthAsync(); + + result.Status.Should().Be(HealthStatus.Healthy); + } + + private sealed class HealthyAuditStore : IAuditStore + { + public Task AppendAsync(AuditRecord record, CancellationToken cancellationToken = default) + { + return Task.CompletedTask; + } + + public Task> QueryAsync(DateTimeOffset? from = null, DateTimeOffset? to = null, int skip = 0, int take = 100, CancellationToken cancellationToken = default) + { + return Task.FromResult>([]); + } + + public Task> QueryByDataSubjectAsync(string dataSubjectId, DateTimeOffset? from = null, DateTimeOffset? to = null, int skip = 0, int take = 100, CancellationToken cancellationToken = default) + { + return Task.FromResult>([]); + } + } + + private sealed class HealthyTokenStore : ITokenStore + { + public Task GetOrCreateTokenAsync(string value, CancellationToken cancellationToken = default) + { + return Task.FromResult("token"); + } + + public Task ResolveTokenAsync(string token, CancellationToken cancellationToken = default) + { + return Task.FromResult("value"); + } + } +} diff --git a/tests/SensitiveFlow.Retention.Tests/RetentionExecutorTests.cs b/tests/SensitiveFlow.Retention.Tests/RetentionExecutorTests.cs index b35d338..7d84e56 100644 --- a/tests/SensitiveFlow.Retention.Tests/RetentionExecutorTests.cs +++ b/tests/SensitiveFlow.Retention.Tests/RetentionExecutorTests.cs @@ -229,4 +229,17 @@ public async Task ExecuteAsync_UnknownPolicy_ReportsNone() report.Entries.Should().ContainSingle(e => e.FieldName == nameof(UnknownPolicyEntity.Secret) && e.Action == RetentionAction.None); } + + [Fact] + public async Task DryRunAsync_ReportsActionsWithoutMutatingEntity() + { + var entity = new AnonymizeEntity { CreatedAt = DateTimeOffset.UtcNow.AddYears(-2) }; + var executor = new RetentionExecutor(); + + var report = await executor.DryRunAsync([entity], e => ((AnonymizeEntity)e).CreatedAt); + + entity.Email.Should().Be("alice@example.com"); + entity.Score.Should().Be(42); + report.AnonymizedFieldCount.Should().Be(2); + } } diff --git a/tests/SensitiveFlow.TestKit.Tests/ExpandedContractTests.cs b/tests/SensitiveFlow.TestKit.Tests/ExpandedContractTests.cs new file mode 100644 index 0000000..e0b406b --- /dev/null +++ b/tests/SensitiveFlow.TestKit.Tests/ExpandedContractTests.cs @@ -0,0 +1,129 @@ +using SensitiveFlow.Core.Interfaces; +using SensitiveFlow.Core.Models; +using SensitiveFlow.Retention.Contracts; +using SensitiveFlow.TestKit; + +namespace SensitiveFlow.TestKit.Tests; + +public sealed class InMemoryAuditSnapshotStoreContractTests : AuditSnapshotStoreContractTests +{ + protected override Task CreateStoreAsync() + { + return Task.FromResult(new InMemoryAuditSnapshotStore()); + } +} + +public sealed class ReversiblePseudonymizerContractTests : PseudonymizerContractTests +{ + protected override Task CreatePseudonymizerAsync() + { + return Task.FromResult(new ReversiblePseudonymizer()); + } +} + +public sealed class SimpleMaskerContractTests : MaskerContractTests +{ + protected override IMasker CreateMasker() + { + return new SimpleMasker(); + } +} + +public sealed class SimpleAnonymizerContractTests : AnonymizerContractTests +{ + protected override IAnonymizer CreateAnonymizer() + { + return new SimpleAnonymizer(); + } +} + +public sealed class NoopRetentionExpirationHandlerContractTests : RetentionExpirationHandlerContractTests +{ + protected override IRetentionExpirationHandler CreateHandler() + { + return new NoopRetentionExpirationHandler(); + } +} + +internal sealed class InMemoryAuditSnapshotStore : IAuditSnapshotStore +{ + private readonly List _snapshots = []; + + public Task AppendAsync(AuditSnapshot snapshot, CancellationToken cancellationToken = default) + { + _snapshots.Add(snapshot); + return Task.CompletedTask; + } + + public Task> QueryByAggregateAsync( + string aggregate, + string aggregateId, + DateTimeOffset? from = null, + DateTimeOffset? to = null, + int skip = 0, + int take = 100, + CancellationToken cancellationToken = default) + { + return Task.FromResult>(_snapshots + .Where(s => s.Aggregate == aggregate && s.AggregateId == aggregateId) + .Skip(skip) + .Take(take) + .ToArray()); + } + + public Task> QueryByDataSubjectAsync( + string dataSubjectId, + DateTimeOffset? from = null, + DateTimeOffset? to = null, + int skip = 0, + int take = 100, + CancellationToken cancellationToken = default) + { + return Task.FromResult>(_snapshots + .Where(s => s.DataSubjectId == dataSubjectId) + .Skip(skip) + .Take(take) + .ToArray()); + } +} + +internal sealed class ReversiblePseudonymizer : IPseudonymizer +{ + public string Pseudonymize(string value) => "token:" + value; + + public Task PseudonymizeAsync(string value, CancellationToken cancellationToken = default) + { + return Task.FromResult(Pseudonymize(value)); + } + + public string Reverse(string token) => token["token:".Length..]; + + public Task ReverseAsync(string token, CancellationToken cancellationToken = default) + { + return Task.FromResult(Reverse(token)); + } + + public bool CanPseudonymize(string value) => !string.IsNullOrEmpty(value); +} + +internal sealed class SimpleMasker : IMasker +{ + public string Mask(string value) => "***"; + + public bool CanMask(string value) => !string.IsNullOrEmpty(value); +} + +internal sealed class SimpleAnonymizer : IAnonymizer +{ + public string Anonymize(string value) => "[ANONYMIZED]"; + + public bool CanAnonymize(string value) => !string.IsNullOrEmpty(value); +} + +internal sealed class NoopRetentionExpirationHandler : IRetentionExpirationHandler +{ + public Task HandleAsync(object entity, string fieldName, DateTimeOffset expiredAt, CancellationToken cancellationToken = default) + { + return Task.CompletedTask; + } +} diff --git a/tests/SensitiveFlow.TestKit.Tests/SensitiveDataAssertTests.cs b/tests/SensitiveFlow.TestKit.Tests/SensitiveDataAssertTests.cs index 3aa3693..0a9efba 100644 --- a/tests/SensitiveFlow.TestKit.Tests/SensitiveDataAssertTests.cs +++ b/tests/SensitiveFlow.TestKit.Tests/SensitiveDataAssertTests.cs @@ -174,6 +174,46 @@ public void DoesNotLeakKnownValues_RejectsNullKnownValues() act.Should().Throw(); } + [Fact] + public void ContainsMaskedEmail_WhenPayloadHasMaskedEmail_Passes() + { + var act = () => SensitiveDataAssert.ContainsMaskedEmail("email=a****@example.com"); + + act.Should().NotThrow(); + } + + [Fact] + public void ContainsMaskedEmail_WhenPayloadHasNoMaskedEmail_Throws() + { + var act = () => SensitiveDataAssert.ContainsMaskedEmail("email=alice@example.com"); + + act.Should().Throw(); + } + + [Fact] + public void JsonDoesNotExposeAnnotatedProperties_WhenSensitivePropertyIsOmitted_Passes() + { + var act = () => SensitiveDataAssert.JsonDoesNotExposeAnnotatedProperties("{\"Id\":\"1\"}", typeof(Customer)); + + act.Should().NotThrow(); + } + + [Fact] + public void JsonDoesNotExposeAnnotatedProperties_WhenSensitivePropertyExists_Throws() + { + var act = () => SensitiveDataAssert.JsonDoesNotExposeAnnotatedProperties("{\"Name\":\"Alice\"}", typeof(Customer)); + + act.Should().Throw().WithMessage("*Name*"); + } + + [Fact] + public void LogsDoNotContainSensitiveValues_WhenLogContainsKnownValue_Throws() + { + var act = () => SensitiveDataAssert.LogsDoNotContainSensitiveValues("failed for alice@example.com", ["alice@example.com"]); + + act.Should().Throw(); + } + public class Customer { [PersonalData(Category = DataCategory.Identification)] From d8a325295b605a85ad4df841bfc17ad77abd5d12 Mon Sep 17 00:00:00 2001 From: Mateus Silva Date: Mon, 11 May 2026 06:11:24 -0300 Subject: [PATCH 03/23] feat: Document defaults and enhance health checks in SensitiveFlow --- CHANGELOG.md | 3 +- README.md | 8 ++ docs/package-reference.md | 1 + docs/policies-discovery-health.md | 43 +++++++++- samples/MinimalApi.Sample/Program.cs | 36 +++++--- samples/WebApi.Sample/Program.cs | 36 +++++--- .../Discovery/SensitiveDataDiscovery.cs | 62 +++++++------ .../Profiles/SensitiveFlowDefaults.cs | 22 +++++ .../Profiles/SensitiveFlowOptions.cs | 3 +- ...nsitiveFlowHealthCheckBuilderExtensions.cs | 5 +- .../Configuration/JsonRedactionOptions.cs | 10 ++- .../Converters/SensitiveJsonModifier.cs | 83 +++++++++++++++++- .../Services/RetentionExecutorOptions.cs | 3 +- .../SensitiveFlowToolRunner.cs | 41 +++++++-- .../CoreContractTests.cs | 70 +++++++++++++++ .../SensitiveFlowToolRunnerTests.cs | 15 ++++ .../SensitiveFlowValidationTests.cs | 50 +++++++++++ .../SensitiveFlowHealthChecksTests.cs | 57 ++++++++++++ .../JsonRedactionTests.cs | 86 +++++++++++++++++++ 19 files changed, 566 insertions(+), 68 deletions(-) create mode 100644 src/SensitiveFlow.Core/Profiles/SensitiveFlowDefaults.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6710852..091c48d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - `SensitiveFlow.Core` policy engine primitives: `SensitiveFlowOptions`, built-in profiles, category policy registry, output behavior attributes, contextual redaction attributes, data sensitivity levels, discovery reports, export formatters, audit correlation helpers, audit outbox interfaces, data-subject request interfaces, and privacy-safe exception types. -- `SensitiveFlow.Tool` command-line project with `sensitiveflow scan [output-directory]` for JSON/Markdown discovery reports. +- `SensitiveFlow.Tool` command-line project with `sensitiveflow scan [output-directory]` for JSON/Markdown discovery reports. - `SensitiveFlow.HealthChecks` package with audit/token store health checks. - `SensitiveFlow.Diagnostics` startup validation via `AddSensitiveFlowValidation(...)` and `ValidateSensitiveFlow()`. - `SensitiveFlow.Retention` dry-run execution via `RetentionExecutor.DryRunAsync(...)`. - `SensitiveFlow.TestKit` expanded assertions: `ContainsMaskedEmail`, `DoesNotContainRawValues`, `JsonDoesNotExposeAnnotatedProperties`, and `LogsDoNotContainSensitiveValues`. - `SensitiveFlow.TestKit` contract test bases for `IAuditSnapshotStore`, `IPseudonymizer`, `IMasker`, `IAnonymizer`, and `IRetentionExpirationHandler`. +- `SensitiveFlow.Json` now honors output attributes, contextual API response redaction, and category policies before falling back to the configured default mode. - `SensitiveDataAssert.DoesNotContainAny` - checks a payload against explicit string values without requiring an annotated entity. - `SensitiveDataAssert.DoesNotLeakKnownValues` - same as `DoesNotContainAny` but accepts `IEnumerable` for readability. diff --git a/README.md b/README.md index 475a5bc..bc6821d 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,14 @@ builder.Services.AddSensitiveFlowHealthChecks() var startupReport = app.Services.ValidateSensitiveFlow(); ``` +Defaults: + +- profile: `SensitiveFlowProfile.Balanced` +- JSON redaction: `JsonRedactionMode.Mask` +- redaction marker: `[REDACTED]` +- retention anonymization marker: `[ANONYMIZED]` +- health checks: `sensitiveflow-audit-store`, `sensitiveflow-token-store` + Every `SaveChanges` on a field annotated with `[PersonalData]` or `[SensitiveData]` now produces an `AuditRecord` automatically. diff --git a/docs/package-reference.md b/docs/package-reference.md index 53ec387..9581aaf 100644 --- a/docs/package-reference.md +++ b/docs/package-reference.md @@ -39,6 +39,7 @@ Primary APIs: - Enums: `DataCategory`, `SensitiveDataCategory`, `DataSensitivity`, `RetentionPolicy`, `AuditOperation`, `AnonymizationType`. - Policies/profiles: `SensitiveFlowOptions`, `SensitiveFlowProfile`, `SensitiveFlowPolicyRegistry`. - Discovery/export: `SensitiveDataDiscovery`, `JsonDataExportFormatter`, `CsvDataExportFormatter`. +- Defaults: `SensitiveFlowDefaults` documents the default profile, redaction marker, anonymization marker, and health-check names. - Cache: `SensitiveMemberCache`. Install when: diff --git a/docs/policies-discovery-health.md b/docs/policies-discovery-health.md index 9bcbc49..5a36dc9 100644 --- a/docs/policies-discovery-health.md +++ b/docs/policies-discovery-health.md @@ -27,6 +27,46 @@ Built-in profiles: - `Strict`: omission/redaction and required audit for sensitive categories. - `AuditOnly`: audit-focused hints without forcing output behavior. +## Defaults + +SensitiveFlow defaults are intentionally conservative and documented in `SensitiveFlowDefaults`: + +| Setting | Default | +| --- | --- | +| Profile | `SensitiveFlowProfile.Balanced` | +| JSON redaction mode | `JsonRedactionMode.Mask` | +| Full redaction placeholder | `[REDACTED]` | +| Retention anonymization marker | `[ANONYMIZED]` | +| Audit health check name | `sensitiveflow-audit-store` | +| Token health check name | `sensitiveflow-token-store` | +| Logging redactor | redacts marked values to `[REDACTED]` | +| CLI scan input | compiled assembly file or directory containing `.dll` files | + +Precedence for JSON output is: + +1. `[JsonRedaction]` +2. `[Redaction(ApiResponse = ...)]` +3. `[Omit]`, `[Redact]`, `[Mask]` +4. category policies (`OmitInJson`, then `RedactInJson`) +5. `JsonRedactionOptions.DefaultMode` + +Pass policies into JSON redaction explicitly: + +```csharp +var sensitiveFlow = SensitiveFlowPolicyConfiguration.Create(options => +{ + options.UseProfile(SensitiveFlowProfile.Balanced); + options.Policies.ForCategory(DataCategory.Contact).RedactInJson(); +}); + +builder.Services.AddSingleton(sensitiveFlow); +builder.Services.AddSensitiveFlowJsonRedaction(options => +{ + options.DefaultMode = JsonRedactionMode.Mask; + options.Policies = sensitiveFlow.Policies; +}); +``` + ## Discovery report `SensitiveDataDiscovery` scans compiled assemblies for `[PersonalData]`, `[SensitiveData]`, and `[RetentionData]`. @@ -42,11 +82,12 @@ The report includes type, member, annotation kind, category, sensitivity, and re ## CLI tool -`SensitiveFlow.Tool` exposes the same report generator for CI and documentation jobs. +`SensitiveFlow.Tool` exposes the same report generator for CI and documentation jobs. It accepts either one compiled assembly or a directory and scans `.dll` files recursively, skipping `obj` folders. ```bash dotnet tool install SensitiveFlow.Tool sensitiveflow scan ./bin/Release/net10.0/MyApp.dll ./artifacts/privacy +sensitiveflow scan ./src/MyApp/bin/Release/net10.0 ./artifacts/privacy ``` The tool writes: diff --git a/samples/MinimalApi.Sample/Program.cs b/samples/MinimalApi.Sample/Program.cs index c315114..68a970b 100644 --- a/samples/MinimalApi.Sample/Program.cs +++ b/samples/MinimalApi.Sample/Program.cs @@ -72,6 +72,19 @@ builder.Services.AddDataSubjectExport(); builder.Services.AddDataSubjectErasure(); + var sensitiveFlowOptions = SensitiveFlowPolicyConfiguration.Create(options => + { + options.UseProfile(SensitiveFlowProfile.Balanced); + options.Policies.ForCategory(DataCategory.Contact) + .MaskInLogs() + .RedactInJson() + .AuditOnChange(); + options.Policies.ForSensitiveCategory(SensitiveDataCategory.Other) + .OmitInJson() + .RequireAudit(); + }); + + builder.Services.AddSingleton(sensitiveFlowOptions); builder.Services.AddSensitiveFlowLogging(); builder.Services.AddSensitiveFlowEFCore(); builder.Services.AddSensitiveFlowAspNetCore(); @@ -82,26 +95,23 @@ options.RequireJsonRedaction = true; options.RequireRetention = true; }); - builder.Services.AddSensitiveFlowJsonRedaction(options => options.DefaultMode = JsonRedactionMode.Mask); + builder.Services.AddSensitiveFlowJsonRedaction(options => + { + options.DefaultMode = JsonRedactionMode.Mask; + options.Policies = sensitiveFlowOptions.Policies; + }); builder.Services.AddRetention(); builder.Services.AddRetentionExecutor(); - builder.Services.AddSingleton(SensitiveFlowPolicyConfiguration.Create(options => - { - options.UseProfile(SensitiveFlowProfile.Balanced); - options.Policies.ForCategory(DataCategory.Contact) - .MaskInLogs() - .RedactInJson() - .AuditOnChange(); - options.Policies.ForSensitiveCategory(SensitiveDataCategory.Other) - .OmitInJson() - .RequireAudit(); - })); builder.Services.AddSensitiveFlowHealthChecks() .AddAuditStoreCheck() .AddTokenStoreCheck(); builder.Services.ConfigureHttpJsonOptions(options => options.SerializerOptions.WithSensitiveDataRedaction( - new JsonRedactionOptions { DefaultMode = JsonRedactionMode.Mask })); + new JsonRedactionOptions + { + DefaultMode = JsonRedactionMode.Mask, + Policies = sensitiveFlowOptions.Policies, + })); builder.Services.AddOpenTelemetry() .WithTracing(tracing => tracing diff --git a/samples/WebApi.Sample/Program.cs b/samples/WebApi.Sample/Program.cs index 5fc4096..6428f28 100644 --- a/samples/WebApi.Sample/Program.cs +++ b/samples/WebApi.Sample/Program.cs @@ -67,6 +67,19 @@ builder.Services.AddDataSubjectExport(); builder.Services.AddDataSubjectErasure(); + var sensitiveFlowOptions = SensitiveFlowPolicyConfiguration.Create(options => + { + options.UseProfile(SensitiveFlowProfile.Strict); + options.Policies.ForCategory(DataCategory.Contact) + .MaskInLogs() + .RedactInJson() + .AuditOnChange(); + options.Policies.ForSensitiveCategory(SensitiveDataCategory.Other) + .OmitInJson() + .RequireAudit(); + }); + + builder.Services.AddSingleton(sensitiveFlowOptions); builder.Services.AddSensitiveFlowLogging(); builder.Services.AddSensitiveFlowEFCore(); builder.Services.AddSensitiveFlowAspNetCore(); @@ -77,20 +90,13 @@ options.RequireJsonRedaction = true; options.RequireRetention = true; }); - builder.Services.AddSensitiveFlowJsonRedaction(options => options.DefaultMode = JsonRedactionMode.Mask); + builder.Services.AddSensitiveFlowJsonRedaction(options => + { + options.DefaultMode = JsonRedactionMode.Mask; + options.Policies = sensitiveFlowOptions.Policies; + }); builder.Services.AddRetention(); builder.Services.AddRetentionExecutor(); - builder.Services.AddSingleton(SensitiveFlowPolicyConfiguration.Create(options => - { - options.UseProfile(SensitiveFlowProfile.Strict); - options.Policies.ForCategory(DataCategory.Contact) - .MaskInLogs() - .RedactInJson() - .AuditOnChange(); - options.Policies.ForSensitiveCategory(SensitiveDataCategory.Other) - .OmitInJson() - .RequireAudit(); - })); builder.Services.AddSensitiveFlowHealthChecks() .AddAuditStoreCheck() .AddTokenStoreCheck(); @@ -110,7 +116,11 @@ builder.Services.AddControllers() .AddJsonOptions(options => options.JsonSerializerOptions.WithSensitiveDataRedaction( - new JsonRedactionOptions { DefaultMode = JsonRedactionMode.Mask })); + new JsonRedactionOptions + { + DefaultMode = JsonRedactionMode.Mask, + Policies = sensitiveFlowOptions.Policies, + })); builder.Services.AddOpenApi(); var app = builder.Build(); diff --git a/src/SensitiveFlow.Core/Discovery/SensitiveDataDiscovery.cs b/src/SensitiveFlow.Core/Discovery/SensitiveDataDiscovery.cs index 4775ac4..bdecd01 100644 --- a/src/SensitiveFlow.Core/Discovery/SensitiveDataDiscovery.cs +++ b/src/SensitiveFlow.Core/Discovery/SensitiveDataDiscovery.cs @@ -12,40 +12,50 @@ public static class SensitiveDataDiscovery public static SensitiveDataDiscoveryReport Scan(Assembly assembly) { ArgumentNullException.ThrowIfNull(assembly); + return Scan([assembly]); + } + + /// Scans assemblies for annotated fields and properties. + public static SensitiveDataDiscoveryReport Scan(IEnumerable assemblies) + { + ArgumentNullException.ThrowIfNull(assemblies); var entries = new List(); - foreach (var type in assembly.GetTypes().Where(static t => !t.IsGenericTypeDefinition)) + foreach (var assembly in assemblies) { - foreach (var member in type.GetMembers(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)) + foreach (var type in assembly.GetTypes().Where(static t => !t.IsGenericTypeDefinition)) { - if (member.MemberType is not (MemberTypes.Property or MemberTypes.Field)) + foreach (var member in type.GetMembers(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)) { - continue; - } + if (member.MemberType is not (MemberTypes.Property or MemberTypes.Field)) + { + continue; + } - var personal = member.GetCustomAttribute(inherit: true); - var sensitive = member.GetCustomAttribute(inherit: true); - var retention = member.GetCustomAttribute(inherit: true); + var personal = member.GetCustomAttribute(inherit: true); + var sensitive = member.GetCustomAttribute(inherit: true); + var retention = member.GetCustomAttribute(inherit: true); - if (personal is null && sensitive is null && retention is null) - { - continue; - } + if (personal is null && sensitive is null && retention is null) + { + continue; + } - entries.Add(new SensitiveDataDiscoveryEntry - { - TypeName = type.Name, - MemberName = member.Name, - Annotation = sensitive is not null - ? nameof(SensitiveDataAttribute) - : personal is not null ? nameof(PersonalDataAttribute) : nameof(RetentionDataAttribute), - Category = personal?.Category, - SensitiveCategory = sensitive?.Category, - Sensitivity = sensitive?.Sensitivity ?? personal?.Sensitivity ?? Core.Enums.DataSensitivity.Medium, - RetentionYears = retention?.Years, - RetentionMonths = retention?.Months, - RetentionPolicy = retention?.Policy, - }); + entries.Add(new SensitiveDataDiscoveryEntry + { + TypeName = type.Name, + MemberName = member.Name, + Annotation = sensitive is not null + ? nameof(SensitiveDataAttribute) + : personal is not null ? nameof(PersonalDataAttribute) : nameof(RetentionDataAttribute), + Category = personal?.Category, + SensitiveCategory = sensitive?.Category, + Sensitivity = sensitive?.Sensitivity ?? personal?.Sensitivity ?? Core.Enums.DataSensitivity.Medium, + RetentionYears = retention?.Years, + RetentionMonths = retention?.Months, + RetentionPolicy = retention?.Policy, + }); + } } } diff --git a/src/SensitiveFlow.Core/Profiles/SensitiveFlowDefaults.cs b/src/SensitiveFlow.Core/Profiles/SensitiveFlowDefaults.cs new file mode 100644 index 0000000..0d85108 --- /dev/null +++ b/src/SensitiveFlow.Core/Profiles/SensitiveFlowDefaults.cs @@ -0,0 +1,22 @@ +namespace SensitiveFlow.Core.Profiles; + +/// +/// Documented defaults shared across SensitiveFlow packages. +/// +public static class SensitiveFlowDefaults +{ + /// Default profile used when callers do not select one explicitly. + public const SensitiveFlowProfile Profile = SensitiveFlowProfile.Balanced; + + /// Default marker used for full redaction. + public const string RedactedPlaceholder = "[REDACTED]"; + + /// Default marker used by retention anonymization. + public const string AnonymousValue = "[ANONYMIZED]"; + + /// Default health check name for audit stores. + public const string AuditStoreHealthCheckName = "sensitiveflow-audit-store"; + + /// Default health check name for token stores. + public const string TokenStoreHealthCheckName = "sensitiveflow-token-store"; +} diff --git a/src/SensitiveFlow.Core/Profiles/SensitiveFlowOptions.cs b/src/SensitiveFlow.Core/Profiles/SensitiveFlowOptions.cs index 2ecd755..60400fd 100644 --- a/src/SensitiveFlow.Core/Profiles/SensitiveFlowOptions.cs +++ b/src/SensitiveFlow.Core/Profiles/SensitiveFlowOptions.cs @@ -9,7 +9,7 @@ namespace SensitiveFlow.Core.Profiles; public sealed class SensitiveFlowOptions { /// Gets the active profile. - public SensitiveFlowProfile Profile { get; private set; } = SensitiveFlowProfile.Balanced; + public SensitiveFlowProfile Profile { get; private set; } = SensitiveFlowDefaults.Profile; /// Gets the policy registry. public SensitiveFlowPolicyRegistry Policies { get; } = new(); @@ -50,4 +50,3 @@ public SensitiveFlowOptions UseProfile(SensitiveFlowProfile profile) return this; } } - diff --git a/src/SensitiveFlow.HealthChecks/Extensions/SensitiveFlowHealthCheckBuilderExtensions.cs b/src/SensitiveFlow.HealthChecks/Extensions/SensitiveFlowHealthCheckBuilderExtensions.cs index db73f3c..cc33f15 100644 --- a/src/SensitiveFlow.HealthChecks/Extensions/SensitiveFlowHealthCheckBuilderExtensions.cs +++ b/src/SensitiveFlow.HealthChecks/Extensions/SensitiveFlowHealthCheckBuilderExtensions.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Diagnostics.HealthChecks; +using SensitiveFlow.Core.Profiles; using SensitiveFlow.HealthChecks.Checks; namespace SensitiveFlow.HealthChecks.Extensions; @@ -19,7 +20,7 @@ public static IHealthChecksBuilder AddSensitiveFlowHealthChecks(this IServiceCol /// Adds an audit store health check. public static IHealthChecksBuilder AddAuditStoreCheck( this IHealthChecksBuilder builder, - string name = "sensitiveflow-audit-store", + string name = SensitiveFlowDefaults.AuditStoreHealthCheckName, HealthStatus? failureStatus = null, IEnumerable? tags = null) { @@ -30,7 +31,7 @@ public static IHealthChecksBuilder AddAuditStoreCheck( /// Adds a token store health check. public static IHealthChecksBuilder AddTokenStoreCheck( this IHealthChecksBuilder builder, - string name = "sensitiveflow-token-store", + string name = SensitiveFlowDefaults.TokenStoreHealthCheckName, HealthStatus? failureStatus = null, IEnumerable? tags = null) { diff --git a/src/SensitiveFlow.Json/Configuration/JsonRedactionOptions.cs b/src/SensitiveFlow.Json/Configuration/JsonRedactionOptions.cs index ce25d74..75360f5 100644 --- a/src/SensitiveFlow.Json/Configuration/JsonRedactionOptions.cs +++ b/src/SensitiveFlow.Json/Configuration/JsonRedactionOptions.cs @@ -1,4 +1,6 @@ using SensitiveFlow.Json.Enums; +using SensitiveFlow.Core.Policies; +using SensitiveFlow.Core.Profiles; namespace SensitiveFlow.Json.Configuration; @@ -19,5 +21,11 @@ public sealed class JsonRedactionOptions /// fallback for non-string values in . /// Defaults to "[REDACTED]". /// - public string RedactedPlaceholder { get; set; } = "[REDACTED]"; + public string RedactedPlaceholder { get; set; } = SensitiveFlowDefaults.RedactedPlaceholder; + + /// + /// Optional category policy registry. When present, JSON actions from policies are used + /// before and after per-property attributes. + /// + public SensitiveFlowPolicyRegistry? Policies { get; set; } } diff --git a/src/SensitiveFlow.Json/Converters/SensitiveJsonModifier.cs b/src/SensitiveFlow.Json/Converters/SensitiveJsonModifier.cs index 14b9e68..daca5c0 100644 --- a/src/SensitiveFlow.Json/Converters/SensitiveJsonModifier.cs +++ b/src/SensitiveFlow.Json/Converters/SensitiveJsonModifier.cs @@ -2,6 +2,8 @@ using System.Text.Json; using System.Text.Json.Serialization.Metadata; using SensitiveFlow.Anonymization.Extensions; +using SensitiveFlow.Core.Attributes; +using SensitiveFlow.Core.Policies; using SensitiveFlow.Core.Reflection; using SensitiveFlow.Json.Attributes; using SensitiveFlow.Json.Configuration; @@ -56,7 +58,7 @@ public static Action Create(JsonRedactionOptions options) continue; } - var mode = ResolveMode(clrProperty, options.DefaultMode); + var mode = ResolveMode(clrProperty, options); if (mode == JsonRedactionMode.None) { @@ -74,10 +76,85 @@ public static Action Create(JsonRedactionOptions options) }; } - private static JsonRedactionMode ResolveMode(PropertyInfo property, JsonRedactionMode defaultMode) + private static JsonRedactionMode ResolveMode(PropertyInfo property, JsonRedactionOptions options) { var overrideAttr = property.GetCustomAttribute(inherit: true); - return overrideAttr?.Mode ?? defaultMode; + if (overrideAttr is not null) + { + return overrideAttr.Mode; + } + + var contextual = property.GetCustomAttribute(inherit: true); + var contextualAction = contextual?.ForContext(Core.Enums.RedactionContext.ApiResponse) + ?? Core.Enums.OutputRedactionAction.None; + if (contextualAction != Core.Enums.OutputRedactionAction.None) + { + return ToJsonMode(contextualAction); + } + + if (property.GetCustomAttribute(inherit: true) is not null) + { + return JsonRedactionMode.Omit; + } + + if (property.GetCustomAttribute(inherit: true) is not null) + { + return JsonRedactionMode.Redacted; + } + + if (property.GetCustomAttribute(inherit: true) is not null) + { + return JsonRedactionMode.Mask; + } + + var policyMode = ResolvePolicyMode(property, options.Policies); + return policyMode ?? options.DefaultMode; + } + + private static JsonRedactionMode? ResolvePolicyMode(PropertyInfo property, SensitiveFlowPolicyRegistry? policies) + { + if (policies is null) + { + return null; + } + + var personal = property.GetCustomAttribute(inherit: true); + var personalRule = personal is null ? null : policies.Find(personal.Category); + if (personalRule is not null) + { + return ToJsonMode(personalRule.Actions); + } + + var sensitive = property.GetCustomAttribute(inherit: true); + var sensitiveRule = sensitive is null ? null : policies.Find(sensitive.Category); + return sensitiveRule is null ? null : ToJsonMode(sensitiveRule.Actions); + } + + private static JsonRedactionMode? ToJsonMode(SensitiveFlowPolicyAction actions) + { + if ((actions & SensitiveFlowPolicyAction.OmitInJson) == SensitiveFlowPolicyAction.OmitInJson) + { + return JsonRedactionMode.Omit; + } + + if ((actions & SensitiveFlowPolicyAction.RedactInJson) == SensitiveFlowPolicyAction.RedactInJson) + { + return JsonRedactionMode.Redacted; + } + + return null; + } + + private static JsonRedactionMode ToJsonMode(Core.Enums.OutputRedactionAction action) + { + return action switch + { + Core.Enums.OutputRedactionAction.Omit => JsonRedactionMode.Omit, + Core.Enums.OutputRedactionAction.Redact => JsonRedactionMode.Redacted, + Core.Enums.OutputRedactionAction.Mask => JsonRedactionMode.Mask, + Core.Enums.OutputRedactionAction.None => JsonRedactionMode.None, + _ => JsonRedactionMode.Redacted, + }; } private static void ApplyRedactingGetter(JsonPropertyInfo jsonProperty, JsonRedactionMode mode, string placeholder) diff --git a/src/SensitiveFlow.Retention/Services/RetentionExecutorOptions.cs b/src/SensitiveFlow.Retention/Services/RetentionExecutorOptions.cs index 29cb978..0bedd51 100644 --- a/src/SensitiveFlow.Retention/Services/RetentionExecutorOptions.cs +++ b/src/SensitiveFlow.Retention/Services/RetentionExecutorOptions.cs @@ -1,4 +1,5 @@ using System.Reflection; +using SensitiveFlow.Core.Profiles; namespace SensitiveFlow.Retention.Services; @@ -21,7 +22,7 @@ public sealed class RetentionExecutorOptions /// Default placeholder used by for string properties. /// Value types collapse to default(T). /// - public string AnonymousStringMarker { get; set; } = "[ANONYMIZED]"; + public string AnonymousStringMarker { get; set; } = SensitiveFlowDefaults.AnonymousValue; /// /// Returns the type's anonymous default. Strings return , diff --git a/src/SensitiveFlow.Tool/SensitiveFlowToolRunner.cs b/src/SensitiveFlow.Tool/SensitiveFlowToolRunner.cs index b6baf48..4a25b54 100644 --- a/src/SensitiveFlow.Tool/SensitiveFlowToolRunner.cs +++ b/src/SensitiveFlow.Tool/SensitiveFlowToolRunner.cs @@ -21,10 +21,10 @@ public static int Run(string[] args, TextWriter output, TextWriter error) return 2; } - var assemblyPath = Path.GetFullPath(args[1]); - if (!File.Exists(assemblyPath)) + var inputPath = Path.GetFullPath(args[1]); + if (!File.Exists(inputPath) && !Directory.Exists(inputPath)) { - error.WriteLine($"Assembly not found: {assemblyPath}"); + error.WriteLine($"Assembly or directory not found: {inputPath}"); return 3; } @@ -33,8 +33,14 @@ public static int Run(string[] args, TextWriter output, TextWriter error) : Directory.GetCurrentDirectory(); Directory.CreateDirectory(outputDirectory); - var assembly = Assembly.LoadFrom(assemblyPath); - var report = SensitiveDataDiscovery.Scan(assembly); + var assemblies = ResolveAssemblies(inputPath).ToArray(); + if (assemblies.Length == 0) + { + error.WriteLine($"No assemblies found to scan: {inputPath}"); + return 4; + } + + var report = SensitiveDataDiscovery.Scan(assemblies); var jsonPath = Path.Combine(outputDirectory, "sensitiveflow-report.json"); var markdownPath = Path.Combine(outputDirectory, "sensitiveflow-report.md"); @@ -46,4 +52,29 @@ public static int Run(string[] args, TextWriter output, TextWriter error) output.WriteLine($"SensitiveFlow report written: {markdownPath}"); return 0; } + + private static IEnumerable ResolveAssemblies(string inputPath) + { + if (File.Exists(inputPath)) + { + yield return Assembly.LoadFrom(inputPath); + yield break; + } + + foreach (var file in Directory.EnumerateFiles(inputPath, "*.dll", SearchOption.AllDirectories) + .Where(static p => !p.Contains($"{Path.DirectorySeparatorChar}obj{Path.DirectorySeparatorChar}", StringComparison.OrdinalIgnoreCase))) + { + Assembly assembly; + try + { + assembly = Assembly.LoadFrom(file); + } + catch (BadImageFormatException) + { + continue; + } + + yield return assembly; + } + } } diff --git a/tests/SensitiveFlow.Core.Tests/CoreContractTests.cs b/tests/SensitiveFlow.Core.Tests/CoreContractTests.cs index 06b6476..ecb1f9c 100644 --- a/tests/SensitiveFlow.Core.Tests/CoreContractTests.cs +++ b/tests/SensitiveFlow.Core.Tests/CoreContractTests.cs @@ -1,6 +1,7 @@ using FluentAssertions; using NSubstitute; using SensitiveFlow.Core.Attributes; +using SensitiveFlow.Core.Correlation; using SensitiveFlow.Core.Discovery; using SensitiveFlow.Core.Enums; using SensitiveFlow.Core.Exceptions; @@ -227,6 +228,67 @@ public void SensitiveFlowOptions_StrictProfile_RequiresAuditForSensitiveCategori (rule.Actions & SensitiveFlowPolicyAction.RequireAudit).Should().Be(SensitiveFlowPolicyAction.RequireAudit); } + [Fact] + public void SensitiveFlowDefaults_AreDocumented() + { + SensitiveFlowDefaults.Profile.Should().Be(SensitiveFlowProfile.Balanced); + SensitiveFlowDefaults.RedactedPlaceholder.Should().Be("[REDACTED]"); + SensitiveFlowDefaults.AnonymousValue.Should().Be("[ANONYMIZED]"); + SensitiveFlowDefaults.AuditStoreHealthCheckName.Should().Be("sensitiveflow-audit-store"); + SensitiveFlowDefaults.TokenStoreHealthCheckName.Should().Be("sensitiveflow-token-store"); + } + + [Theory] + [InlineData(SensitiveFlowProfile.Development)] + [InlineData(SensitiveFlowProfile.Balanced)] + [InlineData(SensitiveFlowProfile.Strict)] + [InlineData(SensitiveFlowProfile.AuditOnly)] + public void SensitiveFlowOptions_AllBuiltInProfiles_AddRules(SensitiveFlowProfile profile) + { + var options = new SensitiveFlowOptions().UseProfile(profile); + + options.Profile.Should().Be(profile); + options.Policies.Rules.Should().NotBeEmpty(); + } + + [Fact] + public void SensitiveFlowPolicyConfiguration_Create_AppliesCallback() + { + var options = SensitiveFlowPolicyConfiguration.Create(o => + o.Policies.ForCategory(DataCategory.Contact).RedactInJson()); + + options.Policies.Find(DataCategory.Contact)!.Actions + .Should().Be(SensitiveFlowPolicyAction.RedactInJson); + } + + [Fact] + public void JsonDataExportFormatter_FormatsRows() + { + var formatter = new JsonDataExportFormatter(); + + var json = formatter.Format([ + new Dictionary { ["Email"] = "alice@example.com" }, + ]); + + json.Should().Contain("alice@example.com"); + } + + [Fact] + public void SensitiveFlowCorrelation_StoresAsyncLocalSnapshot() + { + var snapshot = new AuditCorrelationSnapshot + { + CorrelationId = "corr-1", + RequestId = "req-1", + TraceId = "trace-1", + }; + + SensitiveFlowCorrelation.Current = snapshot; + + SensitiveFlowCorrelation.Current.Should().Be(snapshot); + SensitiveFlowCorrelation.Current = null; + } + [Fact] public void SensitiveDataDiscovery_Scan_ReturnsAnnotatedMembers() { @@ -239,6 +301,14 @@ public void SensitiveDataDiscovery_Scan_ReturnsAnnotatedMembers() report.ToJson().Should().Contain("DiscoveryCustomer"); } + [Fact] + public void SensitiveDataDiscovery_ScanMany_ReturnsAnnotatedMembers() + { + var report = SensitiveDataDiscovery.Scan([typeof(CoreContractTests).Assembly]); + + report.Entries.Should().Contain(e => e.TypeName == nameof(DiscoveryCustomer)); + } + [Fact] public void SensitiveFlowExceptionSanitizer_RemovesCommonRawValues() { diff --git a/tests/SensitiveFlow.Core.Tests/SensitiveFlowToolRunnerTests.cs b/tests/SensitiveFlow.Core.Tests/SensitiveFlowToolRunnerTests.cs index 6122c96..0858502 100644 --- a/tests/SensitiveFlow.Core.Tests/SensitiveFlowToolRunnerTests.cs +++ b/tests/SensitiveFlow.Core.Tests/SensitiveFlowToolRunnerTests.cs @@ -33,5 +33,20 @@ public void Run_WithoutScanCommand_ReturnsUsageError() exitCode.Should().Be(2); error.ToString().Should().Contain("Usage:"); } + + [Fact] + public void Run_ScanDirectory_WritesReports() + { + var outputDirectory = Path.Combine(Path.GetTempPath(), "sensitiveflow-tool-tests", Guid.NewGuid().ToString("N")); + var assemblyDirectory = Path.GetDirectoryName(typeof(SensitiveFlowToolRunnerTests).Assembly.Location)!; + using var output = new StringWriter(); + using var error = new StringWriter(); + + var exitCode = SensitiveFlowToolRunner.Run(["scan", assemblyDirectory, outputDirectory], output, error); + + exitCode.Should().Be(0); + File.Exists(Path.Combine(outputDirectory, "sensitiveflow-report.json")).Should().BeTrue(); + File.Exists(Path.Combine(outputDirectory, "sensitiveflow-report.md")).Should().BeTrue(); + } } #endif diff --git a/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs b/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs index f1695ed..ab18d2e 100644 --- a/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs +++ b/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs @@ -32,6 +32,37 @@ public void ValidateSensitiveFlow_WhenRequiredStoresExist_ReturnsNoStoreWarnings report.Diagnostics.Should().NotContain(d => d.Code == "SF-CONFIG-001"); } + [Fact] + public void ValidateSensitiveFlow_WhenTokenStoreRequiredAndMissing_ReportsWarning() + { + var services = new ServiceCollection(); + services.AddSensitiveFlowValidation(o => o.RequireTokenStore = true); + + var report = services.BuildServiceProvider().ValidateSensitiveFlow(); + + report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-002"); + } + + [Fact] + public void ValidateSensitiveFlow_WhenPseudonymizerWithoutTokenStore_ReportsWarning() + { + var services = new ServiceCollection(); + services.AddSingleton(); + services.AddSensitiveFlowValidation(); + + var report = services.BuildServiceProvider().ValidateSensitiveFlow(); + + report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-003"); + } + + [Fact] + public void ValidateSensitiveFlow_WithoutRegisteredValidator_UsesDefaultValidator() + { + var report = new ServiceCollection().BuildServiceProvider().ValidateSensitiveFlow(); + + report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-001"); + } + private sealed class HealthyAuditStore : IAuditStore { public Task AppendAsync(AuditRecord record, CancellationToken cancellationToken = default) @@ -49,4 +80,23 @@ public Task> QueryByDataSubjectAsync(string dataSubje return Task.FromResult>([]); } } + + private sealed class FakePseudonymizer : IPseudonymizer + { + public string Pseudonymize(string value) => "token"; + + public Task PseudonymizeAsync(string value, CancellationToken cancellationToken = default) + { + return Task.FromResult("token"); + } + + public string Reverse(string token) => "value"; + + public Task ReverseAsync(string token, CancellationToken cancellationToken = default) + { + return Task.FromResult("value"); + } + + public bool CanPseudonymize(string value) => true; + } } diff --git a/tests/SensitiveFlow.HealthChecks.Tests/SensitiveFlowHealthChecksTests.cs b/tests/SensitiveFlow.HealthChecks.Tests/SensitiveFlowHealthChecksTests.cs index 47a8086..1c783c6 100644 --- a/tests/SensitiveFlow.HealthChecks.Tests/SensitiveFlowHealthChecksTests.cs +++ b/tests/SensitiveFlow.HealthChecks.Tests/SensitiveFlowHealthChecksTests.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging; using SensitiveFlow.Core.Interfaces; using SensitiveFlow.Core.Models; +using SensitiveFlow.HealthChecks.Checks; using SensitiveFlow.HealthChecks.Extensions; namespace SensitiveFlow.HealthChecks.Tests; @@ -46,6 +47,26 @@ public async Task AddSensitiveFlowHealthChecks_RegisteredChecksRunHealthy() result.Status.Should().Be(HealthStatus.Healthy); } + [Fact] + public async Task AuditStoreCheck_WhenProbeFails_ReturnsUnhealthy() + { + var check = new AuditStoreHealthCheck(new FailingAuditStore()); + + var result = await check.CheckHealthAsync(new HealthCheckContext()); + + result.Status.Should().Be(HealthStatus.Unhealthy); + } + + [Fact] + public async Task TokenStoreCheck_WhenProbeFails_ReturnsUnhealthy() + { + var check = new TokenStoreHealthCheck(new FailingTokenStore()); + + var result = await check.CheckHealthAsync(new HealthCheckContext()); + + result.Status.Should().Be(HealthStatus.Unhealthy); + } + private sealed class HealthyAuditStore : IAuditStore { public Task AppendAsync(AuditRecord record, CancellationToken cancellationToken = default) @@ -76,4 +97,40 @@ public Task ResolveTokenAsync(string token, CancellationToken cancellati return Task.FromResult("value"); } } + + private sealed class FailingAuditStore : IAuditStore + { + public Task AppendAsync(AuditRecord record, CancellationToken cancellationToken = default) + { + throw new InvalidOperationException("append failed"); + } + + public Task> QueryAsync(DateTimeOffset? from = null, DateTimeOffset? to = null, int skip = 0, int take = 100, CancellationToken cancellationToken = default) + { + throw new InvalidOperationException("query failed"); + } + + public Task> QueryByDataSubjectAsync(string dataSubjectId, DateTimeOffset? from = null, DateTimeOffset? to = null, int skip = 0, int take = 100, CancellationToken cancellationToken = default) + { + throw new InvalidOperationException("query failed"); + } + } + + private sealed class FailingTokenStore : ITokenStore, IHealthProbe + { + public Task ProbeAsync(CancellationToken cancellationToken = default) + { + throw new InvalidOperationException("probe failed"); + } + + public Task GetOrCreateTokenAsync(string value, CancellationToken cancellationToken = default) + { + throw new InvalidOperationException("token failed"); + } + + public Task ResolveTokenAsync(string token, CancellationToken cancellationToken = default) + { + throw new InvalidOperationException("token failed"); + } + } } diff --git a/tests/SensitiveFlow.Json.Tests/JsonRedactionTests.cs b/tests/SensitiveFlow.Json.Tests/JsonRedactionTests.cs index 663802c..46eaa46 100644 --- a/tests/SensitiveFlow.Json.Tests/JsonRedactionTests.cs +++ b/tests/SensitiveFlow.Json.Tests/JsonRedactionTests.cs @@ -9,6 +9,7 @@ using SensitiveFlow.Json.Configuration; using SensitiveFlow.Json.Enums; using SensitiveFlow.Json.Extensions; +using SensitiveFlow.Core.Policies; namespace SensitiveFlow.Json.Tests; @@ -301,6 +302,57 @@ public void AddSensitiveFlowJsonRedaction_RejectsNullServices() act.Should().Throw(); } + [Fact] + public void OutputAttributes_TakePrecedenceOverDefaultMode() + { + var json = JsonSerializer.Serialize(new OutputAttributeCustomer + { + Email = "alice@example.com", + TaxId = "12345678900", + Secret = "hidden", + }, Build(new JsonRedactionOptions { DefaultMode = JsonRedactionMode.Mask })); + + json.Should().Contain("\"Email\":\"[REDACTED]\""); + json.Should().Contain("\"TaxId\":\"1**********\""); + json.Should().NotContain("Secret"); + json.Should().NotContain("hidden"); + } + + [Fact] + public void ContextualRedaction_ApiResponseMode_TakesPrecedenceOverDefaultMode() + { + var json = JsonSerializer.Serialize(new ContextualCustomer + { + Email = "alice@example.com", + }, Build(new JsonRedactionOptions { DefaultMode = JsonRedactionMode.Mask })); + + json.Should().NotContain("Email"); + json.Should().NotContain("alice@example.com"); + } + + [Fact] + public void Policies_ResolveJsonActionsByCategory() + { + var policies = new SensitiveFlowPolicyRegistry(); + policies.ForCategory(DataCategory.Contact).RedactInJson(); + policies.ForSensitiveCategory(SensitiveDataCategory.Other).OmitInJson(); + + var json = JsonSerializer.Serialize(new PolicyCustomer + { + Email = "alice@example.com", + TaxId = "12345678900", + Name = "Alice", + }, Build(new JsonRedactionOptions + { + DefaultMode = JsonRedactionMode.Mask, + Policies = policies, + })); + + json.Should().Contain("\"Email\":\"[REDACTED]\""); + json.Should().NotContain("TaxId"); + json.Should().Contain("\"Name\":\"A****\""); + } + public class Customer { public int Id { get; set; } @@ -381,4 +433,38 @@ public class UnknownModeSensitiveData } public sealed record Payload(string Value); + + public class OutputAttributeCustomer + { + [PersonalData(Category = DataCategory.Contact)] + [Redact] + public string Email { get; set; } = string.Empty; + + [SensitiveData(Category = SensitiveDataCategory.Other)] + [Mask(MaskKind.Generic)] + public string TaxId { get; set; } = string.Empty; + + [SensitiveData(Category = SensitiveDataCategory.Other)] + [Omit] + public string Secret { get; set; } = string.Empty; + } + + public class ContextualCustomer + { + [PersonalData(Category = DataCategory.Contact)] + [Redaction(ApiResponse = OutputRedactionAction.Omit, Logs = OutputRedactionAction.Mask)] + public string Email { get; set; } = string.Empty; + } + + public class PolicyCustomer + { + [PersonalData(Category = DataCategory.Contact)] + public string Email { get; set; } = string.Empty; + + [SensitiveData(Category = SensitiveDataCategory.Other)] + public string TaxId { get; set; } = string.Empty; + + [PersonalData(Category = DataCategory.Identification)] + public string Name { get; set; } = string.Empty; + } } From a40578faab9d11a9e6b5a8ec12193f30303c6ce9 Mon Sep 17 00:00:00 2001 From: Mateus Silva Date: Mon, 11 May 2026 06:43:18 -0300 Subject: [PATCH 04/23] feat: Enhance SensitiveFlow Logging and Redaction Features - Updated logging documentation to clarify sensitive value redaction. - Introduced `SensitiveLoggingOptions` for customizable logging behavior. - Enhanced `RedactingLogger` to support contextual redaction based on attributes. - Implemented masking for structured object properties based on defined policies. - Added support for contextual export redaction in `DataSubjectExporter`. - Improved validation diagnostics for sensitive flow configurations. - Added tests for structured logging redaction and policy enforcement. - Updated default redaction markers to use `SensitiveFlowDefaults`. --- CHANGELOG.md | 2 + README.md | 1 + docs/anonymization.md | 12 + docs/logging.md | 37 ++- docs/package-reference.md | 11 +- samples/MinimalApi.Sample/Program.cs | 5 +- samples/WebApi.Sample/Program.cs | 5 +- .../Export/DataSubjectExporter.cs | 122 ++++++++- .../Profiles/SensitiveFlowDefaults.cs | 26 +- .../Profiles/SensitiveFlowOptions.cs | 5 +- .../SensitiveFlowConfigurationValidator.cs | 54 +++- .../Configuration/JsonRedactionOptions.cs | 2 +- .../Configuration/SensitiveLoggingOptions.cs | 36 +++ .../LoggingServiceCollectionExtensions.cs | 43 ++- .../Loggers/RedactingLogger.cs | 253 +++++++++++++++++- .../Loggers/RedactingLoggerProvider.cs | 10 +- .../DefaultSensitiveValueRedactor.cs | 6 +- .../Services/RetentionExecutorOptions.cs | 3 +- .../Export/DataSubjectExporterTests.cs | 25 ++ .../SensitiveFlowValidationTests.cs | 44 +++ .../LoggingExtensionsTests.cs | 20 ++ .../RedactingLoggerStructuredTests.cs | 114 ++++++++ 22 files changed, 801 insertions(+), 35 deletions(-) create mode 100644 src/SensitiveFlow.Logging/Configuration/SensitiveLoggingOptions.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 091c48d..2b2bde4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `SensitiveFlow.TestKit` expanded assertions: `ContainsMaskedEmail`, `DoesNotContainRawValues`, `JsonDoesNotExposeAnnotatedProperties`, and `LogsDoNotContainSensitiveValues`. - `SensitiveFlow.TestKit` contract test bases for `IAuditSnapshotStore`, `IPseudonymizer`, `IMasker`, `IAnonymizer`, and `IRetentionExpirationHandler`. - `SensitiveFlow.Json` now honors output attributes, contextual API response redaction, and category policies before falling back to the configured default mode. +- `SensitiveFlow.Logging` now redacts annotated structured object members by default and applies `.MaskInLogs()` policies when a policy registry is supplied through `SensitiveLoggingOptions`. +- `DataSubjectExporter` now honors contextual `[Redaction(Export = ...)]` attributes while keeping raw export as the default. - `SensitiveDataAssert.DoesNotContainAny` - checks a payload against explicit string values without requiring an annotated entity. - `SensitiveDataAssert.DoesNotLeakKnownValues` - same as `DoesNotContainAny` but accepts `IEnumerable` for readability. diff --git a/README.md b/README.md index bc6821d..87eea4a 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,7 @@ Defaults: - profile: `SensitiveFlowProfile.Balanced` - JSON redaction: `JsonRedactionMode.Mask` - redaction marker: `[REDACTED]` +- logging: `[Sensitive]` markers are redacted; annotated structured object members are redacted unless policies request `MaskInLogs()` - retention anonymization marker: `[ANONYMIZED]` - health checks: `sensitiveflow-audit-store`, `sensitiveflow-token-store` diff --git a/docs/anonymization.md b/docs/anonymization.md index 58af9ec..ab6880a 100644 --- a/docs/anonymization.md +++ b/docs/anonymization.md @@ -235,6 +235,18 @@ new HashStrategy("my-fixed-salt-16ch").Apply("value"); ## Data subject export (portability) +By default, data-subject export returns raw annotated values because portability responses often need to disclose the data back to the subject. To protect specific export fields, use contextual redaction: + +```csharp +[PersonalData(Category = DataCategory.Contact)] +[Redaction(Export = OutputRedactionAction.Mask)] +public string Email { get; set; } = string.Empty; + +[SensitiveData(Category = SensitiveDataCategory.Other)] +[Redaction(Export = OutputRedactionAction.Omit)] +public string InternalRiskNote { get; set; } = string.Empty; +``` + `IDataSubjectExporter` is the read-side counterpart of `IDataSubjectErasureService`. Given an entity, it returns a dictionary keyed by property name with every annotated value (`[PersonalData]`, `[SensitiveData]`, `[RetentionData]`) — useful for satisfying portability requests where a user asks for a copy of the personal data the application holds about them. ```csharp diff --git a/docs/logging.md b/docs/logging.md index 686c758..fcadeb3 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -1,6 +1,6 @@ # Log Redaction -`SensitiveFlow.Logging` prevents PII from reaching log sinks by intercepting messages before they are written. +`SensitiveFlow.Logging` prevents sensitive values from reaching log sinks by intercepting messages before they are written. ## Installation @@ -12,7 +12,9 @@ dotnet add package SensitiveFlow.Logging `RedactingLogger` is an `ILogger` decorator. It wraps any existing `ILogger` and, before forwarding a message, replaces every occurrence of the pattern `[Sensitive]` with a fixed redaction marker. -The pattern matches immediately after the `[Sensitive]` tag up to the first whitespace, comma, or closing brace — enough to redact a structured log parameter value without disturbing the rest of the message template. +The pattern matches immediately after the `[Sensitive]` tag up to the first whitespace, comma, or closing brace, enough to redact a structured log parameter value without disturbing the rest of the message template. + +For structured object values, the logger also inspects public properties annotated with `[PersonalData]` or `[SensitiveData]`. Annotated object members are redacted by default. If you pass a `SensitiveFlowPolicyRegistry`, categories configured with `.MaskInLogs()` are masked instead. ## Marking sensitive values @@ -23,13 +25,31 @@ logger.LogInformation("User {[Sensitive]Email} logged in from {Ip}", email, ip); // Logged as: "User [REDACTED] logged in from 192.168.1.1" ``` -Only `[Sensitive]`-tagged values are redacted. Other parameters pass through unchanged. +`[Sensitive]`-tagged values are always redacted. Structured object members are redacted when they carry SensitiveFlow annotations. Other parameters pass through unchanged. + +```csharp +var sensitiveFlow = SensitiveFlowPolicyConfiguration.Create(options => +{ + options.Policies.ForCategory(DataCategory.Contact).MaskInLogs(); +}); + +builder.Services.AddSensitiveFlowLogging(options => +{ + options.Policies = sensitiveFlow.Policies; +}); + +logger.LogInformation("Customer {@Customer}", customer); +``` ## Registration ```csharp builder.Services.AddSensitiveFlowLogging(); // uses [REDACTED] marker builder.Services.AddSensitiveFlowLogging("***"); // custom marker +builder.Services.AddSensitiveFlowLogging(options => +{ + options.Policies = sensitiveFlow.Policies; +}); ``` This registers `DefaultSensitiveValueRedactor` as a singleton `ISensitiveValueRedactor`. @@ -68,7 +88,7 @@ public interface ISensitiveValueRedactor } ``` -Example — hash-based redaction for deterministic deduplication: +Example hash-based redaction for deterministic deduplication: ```csharp public sealed class HashRedactor : ISensitiveValueRedactor @@ -94,6 +114,15 @@ redactor.Redact("secret@example.com"); // returns "[REDACTED]" The marker must be a non-whitespace string; an empty or whitespace-only value throws `ArgumentException`. +## Defaults + +| Option | Default | +| --- | --- | +| `SensitiveLoggingOptions.RedactedPlaceholder` | `SensitiveFlowDefaults.RedactedPlaceholder` (`[REDACTED]`) | +| `SensitiveLoggingOptions.RedactAnnotatedObjects` | `true` | +| `SensitiveLoggingOptions.DefaultAnnotatedMemberAction` | `OutputRedactionAction.Redact` | +| `SensitiveLoggingOptions.Policies` | `null` | + ## OpenTelemetry metrics `RedactingLogger` emits the `sensitiveflow.redact.fields.count` counter via `System.Diagnostics.Metrics`. Pair it with `SensitiveFlow.Diagnostics` or register the `SensitiveFlow` meter directly: diff --git a/docs/package-reference.md b/docs/package-reference.md index 9581aaf..be14854 100644 --- a/docs/package-reference.md +++ b/docs/package-reference.md @@ -15,14 +15,14 @@ This document summarizes each SensitiveFlow package individually: purpose, prima | `SensitiveFlow.AspNetCore` | You need actor/IP context from HTTP requests. | `AddSensitiveFlowAspNetCore()` and `UseSensitiveFlowAudit()`. | Requires a durable `IPseudonymizer`/`ITokenStore` for reversible IP tokens. | | `SensitiveFlow.Anonymization` | You mask, pseudonymize, export, erase, or fingerprint values. | Register an `ITokenStore` for reversible tokens; use services/extensions. | Masking/pseudonymization are still personal data. | | `SensitiveFlow.Json` | You need automatic response serialization redaction. | `WithSensitiveDataRedaction()` on `System.Text.Json`. | Does not cover Newtonsoft.Json. | -| `SensitiveFlow.Logging` | You need sensitive log value redaction. | `AddSensitiveFlowLogging()` and/or provider wrapper. | Not semantic PII detection; values must flow through known redaction paths. | +| `SensitiveFlow.Logging` | You need sensitive log value redaction. | `AddSensitiveFlowLogging()` and/or provider wrapper. | Not semantic PII detection; scalar values still need explicit markers or structured metadata. | | `SensitiveFlow.Diagnostics` | You want OpenTelemetry spans/metrics. | `AddSensitiveFlowDiagnostics()` after audit store/decorators. | Decorator order changes what latency is measured. | | `SensitiveFlow.HealthChecks` | You want ASP.NET Core health checks for SensitiveFlow infrastructure. | `AddSensitiveFlowHealthChecks().AddAuditStoreCheck().AddTokenStoreCheck()`. | Token stores without `IHealthProbe` are resolution-only checks to avoid mutating data. | | `SensitiveFlow.Retention` | You evaluate retention policies. | `AddRetention()` / `AddRetentionExecutor()` and run a scheduled job. | It will not delete database rows automatically. | | `SensitiveFlow.Analyzers` | You want compile-time guardrails. | Add analyzer package to application projects. | Warnings still require engineering judgment. | | `SensitiveFlow.SourceGenerators` | You want generated sensitive metadata. | Add source generator package. | Keep generator tests aligned with reflection fallback. | | `SensitiveFlow.TestKit` | You implement custom stores or leak tests. | Inherit contract tests. | Contract tests need isolated fresh stores. | -| `SensitiveFlow.Tool` | You want CI/documentation reports from annotated assemblies. | `dotnet tool install SensitiveFlow.Tool`; run `sensitiveflow scan `. | Scans compiled assemblies, not source directories. | +| `SensitiveFlow.Tool` | You want CI/documentation reports from annotated assemblies. | `dotnet tool install SensitiveFlow.Tool`; run `sensitiveflow scan `. | Scans compiled assemblies, not source directories. | ## SensitiveFlow.Core @@ -265,6 +265,7 @@ Operational notes: - Only anonymization may remove data from personal-data scope. Masking and pseudonymization remain personal data. - `TokenPseudonymizer` is reversible only if `ITokenStore` remains durable. - `CachingTokenStore` stores original values in process memory. +- `DataSubjectExporter` returns raw values by default and only masks/redacts/omits fields that opt in with `[Redaction(Export = ...)]`. ## SensitiveFlow.Json @@ -309,6 +310,7 @@ Primary APIs: - `AddSensitiveFlowLogging(...)` - `AddSensitiveFlowLogging(...)` - `RedactingLoggerProvider` +- `SensitiveLoggingOptions` - `ISensitiveValueRedactor` - `DefaultSensitiveValueRedactor` @@ -319,7 +321,10 @@ Install when: Operational notes: - This is not full semantic PII detection. -- Prefer structured logging and pass values through known redaction paths. +- `[Sensitive]` template markers are always redacted. +- Structured object values with `[PersonalData]` or `[SensitiveData]` members are redacted by default. +- Pass `SensitiveLoggingOptions.Policies` to make `.MaskInLogs()` category policies mask annotated structured object members. +- Prefer structured logging and pass scalar values through known redaction paths. ## SensitiveFlow.Diagnostics diff --git a/samples/MinimalApi.Sample/Program.cs b/samples/MinimalApi.Sample/Program.cs index 68a970b..f7569bc 100644 --- a/samples/MinimalApi.Sample/Program.cs +++ b/samples/MinimalApi.Sample/Program.cs @@ -85,7 +85,10 @@ }); builder.Services.AddSingleton(sensitiveFlowOptions); - builder.Services.AddSensitiveFlowLogging(); + builder.Services.AddSensitiveFlowLogging(options => + { + options.Policies = sensitiveFlowOptions.Policies; + }); builder.Services.AddSensitiveFlowEFCore(); builder.Services.AddSensitiveFlowAspNetCore(); builder.Services.AddSensitiveFlowValidation(options => diff --git a/samples/WebApi.Sample/Program.cs b/samples/WebApi.Sample/Program.cs index 6428f28..b179121 100644 --- a/samples/WebApi.Sample/Program.cs +++ b/samples/WebApi.Sample/Program.cs @@ -80,7 +80,10 @@ }); builder.Services.AddSingleton(sensitiveFlowOptions); - builder.Services.AddSensitiveFlowLogging(); + builder.Services.AddSensitiveFlowLogging(options => + { + options.Policies = sensitiveFlowOptions.Policies; + }); builder.Services.AddSensitiveFlowEFCore(); builder.Services.AddSensitiveFlowAspNetCore(); builder.Services.AddSensitiveFlowValidation(options => diff --git a/src/SensitiveFlow.Anonymization/Export/DataSubjectExporter.cs b/src/SensitiveFlow.Anonymization/Export/DataSubjectExporter.cs index 5933554..fa55d68 100644 --- a/src/SensitiveFlow.Anonymization/Export/DataSubjectExporter.cs +++ b/src/SensitiveFlow.Anonymization/Export/DataSubjectExporter.cs @@ -1,3 +1,7 @@ +using System.Reflection; +using SensitiveFlow.Core.Attributes; +using SensitiveFlow.Core.Enums; +using SensitiveFlow.Core.Profiles; using SensitiveFlow.Core.Reflection; namespace SensitiveFlow.Anonymization.Export; @@ -29,7 +33,7 @@ public sealed class DataSubjectExporter : IDataSubjectExporter continue; } - result[property.Name] = property.GetValue(entity); + AddExportValue(result, entity, property); } foreach (var entry in retention) @@ -39,9 +43,123 @@ public sealed class DataSubjectExporter : IDataSubjectExporter continue; } - result[entry.Property.Name] = entry.Property.GetValue(entity); + AddExportValue(result, entity, entry.Property); } return result; } + + private static void AddExportValue(Dictionary result, object entity, PropertyInfo property) + { + var action = property.GetCustomAttribute(inherit: true) + ?.ForContext(RedactionContext.Export) + ?? OutputRedactionAction.None; + + if (action == OutputRedactionAction.Omit) + { + return; + } + + var value = property.GetValue(entity); + result[property.Name] = action switch + { + OutputRedactionAction.Redact => SensitiveFlowDefaults.RedactedPlaceholder, + OutputRedactionAction.Mask => MaskValue(value, property), + _ => value, + }; + } + + private static object? MaskValue(object? value, PropertyInfo property) + { + if (value is not string text) + { + return null; + } + + if (text.Length == 0) + { + return text; + } + + var kind = InferMaskKind(property.Name); + return kind switch + { + MaskKind.Email => MaskEmail(text), + MaskKind.Phone => MaskPhone(text), + MaskKind.Name => MaskName(text), + _ => GenericMask(text), + }; + } + + private static MaskKind InferMaskKind(string propertyName) + { + if (propertyName.Contains("Email", StringComparison.OrdinalIgnoreCase)) + { + return MaskKind.Email; + } + + if (propertyName.Contains("Phone", StringComparison.OrdinalIgnoreCase)) + { + return MaskKind.Phone; + } + + if (propertyName.Contains("Name", StringComparison.OrdinalIgnoreCase)) + { + return MaskKind.Name; + } + + return MaskKind.Generic; + } + + private static string MaskEmail(string value) + { + var at = value.IndexOf('@', StringComparison.Ordinal); + if (at <= 1) + { + return GenericMask(value); + } + + return value[0] + new string('*', at - 1) + value[at..]; + } + + private static string MaskPhone(string value) + { + var chars = value.ToCharArray(); + var digitsSeenFromEnd = 0; + for (var i = chars.Length - 1; i >= 0; i--) + { + if (!char.IsDigit(chars[i])) + { + continue; + } + + digitsSeenFromEnd++; + if (digitsSeenFromEnd > 2) + { + chars[i] = '*'; + } + } + + return new string(chars); + } + + private static string MaskName(string value) + => string.Join(" ", value.Split(' ', StringSplitOptions.RemoveEmptyEntries).Select(GenericMask)); + + private static string GenericMask(string value) + { + if (value.Length == 1) + { + return "*"; + } + + return string.Create(value.Length, value, static (span, source) => + { + span[0] = source[0]; + for (var i = 1; i < span.Length; i++) + { + span[i] = '*'; + } + }); + } } diff --git a/src/SensitiveFlow.Core/Profiles/SensitiveFlowDefaults.cs b/src/SensitiveFlow.Core/Profiles/SensitiveFlowDefaults.cs index 0d85108..304fad2 100644 --- a/src/SensitiveFlow.Core/Profiles/SensitiveFlowDefaults.cs +++ b/src/SensitiveFlow.Core/Profiles/SensitiveFlowDefaults.cs @@ -1,22 +1,36 @@ namespace SensitiveFlow.Core.Profiles; /// -/// Documented defaults shared across SensitiveFlow packages. +/// Public defaults used by SensitiveFlow packages when callers do not provide +/// explicit options. /// public static class SensitiveFlowDefaults { - /// Default profile used when callers do not select one explicitly. + /// + /// Default profile used when callers do not select one explicitly. + /// The default is . + /// public const SensitiveFlowProfile Profile = SensitiveFlowProfile.Balanced; - /// Default marker used for full redaction. + /// + /// Default replacement marker used by JSON and logging redaction. + /// The default value is [REDACTED]. + /// public const string RedactedPlaceholder = "[REDACTED]"; - /// Default marker used by retention anonymization. + /// + /// Default marker used when retention anonymization replaces a string value. + /// The default value is [ANONYMIZED]. + /// public const string AnonymousValue = "[ANONYMIZED]"; - /// Default health check name for audit stores. + /// + /// Default health-check registration name for audit-store checks. + /// public const string AuditStoreHealthCheckName = "sensitiveflow-audit-store"; - /// Default health check name for token stores. + /// + /// Default health-check registration name for token-store checks. + /// public const string TokenStoreHealthCheckName = "sensitiveflow-token-store"; } diff --git a/src/SensitiveFlow.Core/Profiles/SensitiveFlowOptions.cs b/src/SensitiveFlow.Core/Profiles/SensitiveFlowOptions.cs index 60400fd..d840892 100644 --- a/src/SensitiveFlow.Core/Profiles/SensitiveFlowOptions.cs +++ b/src/SensitiveFlow.Core/Profiles/SensitiveFlowOptions.cs @@ -8,7 +8,10 @@ namespace SensitiveFlow.Core.Profiles; /// public sealed class SensitiveFlowOptions { - /// Gets the active profile. + /// + /// Gets the active profile. The default is + /// (). + /// public SensitiveFlowProfile Profile { get; private set; } = SensitiveFlowDefaults.Profile; /// Gets the policy registry. diff --git a/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs index a0b844b..a91dda7 100644 --- a/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs +++ b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs @@ -1,5 +1,7 @@ using Microsoft.Extensions.DependencyInjection; using SensitiveFlow.Core.Interfaces; +using SensitiveFlow.Core.Policies; +using SensitiveFlow.Core.Profiles; namespace SensitiveFlow.Diagnostics.Validation; @@ -50,9 +52,60 @@ public SensitiveFlowConfigurationReport Validate(IServiceProvider services) diagnostics.Add(Warning("SF-CONFIG-005", "Retention validation was requested, but no retention services were detected.")); } + var sensitiveFlowOptions = provider.GetService(); + if (sensitiveFlowOptions is not null) + { + AddPolicyDiagnostics(provider, sensitiveFlowOptions, diagnostics); + } + return new SensitiveFlowConfigurationReport(diagnostics); } + private static void AddPolicyDiagnostics( + IServiceProvider provider, + SensitiveFlowOptions options, + ICollection diagnostics) + { + if (options.Policies.Rules.Any(static r => HasAction(r, SensitiveFlowPolicyAction.RequireAudit)) + && provider.GetService() is null) + { + diagnostics.Add(Warning("SF-CONFIG-006", "SensitiveFlow policies require audit support, but no IAuditStore registration was found.")); + } + + if (options.Policies.Rules.Any(static r => HasAction(r, SensitiveFlowPolicyAction.RedactInJson | SensitiveFlowPolicyAction.OmitInJson)) + && !HasRegisteredOptions(provider, "SensitiveFlow.Json.Configuration.JsonRedactionOptions, SensitiveFlow.Json")) + { + diagnostics.Add(Warning("SF-CONFIG-007", "SensitiveFlow policies configure JSON redaction, but JsonRedactionOptions were not found.")); + } + + if (options.Policies.Rules.Any(static r => HasAction(r, SensitiveFlowPolicyAction.MaskInLogs)) + && !HasRegisteredType(provider, "SensitiveFlow.Logging.Redaction.ISensitiveValueRedactor, SensitiveFlow.Logging")) + { + diagnostics.Add(Warning("SF-CONFIG-008", "SensitiveFlow policies configure log masking, but ISensitiveValueRedactor was not found.")); + } + } + + private static bool HasAction(SensitiveFlowPolicyRule rule, SensitiveFlowPolicyAction action) + => (rule.Actions & action) != SensitiveFlowPolicyAction.None; + + private static bool HasRegisteredType(IServiceProvider provider, string assemblyQualifiedTypeName) + { + var type = Type.GetType(assemblyQualifiedTypeName); + return type is not null && provider.GetService(type) is not null; + } + + private static bool HasRegisteredOptions(IServiceProvider provider, string assemblyQualifiedTypeName) + { + var type = Type.GetType(assemblyQualifiedTypeName); + if (type is null) + { + return false; + } + + var optionsType = typeof(Microsoft.Extensions.Options.IOptions<>).MakeGenericType(type); + return provider.GetService(optionsType) is not null; + } + private static SensitiveFlowConfigurationDiagnostic Warning(string code, string message) { return new SensitiveFlowConfigurationDiagnostic @@ -63,4 +116,3 @@ private static SensitiveFlowConfigurationDiagnostic Warning(string code, string }; } } - diff --git a/src/SensitiveFlow.Json/Configuration/JsonRedactionOptions.cs b/src/SensitiveFlow.Json/Configuration/JsonRedactionOptions.cs index 75360f5..311cb5b 100644 --- a/src/SensitiveFlow.Json/Configuration/JsonRedactionOptions.cs +++ b/src/SensitiveFlow.Json/Configuration/JsonRedactionOptions.cs @@ -19,7 +19,7 @@ public sealed class JsonRedactionOptions /// /// Replacement string used by and as the /// fallback for non-string values in . - /// Defaults to "[REDACTED]". + /// Defaults to . /// public string RedactedPlaceholder { get; set; } = SensitiveFlowDefaults.RedactedPlaceholder; diff --git a/src/SensitiveFlow.Logging/Configuration/SensitiveLoggingOptions.cs b/src/SensitiveFlow.Logging/Configuration/SensitiveLoggingOptions.cs new file mode 100644 index 0000000..9049440 --- /dev/null +++ b/src/SensitiveFlow.Logging/Configuration/SensitiveLoggingOptions.cs @@ -0,0 +1,36 @@ +using SensitiveFlow.Core.Enums; +using SensitiveFlow.Core.Policies; +using SensitiveFlow.Core.Profiles; + +namespace SensitiveFlow.Logging.Configuration; + +/// +/// Options used by SensitiveFlow logging redaction. +/// +public sealed class SensitiveLoggingOptions +{ + /// + /// Replacement marker used for explicit redaction. The default value is + /// . + /// + public string RedactedPlaceholder { get; set; } = SensitiveFlowDefaults.RedactedPlaceholder; + + /// + /// Optional category policy registry. When present, properties in categories + /// configured with are masked + /// before they reach the inner logger. + /// + public SensitiveFlowPolicyRegistry? Policies { get; set; } + + /// + /// Controls whether structured object values are inspected for annotated members. + /// The default is true. + /// + public bool RedactAnnotatedObjects { get; set; } = true; + + /// + /// Fallback action for annotated object members when no contextual attribute or + /// policy rule applies. The default is . + /// + public OutputRedactionAction DefaultAnnotatedMemberAction { get; set; } = OutputRedactionAction.Redact; +} diff --git a/src/SensitiveFlow.Logging/Extensions/LoggingServiceCollectionExtensions.cs b/src/SensitiveFlow.Logging/Extensions/LoggingServiceCollectionExtensions.cs index fb473fa..9c11f3a 100644 --- a/src/SensitiveFlow.Logging/Extensions/LoggingServiceCollectionExtensions.cs +++ b/src/SensitiveFlow.Logging/Extensions/LoggingServiceCollectionExtensions.cs @@ -1,5 +1,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using SensitiveFlow.Core.Profiles; +using SensitiveFlow.Logging.Configuration; using SensitiveFlow.Logging.Loggers; using SensitiveFlow.Logging.Redaction; @@ -14,12 +16,26 @@ public static class LoggingServiceCollectionExtensions /// Registers the as . /// /// The service collection. - /// The redaction marker. Defaults to [REDACTED]. + /// The redaction marker. Defaults to . public static IServiceCollection AddSensitiveFlowLogging( this IServiceCollection services, - string marker = "[REDACTED]") + string marker = SensitiveFlowDefaults.RedactedPlaceholder) + => services.AddSensitiveFlowLogging(options => options.RedactedPlaceholder = marker); + + /// + /// Registers SensitiveFlow logging redaction with explicit options. + /// + /// The service collection. + /// Optional logging-redaction configuration. + public static IServiceCollection AddSensitiveFlowLogging( + this IServiceCollection services, + Action? configure) { - services.AddSingleton(new DefaultSensitiveValueRedactor(marker)); + var options = new SensitiveLoggingOptions(); + configure?.Invoke(options); + + services.AddSingleton(options); + services.AddSingleton(new DefaultSensitiveValueRedactor(options.RedactedPlaceholder)); return services; } @@ -28,17 +44,30 @@ public static IServiceCollection AddSensitiveFlowLogging( /// /// The provider to wrap with . /// The logging builder. - /// The redaction marker. Defaults to [REDACTED]. + /// The redaction marker. Defaults to . + public static ILoggingBuilder AddSensitiveFlowLogging( + this ILoggingBuilder builder, + string marker = SensitiveFlowDefaults.RedactedPlaceholder) + where TProvider : class, ILoggerProvider + => builder.AddSensitiveFlowLogging(options => options.RedactedPlaceholder = marker); + + /// + /// Registers SensitiveFlow redaction for an inner logging provider with explicit options. + /// + /// The provider to wrap with . + /// The logging builder. + /// Optional logging-redaction configuration. public static ILoggingBuilder AddSensitiveFlowLogging( this ILoggingBuilder builder, - string marker = "[REDACTED]") + Action? configure) where TProvider : class, ILoggerProvider { - builder.Services.AddSensitiveFlowLogging(marker); + builder.Services.AddSensitiveFlowLogging(configure); builder.Services.AddSingleton(sp => new RedactingLoggerProvider( ActivatorUtilities.CreateInstance(sp), - sp.GetRequiredService())); + sp.GetRequiredService(), + sp.GetRequiredService())); return builder; } diff --git a/src/SensitiveFlow.Logging/Loggers/RedactingLogger.cs b/src/SensitiveFlow.Logging/Loggers/RedactingLogger.cs index 2d7e4ea..588b294 100644 --- a/src/SensitiveFlow.Logging/Loggers/RedactingLogger.cs +++ b/src/SensitiveFlow.Logging/Loggers/RedactingLogger.cs @@ -1,7 +1,13 @@ using System.Diagnostics.Metrics; +using System.Reflection; using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; +using SensitiveFlow.Core.Attributes; using SensitiveFlow.Core.Diagnostics; +using SensitiveFlow.Core.Enums; +using SensitiveFlow.Core.Policies; +using SensitiveFlow.Core.Reflection; +using SensitiveFlow.Logging.Configuration; using SensitiveFlow.Logging.Redaction; namespace SensitiveFlow.Logging.Loggers; @@ -43,12 +49,17 @@ public sealed class RedactingLogger : ILogger private readonly ILogger _inner; private readonly ISensitiveValueRedactor _redactor; + private readonly SensitiveLoggingOptions _options; /// Initializes a new instance of . - public RedactingLogger(ILogger inner, ISensitiveValueRedactor redactor) + public RedactingLogger( + ILogger inner, + ISensitiveValueRedactor redactor, + SensitiveLoggingOptions? options = null) { _inner = inner; _redactor = redactor; + _options = options ?? new SensitiveLoggingOptions(); } /// @@ -92,8 +103,12 @@ public void Log( // No template available — render with the original formatter and redact // any [Sensitive] patterns left in the output. - var message = formatter(state, ex); - return RedactTemplate(message); + if (s is TState typedState) + { + return RedactTemplate(formatter(typedState, ex)); + } + + return RenderPairsFallback(s); }); return; } @@ -143,6 +158,11 @@ private static string RenderTemplate(string template, IEnumerable> pairs) + => string.Join(", ", pairs + .Where(static pair => pair.Key != OriginalFormatKey) + .Select(static pair => $"{pair.Key}={pair.Value}")); + private List> RedactPairs(IEnumerable> pairs) { var result = new List>(); @@ -154,6 +174,11 @@ private static string RenderTemplate(string template, IEnumerable(pair.Key, _redactor.Redact(string.Empty))); redactedCount++; } + else if (_options.RedactAnnotatedObjects && TryRedactAnnotatedObject(pair.Value, out var redactedValue, out var objectRedactedCount)) + { + result.Add(new KeyValuePair(pair.Key, redactedValue)); + redactedCount += objectRedactedCount; + } else { result.Add(pair); @@ -167,4 +192,226 @@ private static string RenderTemplate(string template, IEnumerable p.Name, StringComparer.Ordinal); + var projected = new Dictionary(StringComparer.Ordinal); + + foreach (var property in type.GetProperties(BindingFlags.Public | BindingFlags.Instance)) + { + if (!property.CanRead || property.GetIndexParameters().Length != 0) + { + continue; + } + + if (!sensitive.TryGetValue(property.Name, out var sensitiveProperty)) + { + projected[property.Name] = property.GetValue(value); + continue; + } + + var action = ResolveLogAction(sensitiveProperty); + if (action == OutputRedactionAction.None) + { + projected[property.Name] = property.GetValue(value); + continue; + } + + if (action == OutputRedactionAction.Omit) + { + redactedCount++; + continue; + } + + var raw = property.GetValue(value); + projected[property.Name] = action == OutputRedactionAction.Mask + ? MaskValue(raw, sensitiveProperty) + : _redactor.Redact(raw?.ToString() ?? string.Empty); + redactedCount++; + } + + redactedValue = projected; + return redactedCount > 0; + } + + private OutputRedactionAction ResolveLogAction(PropertyInfo property) + { + if (property.GetCustomAttribute(inherit: true) is not null) + { + return OutputRedactionAction.None; + } + + var contextual = property.GetCustomAttribute(inherit: true); + var contextualAction = contextual?.ForContext(RedactionContext.Log) ?? OutputRedactionAction.None; + if (contextualAction != OutputRedactionAction.None) + { + return contextualAction; + } + + if (property.GetCustomAttribute(inherit: true) is not null) + { + return OutputRedactionAction.Omit; + } + + if (property.GetCustomAttribute(inherit: true) is not null) + { + return OutputRedactionAction.Redact; + } + + if (property.GetCustomAttribute(inherit: true) is not null) + { + return OutputRedactionAction.Mask; + } + + var policyAction = ResolvePolicyAction(property); + if (policyAction != OutputRedactionAction.None) + { + return policyAction; + } + + return _options.DefaultAnnotatedMemberAction; + } + + private OutputRedactionAction ResolvePolicyAction(PropertyInfo property) + { + var policies = _options.Policies; + if (policies is null) + { + return OutputRedactionAction.None; + } + + var personal = property.GetCustomAttribute(inherit: true); + var personalRule = personal is null ? null : policies.Find(personal.Category); + if (HasMaskInLogs(personalRule)) + { + return OutputRedactionAction.Mask; + } + + var sensitive = property.GetCustomAttribute(inherit: true); + var sensitiveRule = sensitive is null ? null : policies.Find(sensitive.Category); + return HasMaskInLogs(sensitiveRule) ? OutputRedactionAction.Mask : OutputRedactionAction.None; + } + + private static bool HasMaskInLogs(SensitiveFlowPolicyRule? rule) + => rule is not null + && (rule.Actions & SensitiveFlowPolicyAction.MaskInLogs) == SensitiveFlowPolicyAction.MaskInLogs; + + private object? MaskValue(object? value, PropertyInfo property) + { + if (value is not string text) + { + return _redactor.Redact(value?.ToString() ?? string.Empty); + } + + if (text.Length == 0) + { + return text; + } + + var mask = property.GetCustomAttribute(inherit: true); + var kind = mask?.Kind ?? InferMaskKind(property.Name); + return kind switch + { + MaskKind.Email => MaskEmail(text), + MaskKind.Phone => MaskPhone(text), + MaskKind.Name => MaskName(text), + _ => GenericMask(text), + }; + } + + private static MaskKind InferMaskKind(string propertyName) + { + if (propertyName.Contains("Email", StringComparison.OrdinalIgnoreCase)) + { + return MaskKind.Email; + } + + if (propertyName.Contains("Phone", StringComparison.OrdinalIgnoreCase)) + { + return MaskKind.Phone; + } + + if (propertyName.Contains("Name", StringComparison.OrdinalIgnoreCase)) + { + return MaskKind.Name; + } + + return MaskKind.Generic; + } + + private static string MaskEmail(string value) + { + var at = value.IndexOf('@', StringComparison.Ordinal); + if (at <= 1) + { + return GenericMask(value); + } + + return value[0] + new string('*', at - 1) + value[at..]; + } + + private static string MaskPhone(string value) + { + var chars = value.ToCharArray(); + var digitsSeenFromEnd = 0; + for (var i = chars.Length - 1; i >= 0; i--) + { + if (!char.IsDigit(chars[i])) + { + continue; + } + + digitsSeenFromEnd++; + if (digitsSeenFromEnd > 2) + { + chars[i] = '*'; + } + } + + return new string(chars); + } + + private static string MaskName(string value) + { + var parts = value.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length == 0) + { + return value; + } + + return string.Join(" ", parts.Select(GenericMask)); + } + + private static string GenericMask(string value) + { + if (value.Length == 1) + { + return "*"; + } + + return string.Create(value.Length, value, static (span, source) => + { + span[0] = source[0]; + for (var i = 1; i < span.Length; i++) + { + span[i] = '*'; + } + }); + } } diff --git a/src/SensitiveFlow.Logging/Loggers/RedactingLoggerProvider.cs b/src/SensitiveFlow.Logging/Loggers/RedactingLoggerProvider.cs index cede7bd..42e719f 100644 --- a/src/SensitiveFlow.Logging/Loggers/RedactingLoggerProvider.cs +++ b/src/SensitiveFlow.Logging/Loggers/RedactingLoggerProvider.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.Logging; +using SensitiveFlow.Logging.Configuration; using SensitiveFlow.Logging.Redaction; namespace SensitiveFlow.Logging.Loggers; @@ -11,19 +12,24 @@ public sealed class RedactingLoggerProvider : ILoggerProvider { private readonly ILoggerProvider _innerProvider; private readonly ISensitiveValueRedactor _redactor; + private readonly SensitiveLoggingOptions _options; /// Initializes a new instance of . - public RedactingLoggerProvider(ILoggerProvider innerProvider, ISensitiveValueRedactor redactor) + public RedactingLoggerProvider( + ILoggerProvider innerProvider, + ISensitiveValueRedactor redactor, + SensitiveLoggingOptions? options = null) { _innerProvider = innerProvider; _redactor = redactor; + _options = options ?? new SensitiveLoggingOptions(); } /// public ILogger CreateLogger(string categoryName) { var inner = _innerProvider.CreateLogger(categoryName); - return new RedactingLogger(inner, _redactor); + return new RedactingLogger(inner, _redactor, _options); } /// diff --git a/src/SensitiveFlow.Logging/Redaction/DefaultSensitiveValueRedactor.cs b/src/SensitiveFlow.Logging/Redaction/DefaultSensitiveValueRedactor.cs index 94ecfd9..e41cf92 100644 --- a/src/SensitiveFlow.Logging/Redaction/DefaultSensitiveValueRedactor.cs +++ b/src/SensitiveFlow.Logging/Redaction/DefaultSensitiveValueRedactor.cs @@ -1,3 +1,5 @@ +using SensitiveFlow.Core.Profiles; + namespace SensitiveFlow.Logging.Redaction; /// @@ -9,9 +11,9 @@ public sealed class DefaultSensitiveValueRedactor : ISensitiveValueRedactor /// /// Initializes a new instance using the given replacement marker. - /// Defaults to [REDACTED]. + /// Defaults to . /// - public DefaultSensitiveValueRedactor(string marker = "[REDACTED]") + public DefaultSensitiveValueRedactor(string marker = SensitiveFlowDefaults.RedactedPlaceholder) { ArgumentException.ThrowIfNullOrWhiteSpace(marker); _marker = marker; diff --git a/src/SensitiveFlow.Retention/Services/RetentionExecutorOptions.cs b/src/SensitiveFlow.Retention/Services/RetentionExecutorOptions.cs index 0bedd51..32cf001 100644 --- a/src/SensitiveFlow.Retention/Services/RetentionExecutorOptions.cs +++ b/src/SensitiveFlow.Retention/Services/RetentionExecutorOptions.cs @@ -20,7 +20,8 @@ public sealed class RetentionExecutorOptions /// /// Default placeholder used by for string properties. - /// Value types collapse to default(T). + /// The default value is . Value types + /// collapse to default(T). /// public string AnonymousStringMarker { get; set; } = SensitiveFlowDefaults.AnonymousValue; diff --git a/tests/SensitiveFlow.Anonymization.Tests/Export/DataSubjectExporterTests.cs b/tests/SensitiveFlow.Anonymization.Tests/Export/DataSubjectExporterTests.cs index b3883bf..200c3c3 100644 --- a/tests/SensitiveFlow.Anonymization.Tests/Export/DataSubjectExporterTests.cs +++ b/tests/SensitiveFlow.Anonymization.Tests/Export/DataSubjectExporterTests.cs @@ -53,6 +53,16 @@ public void Export_SkipsUnreadableAndDuplicateRetentionProperties() exported.Should().NotContainKey("WriteOnly"); } + [Fact] + public void Export_RespectsContextualExportRedaction() + { + var exported = _exporter.Export(new ContextualExportCustomer()); + + exported.Should().ContainKey("Email").WhoseValue.Should().Be("m****@example.com"); + exported.Should().ContainKey("TaxId").WhoseValue.Should().Be("[REDACTED]"); + exported.Should().NotContainKey("SecretNote"); + } + [Fact] public void Export_OnNullEntity_Throws() { @@ -94,4 +104,19 @@ public string WriteOnly set { _ = value; } } } + + public class ContextualExportCustomer + { + [PersonalData(Category = DataCategory.Contact)] + [Redaction(Export = OutputRedactionAction.Mask)] + public string Email { get; set; } = "maria@example.com"; + + [SensitiveData(Category = SensitiveDataCategory.Other)] + [Redaction(Export = OutputRedactionAction.Redact)] + public string TaxId { get; set; } = "12345678900"; + + [PersonalData(Category = DataCategory.Other)] + [Redaction(Export = OutputRedactionAction.Omit)] + public string SecretNote { get; set; } = "hide me"; + } } diff --git a/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs b/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs index ab18d2e..074d544 100644 --- a/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs +++ b/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs @@ -1,7 +1,9 @@ using FluentAssertions; using Microsoft.Extensions.DependencyInjection; +using SensitiveFlow.Core.Enums; using SensitiveFlow.Core.Interfaces; using SensitiveFlow.Core.Models; +using SensitiveFlow.Core.Profiles; using SensitiveFlow.Diagnostics.Extensions; using SensitiveFlow.Diagnostics.Validation; @@ -63,6 +65,48 @@ public void ValidateSensitiveFlow_WithoutRegisteredValidator_UsesDefaultValidato report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-001"); } + [Fact] + public void ValidateSensitiveFlow_WhenPolicyRequiresAuditAndStoreMissing_ReportsWarning() + { + var services = new ServiceCollection(); + var options = new SensitiveFlowOptions(); + options.Policies.ForSensitiveCategory(SensitiveDataCategory.Other).RequireAudit(); + services.AddSingleton(options); + services.AddSensitiveFlowValidation(o => o.RequireAuditStore = false); + + var report = services.BuildServiceProvider().ValidateSensitiveFlow(); + + report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-006"); + } + + [Fact] + public void ValidateSensitiveFlow_WhenPolicyRequiresJsonAndJsonOptionsMissing_ReportsWarning() + { + var services = new ServiceCollection(); + var options = new SensitiveFlowOptions(); + options.Policies.ForCategory(DataCategory.Contact).RedactInJson(); + services.AddSingleton(options); + services.AddSensitiveFlowValidation(o => o.RequireAuditStore = false); + + var report = services.BuildServiceProvider().ValidateSensitiveFlow(); + + report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-007"); + } + + [Fact] + public void ValidateSensitiveFlow_WhenPolicyRequiresLogMaskingAndRedactorMissing_ReportsWarning() + { + var services = new ServiceCollection(); + var options = new SensitiveFlowOptions(); + options.Policies.ForCategory(DataCategory.Contact).MaskInLogs(); + services.AddSingleton(options); + services.AddSensitiveFlowValidation(o => o.RequireAuditStore = false); + + var report = services.BuildServiceProvider().ValidateSensitiveFlow(); + + report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-008"); + } + private sealed class HealthyAuditStore : IAuditStore { public Task AppendAsync(AuditRecord record, CancellationToken cancellationToken = default) diff --git a/tests/SensitiveFlow.Logging.Tests/LoggingExtensionsTests.cs b/tests/SensitiveFlow.Logging.Tests/LoggingExtensionsTests.cs index 404cf30..102bbe6 100644 --- a/tests/SensitiveFlow.Logging.Tests/LoggingExtensionsTests.cs +++ b/tests/SensitiveFlow.Logging.Tests/LoggingExtensionsTests.cs @@ -2,6 +2,9 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using NSubstitute; +using SensitiveFlow.Core.Enums; +using SensitiveFlow.Core.Policies; +using SensitiveFlow.Logging.Configuration; using SensitiveFlow.Logging.Extensions; using SensitiveFlow.Logging.Loggers; using SensitiveFlow.Logging.Redaction; @@ -59,6 +62,23 @@ public void AddSensitiveFlowLogging_CustomMarker_UsedByRedactor() redactor.Redact("anything").Should().Be("***"); } + [Fact] + public void AddSensitiveFlowLogging_Options_RegistersPolicyRegistry() + { + var services = new ServiceCollection(); + var policies = new SensitiveFlowPolicyRegistry(); + policies.ForCategory(DataCategory.Contact).MaskInLogs(); + + services.AddSensitiveFlowLogging(options => + { + options.Policies = policies; + }); + + var provider = services.BuildServiceProvider(); + + provider.GetRequiredService().Policies.Should().BeSameAs(policies); + } + [Fact] public void AddSensitiveFlowLogging_ReturnsSameServiceCollection() { diff --git a/tests/SensitiveFlow.Logging.Tests/RedactingLoggerStructuredTests.cs b/tests/SensitiveFlow.Logging.Tests/RedactingLoggerStructuredTests.cs index 0f8540f..fc2acf3 100644 --- a/tests/SensitiveFlow.Logging.Tests/RedactingLoggerStructuredTests.cs +++ b/tests/SensitiveFlow.Logging.Tests/RedactingLoggerStructuredTests.cs @@ -1,6 +1,10 @@ using FluentAssertions; using Microsoft.Extensions.Logging; using NSubstitute; +using SensitiveFlow.Core.Attributes; +using SensitiveFlow.Core.Enums; +using SensitiveFlow.Core.Policies; +using SensitiveFlow.Logging.Configuration; using SensitiveFlow.Logging.Loggers; using SensitiveFlow.Logging.Redaction; @@ -176,6 +180,72 @@ public void Log_StructuredState_FormatterIsInvokedWithRedactedState() spy.LastMessage.Should().NotContain("joao@example.com"); } + [Fact] + public void Log_StructuredObject_DefaultRedactsAnnotatedMembers() + { + var spy = new StateSpyLogger(); + var logger = new RedactingLogger(spy, new DefaultSensitiveValueRedactor()); + + logger.Log(LogLevel.Information, new EventId(0), + new List> + { + new("Customer", new CustomerLogShape()), + }, + null, + (s, _) => string.Join(", ", s.Select(kv => $"{kv.Key}={kv.Value}"))); + + var customer = spy.LastState.Should().ContainSingle(kv => kv.Key == "Customer").Subject.Value; + var projected = customer.Should().BeAssignableTo>().Subject; + projected["Email"].Should().Be("[REDACTED]"); + projected["PublicNote"].Should().Be("visible"); + } + + [Fact] + public void Log_StructuredObject_PolicyMasksCategoryInLogs() + { + var policies = new SensitiveFlowPolicyRegistry(); + policies.ForCategory(DataCategory.Contact).MaskInLogs(); + + var spy = new StateSpyLogger(); + var logger = new RedactingLogger( + spy, + new DefaultSensitiveValueRedactor(), + new SensitiveLoggingOptions { Policies = policies }); + + logger.Log(LogLevel.Information, new EventId(0), + new List> + { + new("Customer", new CustomerLogShape()), + }, + null, + (s, _) => string.Empty); + + var projected = spy.LastState.Single(kv => kv.Key == "Customer").Value + .Should().BeAssignableTo>().Subject; + projected["Email"].Should().Be("m****@example.com"); + projected["TaxId"].Should().Be("[REDACTED]"); + } + + [Fact] + public void Log_StructuredObject_ContextualLogActionCanOmit() + { + var spy = new StateSpyLogger(); + var logger = new RedactingLogger(spy, new DefaultSensitiveValueRedactor()); + + logger.Log(LogLevel.Information, new EventId(0), + new List> + { + new("Customer", new ContextualLogShape()), + }, + null, + (s, _) => string.Empty); + + var projected = spy.LastState.Single(kv => kv.Key == "Customer").Value + .Should().BeAssignableTo>().Subject; + projected.Should().NotContainKey("TaxId"); + projected["Email"].Should().Be("m****@example.com"); + } + private sealed class SpyLogger : ILogger { public string? LastMessage { get; private set; } @@ -191,4 +261,48 @@ public void Log( LastMessage = formatter(state, exception); } } + + private sealed class StateSpyLogger : ILogger + { + public IReadOnlyList> LastState { get; private set; } = []; + + public bool IsEnabled(LogLevel logLevel) => true; + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + if (state is IEnumerable> pairs) + { + LastState = pairs.ToArray(); + } + } + } + + private sealed class CustomerLogShape + { + [PersonalData(Category = DataCategory.Contact)] + public string Email { get; set; } = "maria@example.com"; + + [SensitiveData(Category = SensitiveDataCategory.Other)] + public string TaxId { get; set; } = "12345678900"; + + public string PublicNote { get; set; } = "visible"; + } + + private sealed class ContextualLogShape + { + [PersonalData(Category = DataCategory.Contact)] + [Redaction(Logs = OutputRedactionAction.Mask)] + public string Email { get; set; } = "maria@example.com"; + + [SensitiveData(Category = SensitiveDataCategory.Other)] + [Redaction(Logs = OutputRedactionAction.Omit)] + public string TaxId { get; set; } = "12345678900"; + } } From 7add54cfd3a82f69c5491e55dac42db58c35b75d Mon Sep 17 00:00:00 2001 From: Mateus Silva Date: Mon, 11 May 2026 07:31:49 -0300 Subject: [PATCH 05/23] feat: introduce contextual audit redaction and outbox support - Added support for audit redaction context in `SensitiveDataAuditInterceptor` to handle sensitive fields with contextual redaction attributes. - Implemented `OutboxAuditStore` to append audit records to a primary store and enqueue them for asynchronous processing. - Created `InMemoryAuditOutbox` for testing and local development, along with a JSON serializer for outbox payloads. - Enhanced `SensitiveFlow.Tool` to build projects before scanning and updated usage instructions. - Added diagnostics for ASP.NET Core middleware registration and audit configuration validation. - Introduced tests for audit outbox functionality and contextual redaction behavior. --- CHANGELOG.md | 5 + README.md | 2 +- docs/audit.md | 27 +++++ docs/package-reference.md | 7 +- docs/policies-discovery-health.md | 13 ++- ...sitiveFlowAspNetCorePipelineDiagnostics.cs | 30 +++++ .../AspNetCoreServiceCollectionExtensions.cs | 5 + .../SensitiveFlowAuditMiddleware.cs | 13 ++- .../Decorators/OutboxAuditStore.cs | 70 ++++++++++++ .../AuditServiceCollectionExtensions.cs | 66 +++++++++++ .../Outbox/InMemoryAuditOutbox.cs | 26 +++++ .../Outbox/JsonAuditOutboxSerializer.cs | 28 +++++ ...owValidationServiceCollectionExtensions.cs | 17 +++ .../SensitiveFlowConfigurationValidator.cs | 87 +++++++++++++++ .../SensitiveDataAuditInterceptor.cs | 93 +++++++++++++++- .../SensitiveFlowToolRunner.cs | 103 +++++++++++++++++- .../AuditOutboxTests.cs | 91 ++++++++++++++++ .../SensitiveFlowToolRunnerTests.cs | 24 ++++ .../SensitiveFlow.Diagnostics.Tests.csproj | 5 +- .../SensitiveFlowValidationTests.cs | 73 +++++++++++++ .../SensitiveDataAuditInterceptorTests.cs | 74 +++++++++++++ 21 files changed, 845 insertions(+), 14 deletions(-) create mode 100644 src/SensitiveFlow.AspNetCore/Diagnostics/SensitiveFlowAspNetCorePipelineDiagnostics.cs create mode 100644 src/SensitiveFlow.Audit/Decorators/OutboxAuditStore.cs create mode 100644 src/SensitiveFlow.Audit/Outbox/InMemoryAuditOutbox.cs create mode 100644 src/SensitiveFlow.Audit/Outbox/JsonAuditOutboxSerializer.cs create mode 100644 tests/SensitiveFlow.Audit.Tests/AuditOutboxTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b2bde4..26c51d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,12 +13,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `SensitiveFlow.Tool` command-line project with `sensitiveflow scan [output-directory]` for JSON/Markdown discovery reports. - `SensitiveFlow.HealthChecks` package with audit/token store health checks. - `SensitiveFlow.Diagnostics` startup validation via `AddSensitiveFlowValidation(...)` and `ValidateSensitiveFlow()`. +- `SensitiveFlow.Diagnostics` happy-path `AddSensitiveFlow(...)` registration for shared profiles/policies. - `SensitiveFlow.Retention` dry-run execution via `RetentionExecutor.DryRunAsync(...)`. - `SensitiveFlow.TestKit` expanded assertions: `ContainsMaskedEmail`, `DoesNotContainRawValues`, `JsonDoesNotExposeAnnotatedProperties`, and `LogsDoNotContainSensitiveValues`. - `SensitiveFlow.TestKit` contract test bases for `IAuditSnapshotStore`, `IPseudonymizer`, `IMasker`, `IAnonymizer`, and `IRetentionExpirationHandler`. - `SensitiveFlow.Json` now honors output attributes, contextual API response redaction, and category policies before falling back to the configured default mode. - `SensitiveFlow.Logging` now redacts annotated structured object members by default and applies `.MaskInLogs()` policies when a policy registry is supplied through `SensitiveLoggingOptions`. - `DataSubjectExporter` now honors contextual `[Redaction(Export = ...)]` attributes while keeping raw export as the default. +- `SensitiveDataAuditInterceptor` now honors contextual `[Redaction(Audit = ...)]` for per-field audit records. +- `SensitiveFlow.Tool` now builds project/solution inputs before scanning compiled assemblies. +- `SensitiveFlow.Audit` now includes `JsonAuditOutboxSerializer`, `InMemoryAuditOutbox`, and `OutboxAuditStore`. +- Startup diagnostics now detect EF Core interceptor registration without `IAuditStore`, loaded retention annotations without executor/handlers, and ASP.NET Core audit middleware registration/order markers when available. - `SensitiveDataAssert.DoesNotContainAny` - checks a payload against explicit string values without requiring an annotated entity. - `SensitiveDataAssert.DoesNotLeakKnownValues` - same as `DoesNotContainAny` but accepts `IEnumerable` for readability. diff --git a/README.md b/README.md index 87eea4a..eee45c1 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ optionsBuilder.AddInterceptors(app.Services.GetRequiredService +builder.Services.AddSensitiveFlow(options => { options.UseProfile(SensitiveFlowProfile.Strict); options.Policies.ForCategory(DataCategory.Contact) diff --git a/docs/audit.md b/docs/audit.md index 702d93f..7168e07 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -137,6 +137,33 @@ If your audit store can persist multiple records in one logical operation, imple `SensitiveDataAuditInterceptor` will call `AppendRangeAsync` once per `SaveChanges` when the store supports batching. That keeps the audit write path closer to the entity save and avoids one roundtrip per sensitive field. +## Audit redaction context + +By default, audit records do not store raw field values. If a model uses contextual audit redaction, the EF Core interceptor honors it: + +```csharp +[PersonalData(Category = DataCategory.Contact)] +[Redaction(Audit = OutputRedactionAction.Mask)] +public string Email { get; set; } = string.Empty; + +[SensitiveData(Category = SensitiveDataCategory.Other)] +[Redaction(Audit = OutputRedactionAction.Omit)] +public string InternalNote { get; set; } = string.Empty; +``` + +`Audit = Omit` suppresses the per-field audit record. `Redact`, `Mask`, and `Pseudonymize` write a protected value into `AuditRecord.Details`; pseudonymization uses the registered `IPseudonymizer` when one is available and falls back to the default redaction marker otherwise. + +## Audit outbox + +`SensitiveFlow.Audit` ships a concrete in-memory outbox for tests/local development and an `OutboxAuditStore` decorator for production-owned durable outboxes: + +```csharp +builder.Services.AddAuditStore(); +builder.Services.AddAuditOutbox(); +``` + +Use `AddInMemoryAuditOutbox()` only for tests and local samples. It is not durable. + ## Retrying transient failures Audit appends sit in the hot path of `SaveChanges`. A brief network blip or lock contention against the audit store would otherwise cascade into a `SaveChangesAsync` failure. Wrap your store with the bundled `RetryingAuditStore`: diff --git a/docs/package-reference.md b/docs/package-reference.md index be14854..2c392a9 100644 --- a/docs/package-reference.md +++ b/docs/package-reference.md @@ -22,7 +22,7 @@ This document summarizes each SensitiveFlow package individually: purpose, prima | `SensitiveFlow.Analyzers` | You want compile-time guardrails. | Add analyzer package to application projects. | Warnings still require engineering judgment. | | `SensitiveFlow.SourceGenerators` | You want generated sensitive metadata. | Add source generator package. | Keep generator tests aligned with reflection fallback. | | `SensitiveFlow.TestKit` | You implement custom stores or leak tests. | Inherit contract tests. | Contract tests need isolated fresh stores. | -| `SensitiveFlow.Tool` | You want CI/documentation reports from annotated assemblies. | `dotnet tool install SensitiveFlow.Tool`; run `sensitiveflow scan `. | Scans compiled assemblies, not source directories. | +| `SensitiveFlow.Tool` | You want CI/documentation reports from annotated assemblies. | `dotnet tool install SensitiveFlow.Tool`; run `sensitiveflow scan `. | Project/source inputs are built first, then compiled assemblies are scanned. | ## SensitiveFlow.Core @@ -62,8 +62,12 @@ Primary APIs: - `AddAuditStore()` - `AddAuditStoreRetry(...)` - `AddBufferedAuditStore(...)` +- `AddAuditOutbox()` +- `AddInMemoryAuditOutbox()` - `RetryingAuditStore` - `BufferedAuditStore` +- `OutboxAuditStore` +- `JsonAuditOutboxSerializer` - `InMemoryAuditSnapshotStore` Install when: @@ -83,6 +87,7 @@ Operational notes: - `RetryingAuditStore` does not swallow exhausted failures. - `BufferedAuditStore` is advanced. It can lose records on process crash before flush. Avoid using it with scoped stores until lifetime semantics are hardened. +- `AddInMemoryAuditOutbox()` is for tests/local development. Production outboxes should implement `IAuditOutbox` durably and use `AddAuditOutbox()`. ## SensitiveFlow.Audit.EFCore diff --git a/docs/policies-discovery-health.md b/docs/policies-discovery-health.md index 5a36dc9..bfc4b6a 100644 --- a/docs/policies-discovery-health.md +++ b/docs/policies-discovery-health.md @@ -2,10 +2,10 @@ ## Policy engine and profiles -Use `SensitiveFlowOptions` when an application wants one central place to declare how categories should be handled. +Use `SensitiveFlowOptions` when an application wants one central place to declare how categories should be handled. Applications that reference `SensitiveFlow.Diagnostics` can register the shared options with `AddSensitiveFlow(...)`. ```csharp -var options = SensitiveFlowPolicyConfiguration.Create(options => +builder.Services.AddSensitiveFlow(options => { options.UseProfile(SensitiveFlowProfile.Balanced); @@ -82,10 +82,11 @@ The report includes type, member, annotation kind, category, sensitivity, and re ## CLI tool -`SensitiveFlow.Tool` exposes the same report generator for CI and documentation jobs. It accepts either one compiled assembly or a directory and scans `.dll` files recursively, skipping `obj` folders. +`SensitiveFlow.Tool` exposes the same report generator for CI and documentation jobs. It accepts a compiled assembly, a project/solution file, or a directory. When the input is a `.csproj`, `.sln`, `.slnx`, or a directory containing a single project/solution, the tool runs `dotnet build -c Release` first and then scans the compiled assemblies. ```bash dotnet tool install SensitiveFlow.Tool +sensitiveflow scan ./src/MyApp/MyApp.csproj ./artifacts/privacy sensitiveflow scan ./bin/Release/net10.0/MyApp.dll ./artifacts/privacy sensitiveflow scan ./src/MyApp/bin/Release/net10.0 ./artifacts/privacy ``` @@ -109,7 +110,7 @@ builder.Services.AddSensitiveFlowHealthChecks() ## Startup validation -`SensitiveFlow.Diagnostics` includes a startup self-test for common misconfiguration. +`SensitiveFlow.Diagnostics` includes a startup self-test for common misconfiguration. It checks required stores, policy-driven JSON/logging/audit requirements, EF Core audit interceptor registration, loaded retention annotations, and ASP.NET Core middleware registration markers when those packages are present. ```csharp builder.Services.AddSensitiveFlowValidation(options => @@ -126,3 +127,7 @@ Example warnings: - `SF-CONFIG-001`: no `IAuditStore` registration found. - `SF-CONFIG-002`: no durable `ITokenStore` registration found. - `SF-CONFIG-003`: `IPseudonymizer` registered without `ITokenStore`. +- `SF-CONFIG-009`: EF Core audit interceptor registered without `IAuditStore`. +- `SF-CONFIG-010`: retention annotations found without `RetentionExecutor` or handlers. +- `SF-CONFIG-011`: ASP.NET Core services registered but `UseSensitiveFlowAudit()` was not marked. +- `SF-CONFIG-012`: middleware observed an authenticated user before it ran, which can indicate it was placed after authentication. diff --git a/src/SensitiveFlow.AspNetCore/Diagnostics/SensitiveFlowAspNetCorePipelineDiagnostics.cs b/src/SensitiveFlow.AspNetCore/Diagnostics/SensitiveFlowAspNetCorePipelineDiagnostics.cs new file mode 100644 index 0000000..3a5ec37 --- /dev/null +++ b/src/SensitiveFlow.AspNetCore/Diagnostics/SensitiveFlowAspNetCorePipelineDiagnostics.cs @@ -0,0 +1,30 @@ +namespace SensitiveFlow.AspNetCore.Diagnostics; + +/// +/// Runtime state used by startup diagnostics to report how the ASP.NET Core +/// SensitiveFlow middleware was wired. +/// +public sealed class SensitiveFlowAspNetCorePipelineDiagnostics +{ + /// Gets whether UseSensitiveFlowAudit() was added to the pipeline. + public bool AuditMiddlewareRegistered { get; private set; } + + /// + /// Gets whether the audit middleware observed an already-authenticated user + /// before it ran. This can indicate the middleware was placed after + /// authentication. + /// + public bool ObservedAuthenticatedUserBeforeAuditMiddleware { get; private set; } + + /// Marks the audit middleware as registered in the pipeline. + public void MarkAuditMiddlewareRegistered() + { + AuditMiddlewareRegistered = true; + } + + /// Marks that the middleware ran after authentication for at least one request. + public void MarkAuthenticatedUserObserved() + { + ObservedAuthenticatedUserBeforeAuditMiddleware = true; + } +} diff --git a/src/SensitiveFlow.AspNetCore/Extensions/AspNetCoreServiceCollectionExtensions.cs b/src/SensitiveFlow.AspNetCore/Extensions/AspNetCoreServiceCollectionExtensions.cs index f92c0d2..e23edc4 100644 --- a/src/SensitiveFlow.AspNetCore/Extensions/AspNetCoreServiceCollectionExtensions.cs +++ b/src/SensitiveFlow.AspNetCore/Extensions/AspNetCoreServiceCollectionExtensions.cs @@ -1,6 +1,7 @@ using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.DependencyInjection; using SensitiveFlow.AspNetCore.Context; +using SensitiveFlow.AspNetCore.Diagnostics; using SensitiveFlow.Core.Interfaces; namespace SensitiveFlow.AspNetCore.Extensions; @@ -18,6 +19,7 @@ public static IServiceCollection AddSensitiveFlowAspNetCore(this IServiceCollect { services.AddHttpContextAccessor(); services.AddScoped(); + services.AddSingleton(); return services; } @@ -28,6 +30,9 @@ public static IServiceCollection AddSensitiveFlowAspNetCore(this IServiceCollect /// public static IApplicationBuilder UseSensitiveFlowAudit(this IApplicationBuilder app) { + app.ApplicationServices + .GetService() + ?.MarkAuditMiddlewareRegistered(); app.UseMiddleware(); return app; } diff --git a/src/SensitiveFlow.AspNetCore/Middleware/SensitiveFlowAuditMiddleware.cs b/src/SensitiveFlow.AspNetCore/Middleware/SensitiveFlowAuditMiddleware.cs index e4f8a92..e807ff0 100644 --- a/src/SensitiveFlow.AspNetCore/Middleware/SensitiveFlowAuditMiddleware.cs +++ b/src/SensitiveFlow.AspNetCore/Middleware/SensitiveFlowAuditMiddleware.cs @@ -1,4 +1,5 @@ using Microsoft.AspNetCore.Http; +using SensitiveFlow.AspNetCore.Diagnostics; using SensitiveFlow.Core.Interfaces; namespace SensitiveFlow.AspNetCore; @@ -15,17 +16,27 @@ public sealed class SensitiveFlowAuditMiddleware private readonly RequestDelegate _next; private readonly IPseudonymizer _pseudonymizer; + private readonly SensitiveFlowAspNetCorePipelineDiagnostics? _diagnostics; /// Initializes a new instance of . - public SensitiveFlowAuditMiddleware(RequestDelegate next, IPseudonymizer pseudonymizer) + public SensitiveFlowAuditMiddleware( + RequestDelegate next, + IPseudonymizer pseudonymizer, + SensitiveFlowAspNetCorePipelineDiagnostics? diagnostics = null) { _next = next; _pseudonymizer = pseudonymizer; + _diagnostics = diagnostics; } /// Pseudonymizes the remote IP and stores the token before passing to the next middleware. public async Task InvokeAsync(HttpContext context) { + if (context.User.Identity?.IsAuthenticated == true) + { + _diagnostics?.MarkAuthenticatedUserObserved(); + } + var ip = context.Connection.RemoteIpAddress?.ToString(); if (!string.IsNullOrEmpty(ip)) { diff --git a/src/SensitiveFlow.Audit/Decorators/OutboxAuditStore.cs b/src/SensitiveFlow.Audit/Decorators/OutboxAuditStore.cs new file mode 100644 index 0000000..561b5a5 --- /dev/null +++ b/src/SensitiveFlow.Audit/Decorators/OutboxAuditStore.cs @@ -0,0 +1,70 @@ +using SensitiveFlow.Core.Interfaces; +using SensitiveFlow.Core.Models; + +namespace SensitiveFlow.Audit.Decorators; + +/// +/// Audit-store decorator that appends to the primary store and then enqueues the +/// same record in an audit outbox for asynchronous downstream delivery. +/// +public sealed class OutboxAuditStore : IBatchAuditStore +{ + private readonly IAuditStore _inner; + private readonly IAuditOutbox _outbox; + + /// Initializes a new instance. + public OutboxAuditStore(IAuditStore inner, IAuditOutbox outbox) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + _outbox = outbox ?? throw new ArgumentNullException(nameof(outbox)); + } + + /// + public async Task AppendAsync(AuditRecord record, CancellationToken cancellationToken = default) + { + await _inner.AppendAsync(record, cancellationToken); + await _outbox.EnqueueAsync(record, cancellationToken); + } + + /// + public async Task AppendRangeAsync(IReadOnlyCollection records, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(records); + + if (_inner is IBatchAuditStore batch) + { + await batch.AppendRangeAsync(records, cancellationToken); + } + else + { + foreach (var record in records) + { + await _inner.AppendAsync(record, cancellationToken); + } + } + + foreach (var record in records) + { + await _outbox.EnqueueAsync(record, cancellationToken); + } + } + + /// + public Task> QueryAsync( + DateTimeOffset? from = null, + DateTimeOffset? to = null, + int skip = 0, + int take = 100, + CancellationToken cancellationToken = default) + => _inner.QueryAsync(from, to, skip, take, cancellationToken); + + /// + public Task> QueryByDataSubjectAsync( + string dataSubjectId, + DateTimeOffset? from = null, + DateTimeOffset? to = null, + int skip = 0, + int take = 100, + CancellationToken cancellationToken = default) + => _inner.QueryByDataSubjectAsync(dataSubjectId, from, to, skip, take, cancellationToken); +} diff --git a/src/SensitiveFlow.Audit/Extensions/AuditServiceCollectionExtensions.cs b/src/SensitiveFlow.Audit/Extensions/AuditServiceCollectionExtensions.cs index f670bec..87ffa0e 100644 --- a/src/SensitiveFlow.Audit/Extensions/AuditServiceCollectionExtensions.cs +++ b/src/SensitiveFlow.Audit/Extensions/AuditServiceCollectionExtensions.cs @@ -1,6 +1,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using SensitiveFlow.Audit.Decorators; +using SensitiveFlow.Audit.Outbox; using SensitiveFlow.Core.Interfaces; namespace SensitiveFlow.Audit.Extensions; @@ -159,4 +160,69 @@ public static IServiceCollection AddBufferedAuditStore( return services; } + + /// + /// Registers an in-memory audit outbox and wraps the registered + /// with . + /// + /// + /// The in-memory outbox is intended for tests and local development. Production + /// systems should provide a durable implementation + /// and call . + /// + public static IServiceCollection AddInMemoryAuditOutbox(this IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + return services.AddAuditOutboxDecorator(); + } + + /// + /// Registers a custom durable audit outbox and wraps the registered + /// with . + /// + public static IServiceCollection AddAuditOutbox(this IServiceCollection services) + where TOutbox : class, IAuditOutbox + { + services.AddSingleton(); + services.AddSingleton(); + return services.AddAuditOutboxDecorator(); + } + + private static IServiceCollection AddAuditOutboxDecorator(this IServiceCollection services) + { + var existing = services.FirstOrDefault(d => d.ServiceType == typeof(IAuditStore)) + ?? throw new InvalidOperationException( + $"No {nameof(IAuditStore)} registration was found. Register an audit store before adding an audit outbox."); + + services.Remove(existing); + + if (existing.ImplementationType is not null) + { + services.Add(new ServiceDescriptor(existing.ImplementationType, existing.ImplementationType, existing.Lifetime)); + services.Add(new ServiceDescriptor(typeof(IAuditStore), + sp => new OutboxAuditStore( + (IAuditStore)sp.GetRequiredService(existing.ImplementationType), + sp.GetRequiredService()), + existing.Lifetime)); + } + else if (existing.ImplementationFactory is not null) + { + var factory = existing.ImplementationFactory; + services.Add(new ServiceDescriptor(typeof(IAuditStore), + sp => new OutboxAuditStore( + (IAuditStore)factory(sp), + sp.GetRequiredService()), + existing.Lifetime)); + } + else if (existing.ImplementationInstance is IAuditStore instance) + { + services.AddSingleton(sp => new OutboxAuditStore( + instance, + sp.GetRequiredService())); + } + + return services; + } } diff --git a/src/SensitiveFlow.Audit/Outbox/InMemoryAuditOutbox.cs b/src/SensitiveFlow.Audit/Outbox/InMemoryAuditOutbox.cs new file mode 100644 index 0000000..8169521 --- /dev/null +++ b/src/SensitiveFlow.Audit/Outbox/InMemoryAuditOutbox.cs @@ -0,0 +1,26 @@ +using System.Collections.Concurrent; +using SensitiveFlow.Core.Interfaces; +using SensitiveFlow.Core.Models; + +namespace SensitiveFlow.Audit.Outbox; + +/// +/// In-memory audit outbox implementation for tests, samples, and local development. +/// It is not durable and should not be used as the only production outbox. +/// +public sealed class InMemoryAuditOutbox : IAuditOutbox +{ + private readonly ConcurrentQueue _records = new(); + + /// Gets a point-in-time snapshot of queued audit records. + public IReadOnlyList Records => _records.ToArray(); + + /// + public Task EnqueueAsync(AuditRecord record, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(record); + cancellationToken.ThrowIfCancellationRequested(); + _records.Enqueue(record); + return Task.CompletedTask; + } +} diff --git a/src/SensitiveFlow.Audit/Outbox/JsonAuditOutboxSerializer.cs b/src/SensitiveFlow.Audit/Outbox/JsonAuditOutboxSerializer.cs new file mode 100644 index 0000000..beb0437 --- /dev/null +++ b/src/SensitiveFlow.Audit/Outbox/JsonAuditOutboxSerializer.cs @@ -0,0 +1,28 @@ +using System.Text.Json; +using SensitiveFlow.Core.Interfaces; +using SensitiveFlow.Core.Models; + +namespace SensitiveFlow.Audit.Outbox; + +/// +/// JSON serializer for audit outbox payloads. +/// +public sealed class JsonAuditOutboxSerializer : IAuditOutboxSerializer +{ + private static readonly JsonSerializerOptions Options = new(JsonSerializerDefaults.Web); + + /// + public string Serialize(AuditRecord record) + { + ArgumentNullException.ThrowIfNull(record); + return JsonSerializer.Serialize(record, Options); + } + + /// + public AuditRecord Deserialize(string payload) + { + ArgumentException.ThrowIfNullOrWhiteSpace(payload); + return JsonSerializer.Deserialize(payload, Options) + ?? throw new InvalidOperationException("Audit outbox payload did not contain an audit record."); + } +} diff --git a/src/SensitiveFlow.Diagnostics/Extensions/SensitiveFlowValidationServiceCollectionExtensions.cs b/src/SensitiveFlow.Diagnostics/Extensions/SensitiveFlowValidationServiceCollectionExtensions.cs index a44c4ce..f7b865d 100644 --- a/src/SensitiveFlow.Diagnostics/Extensions/SensitiveFlowValidationServiceCollectionExtensions.cs +++ b/src/SensitiveFlow.Diagnostics/Extensions/SensitiveFlowValidationServiceCollectionExtensions.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using SensitiveFlow.Core.Profiles; using SensitiveFlow.Diagnostics.Validation; namespace SensitiveFlow.Diagnostics.Extensions; @@ -8,6 +9,22 @@ namespace SensitiveFlow.Diagnostics.Extensions; /// public static class SensitiveFlowValidationServiceCollectionExtensions { + /// + /// Registers the shared SensitiveFlow options object used by policies, + /// profiles, and diagnostics. + /// + public static IServiceCollection AddSensitiveFlow( + this IServiceCollection services, + Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + + var options = new SensitiveFlowOptions(); + configure?.Invoke(options); + services.AddSingleton(options); + return services; + } + /// Registers startup validation options and validator services. public static IServiceCollection AddSensitiveFlowValidation( this IServiceCollection services, diff --git a/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs index a91dda7..1e3700e 100644 --- a/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs +++ b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using SensitiveFlow.Core.Discovery; using SensitiveFlow.Core.Interfaces; using SensitiveFlow.Core.Policies; using SensitiveFlow.Core.Profiles; @@ -52,6 +53,10 @@ public SensitiveFlowConfigurationReport Validate(IServiceProvider services) diagnostics.Add(Warning("SF-CONFIG-005", "Retention validation was requested, but no retention services were detected.")); } + AddEfCoreDiagnostics(provider, diagnostics); + AddRetentionDiagnostics(provider, diagnostics); + AddAspNetCoreDiagnostics(provider, diagnostics); + var sensitiveFlowOptions = provider.GetService(); if (sensitiveFlowOptions is not null) { @@ -61,6 +66,88 @@ public SensitiveFlowConfigurationReport Validate(IServiceProvider services) return new SensitiveFlowConfigurationReport(diagnostics); } + private static void AddEfCoreDiagnostics( + IServiceProvider provider, + ICollection diagnostics) + { + var interceptorType = Type.GetType("SensitiveFlow.EFCore.Interceptors.SensitiveDataAuditInterceptor, SensitiveFlow.EFCore"); + if (interceptorType is null) + { + return; + } + + try + { + if (provider.GetService(interceptorType) is not null && provider.GetService() is null) + { + diagnostics.Add(Warning("SF-CONFIG-009", "SensitiveDataAuditInterceptor is registered, but no IAuditStore registration was found.")); + } + } + catch (InvalidOperationException ex) when (ex.Message.Contains(nameof(IAuditStore), StringComparison.Ordinal)) + { + diagnostics.Add(Warning("SF-CONFIG-009", "SensitiveDataAuditInterceptor is registered, but no IAuditStore registration was found.")); + } + } + + private static void AddRetentionDiagnostics( + IServiceProvider provider, + ICollection diagnostics) + { + var hasRetentionAnnotations = AppDomain.CurrentDomain.GetAssemblies() + .Where(static a => !a.IsDynamic) + .SelectMany(static assembly => + { + try + { + return SensitiveDataDiscovery.Scan(assembly).Entries; + } + catch + { + return []; + } + }) + .Any(static entry => entry.RetentionPolicy is not null); + + if (!hasRetentionAnnotations) + { + return; + } + + var executorType = Type.GetType("SensitiveFlow.Retention.Services.RetentionExecutor, SensitiveFlow.Retention"); + var handlerType = Type.GetType("SensitiveFlow.Retention.Contracts.IRetentionExpirationHandler, SensitiveFlow.Retention"); + var hasExecutor = executorType is not null && provider.GetService(executorType) is not null; + var hasHandlers = handlerType is not null && provider.GetServices(handlerType).Any(); + + if (!hasExecutor && !hasHandlers) + { + diagnostics.Add(Warning("SF-CONFIG-010", "Retention annotations were found in loaded assemblies, but no RetentionExecutor or IRetentionExpirationHandler registration was found.")); + } + } + + private static void AddAspNetCoreDiagnostics( + IServiceProvider provider, + ICollection diagnostics) + { + var diagnosticsType = Type.GetType("SensitiveFlow.AspNetCore.Diagnostics.SensitiveFlowAspNetCorePipelineDiagnostics, SensitiveFlow.AspNetCore"); + var aspNetCoreDiagnostics = diagnosticsType is null ? null : provider.GetService(diagnosticsType); + if (aspNetCoreDiagnostics is null) + { + return; + } + + var middlewareRegistered = (bool)(diagnosticsType!.GetProperty("AuditMiddlewareRegistered")?.GetValue(aspNetCoreDiagnostics) ?? false); + if (!middlewareRegistered) + { + diagnostics.Add(Warning("SF-CONFIG-011", "SensitiveFlow ASP.NET Core services are registered, but UseSensitiveFlowAudit() has not marked the pipeline.")); + } + + var observedAuthenticatedUser = (bool)(diagnosticsType.GetProperty("ObservedAuthenticatedUserBeforeAuditMiddleware")?.GetValue(aspNetCoreDiagnostics) ?? false); + if (observedAuthenticatedUser) + { + diagnostics.Add(Warning("SF-CONFIG-012", "SensitiveFlow audit middleware observed an already-authenticated user before it ran; it may be registered after authentication.")); + } + } + private static void AddPolicyDiagnostics( IServiceProvider provider, SensitiveFlowOptions options, diff --git a/src/SensitiveFlow.EFCore/Interceptors/SensitiveDataAuditInterceptor.cs b/src/SensitiveFlow.EFCore/Interceptors/SensitiveDataAuditInterceptor.cs index 37b1a44..8379c71 100644 --- a/src/SensitiveFlow.EFCore/Interceptors/SensitiveDataAuditInterceptor.cs +++ b/src/SensitiveFlow.EFCore/Interceptors/SensitiveDataAuditInterceptor.cs @@ -6,6 +6,7 @@ using SensitiveFlow.Core.Enums; using SensitiveFlow.Core.Interfaces; using SensitiveFlow.Core.Models; +using SensitiveFlow.Core.Profiles; using SensitiveFlow.Core.Reflection; namespace SensitiveFlow.EFCore.Interceptors; @@ -19,15 +20,20 @@ public sealed class SensitiveDataAuditInterceptor : SaveChangesInterceptor { private readonly IAuditStore _auditStore; private readonly IAuditContext _auditContext; + private readonly IPseudonymizer? _pseudonymizer; private readonly ConditionalWeakTable _pendingRecords = new(); /// /// Initializes a new instance of . /// - public SensitiveDataAuditInterceptor(IAuditStore auditStore, IAuditContext auditContext) + public SensitiveDataAuditInterceptor( + IAuditStore auditStore, + IAuditContext auditContext, + IPseudonymizer? pseudonymizer = null) { _auditStore = auditStore; _auditContext = auditContext; + _pseudonymizer = pseudonymizer; } /// @@ -136,6 +142,12 @@ private void CaptureAuditRecords(DbContext context) foreach (var property in sensitiveProperties) { + var auditAction = ResolveAuditAction(property); + if (auditAction == OutputRedactionAction.Omit) + { + continue; + } + if (entry.State == EntityState.Modified) { var propEntry = entry.Property(property.Name); @@ -153,7 +165,8 @@ private void CaptureAuditRecords(DbContext context) Operation = operation, Timestamp = timestamp, ActorId = actorId, - IpAddressToken = ipToken + IpAddressToken = ipToken, + Details = BuildAuditDetails(entry.Entity, property, auditAction) }; var pending = _pendingRecords.GetOrCreateValue(context); @@ -162,6 +175,82 @@ private void CaptureAuditRecords(DbContext context) } } + private string? BuildAuditDetails(object entity, System.Reflection.PropertyInfo property, OutputRedactionAction action) + { + if (action == OutputRedactionAction.None) + { + return null; + } + + var value = property.CanRead ? property.GetValue(entity) : null; + var protectedValue = action switch + { + OutputRedactionAction.Redact => SensitiveFlowDefaults.RedactedPlaceholder, + OutputRedactionAction.Mask => MaskValue(value, property.Name), + OutputRedactionAction.Pseudonymize => PseudonymizeValue(value), + _ => null, + }; + + return protectedValue is null + ? $"Audit redaction action: {action}." + : $"Audit redaction action: {action}; value: {protectedValue}."; + } + + private string PseudonymizeValue(object? value) + { + var text = value?.ToString(); + if (string.IsNullOrEmpty(text) || _pseudonymizer is null) + { + return SensitiveFlowDefaults.RedactedPlaceholder; + } + + return _pseudonymizer.Pseudonymize(text); + } + + private static OutputRedactionAction ResolveAuditAction(System.Reflection.PropertyInfo property) + { + var contextual = property.GetCustomAttributes(typeof(RedactionAttribute), inherit: true) + .OfType() + .FirstOrDefault(); + return contextual?.ForContext(RedactionContext.Audit) ?? OutputRedactionAction.None; + } + + private static string MaskValue(object? value, string propertyName) + { + var text = value?.ToString(); + if (string.IsNullOrEmpty(text)) + { + return string.Empty; + } + + if (propertyName.Contains("Email", StringComparison.OrdinalIgnoreCase)) + { + var at = text.IndexOf('@', StringComparison.Ordinal); + return at > 1 + ? text[0] + new string('*', at - 1) + text[at..] + : GenericMask(text); + } + + return GenericMask(text); + } + + private static string GenericMask(string text) + { + if (text.Length == 1) + { + return "*"; + } + + return string.Create(text.Length, text, static (span, source) => + { + span[0] = source[0]; + for (var i = 1; i < span.Length; i++) + { + span[i] = '*'; + } + }); + } + private async Task FlushAuditRecordsAsync(DbContext context, CancellationToken cancellationToken) { if (!_pendingRecords.TryGetValue(context, out var pending) || pending.Records.Count == 0) diff --git a/src/SensitiveFlow.Tool/SensitiveFlowToolRunner.cs b/src/SensitiveFlow.Tool/SensitiveFlowToolRunner.cs index 4a25b54..6c88ba9 100644 --- a/src/SensitiveFlow.Tool/SensitiveFlowToolRunner.cs +++ b/src/SensitiveFlow.Tool/SensitiveFlowToolRunner.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using System.Reflection; using SensitiveFlow.Core.Discovery; @@ -17,14 +18,14 @@ public static int Run(string[] args, TextWriter output, TextWriter error) if (args.Length < 2 || !string.Equals(args[0], "scan", StringComparison.OrdinalIgnoreCase)) { - error.WriteLine("Usage: sensitiveflow scan [output-directory]"); + error.WriteLine("Usage: sensitiveflow scan [output-directory]"); return 2; } var inputPath = Path.GetFullPath(args[1]); if (!File.Exists(inputPath) && !Directory.Exists(inputPath)) { - error.WriteLine($"Assembly or directory not found: {inputPath}"); + error.WriteLine($"Assembly, project, solution, or directory not found: {inputPath}"); return 3; } @@ -33,10 +34,17 @@ public static int Run(string[] args, TextWriter output, TextWriter error) : Directory.GetCurrentDirectory(); Directory.CreateDirectory(outputDirectory); - var assemblies = ResolveAssemblies(inputPath).ToArray(); + var buildResult = TryBuildSourceInput(inputPath, output, error); + if (buildResult.ExitCode != 0) + { + return buildResult.ExitCode; + } + + var scanPath = buildResult.ScanPath ?? inputPath; + var assemblies = ResolveAssemblies(scanPath).ToArray(); if (assemblies.Length == 0) { - error.WriteLine($"No assemblies found to scan: {inputPath}"); + error.WriteLine($"No assemblies found to scan: {scanPath}"); return 4; } @@ -53,6 +61,91 @@ public static int Run(string[] args, TextWriter output, TextWriter error) return 0; } + private static BuildSourceResult TryBuildSourceInput(string inputPath, TextWriter output, TextWriter error) + { + var buildTarget = ResolveBuildTarget(inputPath); + if (buildTarget is null) + { + return new BuildSourceResult(0, null); + } + + output.WriteLine($"Building source project before scan: {buildTarget}"); + var psi = new ProcessStartInfo("dotnet", $"build \"{buildTarget}\" -c Release --nologo") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }; + + using var process = Process.Start(psi); + if (process is null) + { + error.WriteLine("Failed to start dotnet build."); + return new BuildSourceResult(5, null); + } + + var stdoutTask = process.StandardOutput.ReadToEndAsync(); + var stderrTask = process.StandardError.ReadToEndAsync(); + if (!process.WaitForExit(TimeSpan.FromMinutes(2))) + { + try + { + process.Kill(entireProcessTree: true); + } + catch (InvalidOperationException) + { + } + + error.WriteLine("dotnet build timed out."); + return new BuildSourceResult(5, null); + } + + var stdout = stdoutTask.GetAwaiter().GetResult(); + var stderr = stderrTask.GetAwaiter().GetResult(); + if (process.ExitCode != 0) + { + error.WriteLine(stdout); + error.WriteLine(stderr); + error.WriteLine($"dotnet build failed with exit code {process.ExitCode}."); + return new BuildSourceResult(5, null); + } + + var scanPath = Directory.Exists(inputPath) + ? inputPath + : Path.GetDirectoryName(buildTarget); + + return new BuildSourceResult(0, scanPath); + } + + private static string? ResolveBuildTarget(string inputPath) + { + if (File.Exists(inputPath)) + { + var extension = Path.GetExtension(inputPath); + return extension.Equals(".csproj", StringComparison.OrdinalIgnoreCase) + || extension.Equals(".sln", StringComparison.OrdinalIgnoreCase) + || extension.Equals(".slnx", StringComparison.OrdinalIgnoreCase) + ? inputPath + : null; + } + + if (!Directory.Exists(inputPath)) + { + return null; + } + + var solution = Directory.EnumerateFiles(inputPath, "*.slnx", SearchOption.TopDirectoryOnly) + .Concat(Directory.EnumerateFiles(inputPath, "*.sln", SearchOption.TopDirectoryOnly)) + .FirstOrDefault(); + if (solution is not null) + { + return solution; + } + + var projects = Directory.EnumerateFiles(inputPath, "*.csproj", SearchOption.TopDirectoryOnly).ToArray(); + return projects.Length == 1 ? projects[0] : null; + } + private static IEnumerable ResolveAssemblies(string inputPath) { if (File.Exists(inputPath)) @@ -77,4 +170,6 @@ private static IEnumerable ResolveAssemblies(string inputPath) yield return assembly; } } + + private sealed record BuildSourceResult(int ExitCode, string? ScanPath); } diff --git a/tests/SensitiveFlow.Audit.Tests/AuditOutboxTests.cs b/tests/SensitiveFlow.Audit.Tests/AuditOutboxTests.cs new file mode 100644 index 0000000..1c049cb --- /dev/null +++ b/tests/SensitiveFlow.Audit.Tests/AuditOutboxTests.cs @@ -0,0 +1,91 @@ +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using SensitiveFlow.Audit.Extensions; +using SensitiveFlow.Audit.Outbox; +using SensitiveFlow.Core.Enums; +using SensitiveFlow.Core.Interfaces; +using SensitiveFlow.Core.Models; + +namespace SensitiveFlow.Audit.Tests; + +public sealed class AuditOutboxTests +{ + [Fact] + public async Task InMemoryAuditOutbox_EnqueuesRecords() + { + var outbox = new InMemoryAuditOutbox(); + var record = SampleRecord(); + + await outbox.EnqueueAsync(record); + + outbox.Records.Should().ContainSingle(r => r.Id == record.Id); + } + + [Fact] + public void JsonAuditOutboxSerializer_RoundTripsRecord() + { + var serializer = new JsonAuditOutboxSerializer(); + var record = SampleRecord(); + + var payload = serializer.Serialize(record); + var restored = serializer.Deserialize(payload); + + restored.Id.Should().Be(record.Id); + restored.DataSubjectId.Should().Be(record.DataSubjectId); + restored.Operation.Should().Be(record.Operation); + } + + [Fact] + public async Task AddInMemoryAuditOutbox_WrapsAuditStore() + { + var services = new ServiceCollection(); + services.AddSingleton(); + services.AddInMemoryAuditOutbox(); + using var provider = services.BuildServiceProvider(); + + var store = provider.GetRequiredService(); + var outbox = provider.GetRequiredService(); + var record = SampleRecord(); + + await store.AppendAsync(record); + + outbox.Records.Should().ContainSingle(r => r.Id == record.Id); + } + + private static AuditRecord SampleRecord() => new() + { + DataSubjectId = "subject-1", + Entity = "Customer", + Field = "Email", + Operation = AuditOperation.Update, + }; + + private sealed class TestAuditStore : IAuditStore + { + private readonly List _records = []; + + public Task AppendAsync(AuditRecord record, CancellationToken cancellationToken = default) + { + _records.Add(record); + return Task.CompletedTask; + } + + public Task> QueryAsync( + DateTimeOffset? from = null, + DateTimeOffset? to = null, + int skip = 0, + int take = 100, + CancellationToken cancellationToken = default) + => Task.FromResult>(_records); + + public Task> QueryByDataSubjectAsync( + string dataSubjectId, + DateTimeOffset? from = null, + DateTimeOffset? to = null, + int skip = 0, + int take = 100, + CancellationToken cancellationToken = default) + => Task.FromResult>( + _records.Where(r => r.DataSubjectId == dataSubjectId).ToArray()); + } +} diff --git a/tests/SensitiveFlow.Core.Tests/SensitiveFlowToolRunnerTests.cs b/tests/SensitiveFlow.Core.Tests/SensitiveFlowToolRunnerTests.cs index 0858502..a8f46a9 100644 --- a/tests/SensitiveFlow.Core.Tests/SensitiveFlowToolRunnerTests.cs +++ b/tests/SensitiveFlow.Core.Tests/SensitiveFlowToolRunnerTests.cs @@ -48,5 +48,29 @@ public void Run_ScanDirectory_WritesReports() File.Exists(Path.Combine(outputDirectory, "sensitiveflow-report.json")).Should().BeTrue(); File.Exists(Path.Combine(outputDirectory, "sensitiveflow-report.md")).Should().BeTrue(); } + + [Fact] + public void Run_ScanInvalidProject_ReturnsBuildError() + { + var root = Path.Combine(Path.GetTempPath(), "sensitiveflow-tool-project-tests", Guid.NewGuid().ToString("N")); + var projectDirectory = Path.Combine(root, "AnnotatedProject"); + var outputDirectory = Path.Combine(root, "report"); + Directory.CreateDirectory(projectDirectory); + var projectPath = Path.Combine(projectDirectory, "AnnotatedProject.csproj"); + File.WriteAllText(projectPath, """ + + + not-a-real-tfm + + + """); + using var output = new StringWriter(); + using var error = new StringWriter(); + + var exitCode = SensitiveFlowToolRunner.Run(["scan", projectPath, outputDirectory], output, error); + + exitCode.Should().Be(5); + error.ToString().Should().Contain("dotnet build failed"); + } } #endif diff --git a/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlow.Diagnostics.Tests.csproj b/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlow.Diagnostics.Tests.csproj index 474cf1e..e34a20f 100644 --- a/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlow.Diagnostics.Tests.csproj +++ b/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlow.Diagnostics.Tests.csproj @@ -9,5 +9,8 @@ + + + - \ No newline at end of file + diff --git a/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs b/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs index 074d544..e20f083 100644 --- a/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs +++ b/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs @@ -1,11 +1,15 @@ using FluentAssertions; using Microsoft.Extensions.DependencyInjection; +using SensitiveFlow.AspNetCore.Diagnostics; +using SensitiveFlow.AspNetCore.Extensions; using SensitiveFlow.Core.Enums; +using SensitiveFlow.Core.Attributes; using SensitiveFlow.Core.Interfaces; using SensitiveFlow.Core.Models; using SensitiveFlow.Core.Profiles; using SensitiveFlow.Diagnostics.Extensions; using SensitiveFlow.Diagnostics.Validation; +using SensitiveFlow.EFCore.Extensions; namespace SensitiveFlow.Diagnostics.Tests; @@ -107,6 +111,69 @@ public void ValidateSensitiveFlow_WhenPolicyRequiresLogMaskingAndRedactorMissing report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-008"); } + [Fact] + public void ValidateSensitiveFlow_WhenEfCoreInterceptorRegisteredWithoutAuditStore_ReportsWarning() + { + var services = new ServiceCollection(); + services.AddSensitiveFlowEFCore(); + services.AddSensitiveFlowValidation(o => o.RequireAuditStore = false); + + var report = services.BuildServiceProvider().ValidateSensitiveFlow(); + + report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-009"); + } + + [Fact] + public void ValidateSensitiveFlow_WhenRetentionAnnotationsLoadedWithoutExecutorOrHandlers_ReportsWarning() + { + _ = typeof(RetentionAnnotatedShape).Assembly; + var services = new ServiceCollection(); + services.AddSensitiveFlowValidation(o => o.RequireAuditStore = false); + + var report = services.BuildServiceProvider().ValidateSensitiveFlow(); + + report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-010"); + } + + [Fact] + public void ValidateSensitiveFlow_WhenAspNetCoreRegisteredButMiddlewareNotMarked_ReportsWarning() + { + var services = new ServiceCollection(); + services.AddSensitiveFlowAspNetCore(); + services.AddSensitiveFlowValidation(o => o.RequireAuditStore = false); + + var report = services.BuildServiceProvider().ValidateSensitiveFlow(); + + report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-011"); + } + + [Fact] + public void ValidateSensitiveFlow_WhenAuditMiddlewareObservedAuthenticatedUser_ReportsWarning() + { + var services = new ServiceCollection(); + services.AddSensitiveFlowAspNetCore(); + services.AddSensitiveFlowValidation(o => o.RequireAuditStore = false); + var provider = services.BuildServiceProvider(); + var pipeline = provider.GetRequiredService(); + pipeline.MarkAuditMiddlewareRegistered(); + pipeline.MarkAuthenticatedUserObserved(); + + var report = provider.ValidateSensitiveFlow(); + + report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-012"); + } + + [Fact] + public void AddSensitiveFlow_RegistersSharedOptions() + { + var services = new ServiceCollection(); + services.AddSensitiveFlow(options => options.UseProfile(SensitiveFlowProfile.Strict)); + + var options = services.BuildServiceProvider().GetRequiredService(); + + options.Profile.Should().Be(SensitiveFlowProfile.Strict); + } + private sealed class HealthyAuditStore : IAuditStore { public Task AppendAsync(AuditRecord record, CancellationToken cancellationToken = default) @@ -143,4 +210,10 @@ public Task ReverseAsync(string token, CancellationToken cancellationTok public bool CanPseudonymize(string value) => true; } + + private sealed class RetentionAnnotatedShape + { + [RetentionData(Years = 1)] + public string Email { get; set; } = string.Empty; + } } diff --git a/tests/SensitiveFlow.EFCore.Tests/SensitiveDataAuditInterceptorTests.cs b/tests/SensitiveFlow.EFCore.Tests/SensitiveDataAuditInterceptorTests.cs index a09d8b2..2a50033 100644 --- a/tests/SensitiveFlow.EFCore.Tests/SensitiveDataAuditInterceptorTests.cs +++ b/tests/SensitiveFlow.EFCore.Tests/SensitiveDataAuditInterceptorTests.cs @@ -48,6 +48,44 @@ private static DbContextOptions BuildOptions(SensitiveDataAuditInterceptor inter .Options; } + private sealed class AuditRedactionEntity + { + public int Id { get; set; } + public string DataSubjectId { get; set; } = "audit-redaction-subject"; + + [PersonalData(Category = DataCategory.Contact)] + [Redaction(Audit = OutputRedactionAction.Mask)] + public string Email { get; set; } = "maria@example.com"; + + [SensitiveData(Category = SensitiveDataCategory.Other)] + [Redaction(Audit = OutputRedactionAction.Redact)] + public string TaxId { get; set; } = "12345678900"; + + [PersonalData(Category = DataCategory.Other)] + [Redaction(Audit = OutputRedactionAction.Omit)] + public string InternalNote { get; set; } = "omit me"; + + [PersonalData(Category = DataCategory.Identification)] + [Redaction(Audit = OutputRedactionAction.Pseudonymize)] + public string Name { get; set; } = "Maria"; + } + + private sealed class AuditRedactionDbContext : DbContext + { + private readonly SensitiveDataAuditInterceptor _interceptor; + + public AuditRedactionDbContext(SensitiveDataAuditInterceptor interceptor) + : base(new DbContextOptionsBuilder() + .UseInMemoryDatabase(Guid.NewGuid().ToString()) + .AddInterceptors(interceptor) + .Options) + { + _interceptor = interceptor; + } + + public DbSet Items => Set(); + } + private static (TestDbContext, InMemoryAuditStore) BuildContext(IAuditContext? auditContext = null) { var store = new InMemoryAuditStore(); @@ -266,6 +304,27 @@ public async Task AuditRecord_UsesUserId_AsLegacyAlias() records.Should().NotBeEmpty(); } + [Fact] + public async Task AuditRecord_RespectsContextualAuditRedaction() + { + var store = new InMemoryAuditStore(); + var interceptor = new SensitiveDataAuditInterceptor( + store, + NullAuditContext.Instance, + new FakePseudonymizer()); + var db = new AuditRedactionDbContext(interceptor); + db.Database.EnsureCreated(); + + db.Items.Add(new AuditRedactionEntity()); + await db.SaveChangesAsync(); + + var records = await store.QueryByDataSubjectAsync("audit-redaction-subject"); + records.Should().Contain(r => r.Field == "Email" && r.Details!.Contains("m****@example.com", StringComparison.Ordinal)); + records.Should().Contain(r => r.Field == "TaxId" && r.Details!.Contains("[REDACTED]", StringComparison.Ordinal)); + records.Should().Contain(r => r.Field == "Name" && r.Details!.Contains("token-Maria", StringComparison.Ordinal)); + records.Should().NotContain(r => r.Field == "InternalNote"); + } + private sealed class UniqueEntity { public int Id { get; set; } @@ -276,6 +335,21 @@ private sealed class UniqueEntity public string Email { get; set; } = "unique@example.com"; } + private sealed class FakePseudonymizer : IPseudonymizer + { + public string Pseudonymize(string value) => $"token-{value}"; + + public Task PseudonymizeAsync(string value, CancellationToken cancellationToken = default) + => Task.FromResult(Pseudonymize(value)); + + public string Reverse(string token) => token.Replace("token-", string.Empty, StringComparison.Ordinal); + + public Task ReverseAsync(string token, CancellationToken cancellationToken = default) + => Task.FromResult(Reverse(token)); + + public bool CanPseudonymize(string value) => true; + } + private sealed class FailingDbContext : DbContext { private readonly SqliteConnection _connection; From 9bb7a0e0ed142da1dc1ff8d08552f1c8a07c37ee Mon Sep 17 00:00:00 2001 From: Mateus Silva Date: Mon, 11 May 2026 11:14:36 -0300 Subject: [PATCH 06/23] feat: Implement EF Core-backed durable audit outbox - Added `SensitiveFlow.Audit.EFCore.Outbox` project with necessary dependencies. - Created `EfCoreAuditOutbox` class for managing durable audit outbox entries. - Updated `AuditDbContext` to conditionally apply outbox entity configurations. - Enhanced `EfCoreAuditStore` to support transaction execution for batch operations. - Introduced `OutboxAuditStore` decorator to integrate outbox functionality into existing audit stores. - Added hosted service `AuditOutboxDispatcher` for processing outbox entries. - Implemented `AuditOutboxDispatcherOptions` and `BackoffStrategy` for configurable dispatch behavior. - Created health checks for audit outbox configuration. - Added metrics for monitoring audit outbox operations. - Developed unit tests for `EfCoreAuditOutbox` functionality. - Updated existing tests to accommodate new outbox features. --- CHANGELOG.md | 43 ++-- docs/audit.md | 50 +++- docs/outbox-example.md | 239 ++++++++++++++++++ docs/package-reference.md | 79 ++++++ .../SampleAuditOutboxPublisher.cs | 47 ++++ samples/WebApi.Sample/Program.cs | 10 + samples/WebApi.Sample/WebApi.Sample.csproj | 1 + sensitiveflow.slnx | 2 + .../AuditOutboxEntryConfiguration.cs | 56 ++++ .../Entities/AuditOutboxEntryEntity.cs | 41 +++ ...OutboxEFCoreServiceCollectionExtensions.cs | 44 ++++ .../SensitiveFlow.Audit.EFCore.Outbox.csproj | 16 ++ .../Stores/EfCoreAuditOutbox.cs | 162 ++++++++++++ .../AuditDbContext.cs | 11 + .../Stores/EfCoreAuditStore.cs | 25 +- .../Decorators/OutboxAuditStore.cs | 46 +++- .../AuditServiceCollectionExtensions.cs | 17 ++ .../Outbox/AuditOutboxDispatcher.cs | 105 ++++++++ .../Outbox/AuditOutboxDispatcherOptions.cs | 20 ++ .../Outbox/BackoffStrategy.cs | 14 + .../Outbox/InMemoryAuditOutbox.cs | 1 + .../Outbox/SensitiveFlowAuditDiagnostics.cs | 46 ++++ .../SensitiveFlow.Audit.csproj | 1 + .../Interfaces/IAuditOutboxPublisher.cs | 13 + .../Interfaces/IAuditStoreTransaction.cs | 13 + .../Interfaces/IDurableAuditOutbox.cs | 31 +++ .../Models/AuditOutboxEntry.cs | 25 ++ .../Profiles/SensitiveFlowDefaults.cs | 5 + .../SensitiveFlow.Diagnostics.csproj | 1 + .../SensitiveFlowConfigurationValidator.cs | 26 ++ .../Checks/AuditOutboxHealthCheck.cs | 35 +++ ...nsitiveFlowHealthCheckBuilderExtensions.cs | 11 + .../SensitiveFlow.HealthChecks.csproj | 1 + .../EfCoreAuditOutboxTests.cs | 205 +++++++++++++++ ...itiveFlow.Audit.EFCore.Outbox.Tests.csproj | 17 ++ .../AuditOutboxTests.cs | 2 + 36 files changed, 1427 insertions(+), 34 deletions(-) create mode 100644 docs/outbox-example.md create mode 100644 samples/WebApi.Sample/Infrastructure/SampleAuditOutboxPublisher.cs create mode 100644 src/SensitiveFlow.Audit.EFCore.Outbox/Configuration/AuditOutboxEntryConfiguration.cs create mode 100644 src/SensitiveFlow.Audit.EFCore.Outbox/Entities/AuditOutboxEntryEntity.cs create mode 100644 src/SensitiveFlow.Audit.EFCore.Outbox/Extensions/OutboxEFCoreServiceCollectionExtensions.cs create mode 100644 src/SensitiveFlow.Audit.EFCore.Outbox/SensitiveFlow.Audit.EFCore.Outbox.csproj create mode 100644 src/SensitiveFlow.Audit.EFCore.Outbox/Stores/EfCoreAuditOutbox.cs create mode 100644 src/SensitiveFlow.Audit/Outbox/AuditOutboxDispatcher.cs create mode 100644 src/SensitiveFlow.Audit/Outbox/AuditOutboxDispatcherOptions.cs create mode 100644 src/SensitiveFlow.Audit/Outbox/BackoffStrategy.cs create mode 100644 src/SensitiveFlow.Audit/Outbox/SensitiveFlowAuditDiagnostics.cs create mode 100644 src/SensitiveFlow.Core/Interfaces/IAuditOutboxPublisher.cs create mode 100644 src/SensitiveFlow.Core/Interfaces/IAuditStoreTransaction.cs create mode 100644 src/SensitiveFlow.Core/Interfaces/IDurableAuditOutbox.cs create mode 100644 src/SensitiveFlow.Core/Models/AuditOutboxEntry.cs create mode 100644 src/SensitiveFlow.HealthChecks/Checks/AuditOutboxHealthCheck.cs create mode 100644 tests/SensitiveFlow.Audit.EFCore.Outbox.Tests/EfCoreAuditOutboxTests.cs create mode 100644 tests/SensitiveFlow.Audit.EFCore.Outbox.Tests/SensitiveFlow.Audit.EFCore.Outbox.Tests.csproj diff --git a/CHANGELOG.md b/CHANGELOG.md index 26c51d3..f5bc20b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,25 +7,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.0.0-preview.3] - 2026-05-11 + ### Added -- `SensitiveFlow.Core` policy engine primitives: `SensitiveFlowOptions`, built-in profiles, category policy registry, output behavior attributes, contextual redaction attributes, data sensitivity levels, discovery reports, export formatters, audit correlation helpers, audit outbox interfaces, data-subject request interfaces, and privacy-safe exception types. -- `SensitiveFlow.Tool` command-line project with `sensitiveflow scan [output-directory]` for JSON/Markdown discovery reports. -- `SensitiveFlow.HealthChecks` package with audit/token store health checks. -- `SensitiveFlow.Diagnostics` startup validation via `AddSensitiveFlowValidation(...)` and `ValidateSensitiveFlow()`. -- `SensitiveFlow.Diagnostics` happy-path `AddSensitiveFlow(...)` registration for shared profiles/policies. -- `SensitiveFlow.Retention` dry-run execution via `RetentionExecutor.DryRunAsync(...)`. -- `SensitiveFlow.TestKit` expanded assertions: `ContainsMaskedEmail`, `DoesNotContainRawValues`, `JsonDoesNotExposeAnnotatedProperties`, and `LogsDoNotContainSensitiveValues`. -- `SensitiveFlow.TestKit` contract test bases for `IAuditSnapshotStore`, `IPseudonymizer`, `IMasker`, `IAnonymizer`, and `IRetentionExpirationHandler`. -- `SensitiveFlow.Json` now honors output attributes, contextual API response redaction, and category policies before falling back to the configured default mode. -- `SensitiveFlow.Logging` now redacts annotated structured object members by default and applies `.MaskInLogs()` policies when a policy registry is supplied through `SensitiveLoggingOptions`. -- `DataSubjectExporter` now honors contextual `[Redaction(Export = ...)]` attributes while keeping raw export as the default. -- `SensitiveDataAuditInterceptor` now honors contextual `[Redaction(Audit = ...)]` for per-field audit records. -- `SensitiveFlow.Tool` now builds project/solution inputs before scanning compiled assemblies. -- `SensitiveFlow.Audit` now includes `JsonAuditOutboxSerializer`, `InMemoryAuditOutbox`, and `OutboxAuditStore`. -- Startup diagnostics now detect EF Core interceptor registration without `IAuditStore`, loaded retention annotations without executor/handlers, and ASP.NET Core audit middleware registration/order markers when available. -- `SensitiveDataAssert.DoesNotContainAny` - checks a payload against explicit string values without requiring an annotated entity. -- `SensitiveDataAssert.DoesNotLeakKnownValues` - same as `DoesNotContainAny` but accepts `IEnumerable` for readability. +- **Durable audit outbox framework**: `IDurableAuditOutbox` contract with `DequeueBatchAsync`, `MarkProcessedAsync`, `MarkFailedAsync` for production-grade delivery guarantees. +- **Audit outbox entries**: `AuditOutboxEntry` model tracking attempts, timestamps, errors, and dead-letter status for reliable retry logic. +- **Outbox publisher abstraction**: `IAuditOutboxPublisher` for pluggable downstream delivery (SIEM, webhooks, event buses, etc.). +- **Audit outbox dispatcher**: `AuditOutboxDispatcher` background service with configurable polling, exponential backoff, and dead-lettering. +- **Metrics & diagnostics**: New diagnostic codes SF-CONFIG-013 (in-memory outbox outside Development) and SF-CONFIG-014 (durable outbox without publishers). +- **Health checks**: `AuditOutboxHealthCheck` detects in-memory outbox usage in production. +- **EF Core durable outbox** (`SensitiveFlow.Audit.EFCore.Outbox` package): `EfCoreAuditOutbox` with transactional guarantees (audit + outbox in single SaveChanges). +- **Audit outbox diagnostics class**: `SensitiveFlowAuditDiagnostics` tracks enqueued/dispatched/failed/dead-lettered/pending record counts. + +### Deprecated + +- `InMemoryAuditOutbox` — use `AddEfCoreAuditOutbox()` or custom `IDurableAuditOutbox` for production. +- `AddInMemoryAuditOutbox()` — same guidance as above. + +### Changed + +- Audit store decorator `OutboxAuditStore` now reports metrics via `SensitiveFlowAuditDiagnostics`. +- `SensitiveFlowConfigurationValidator` now checks for in-memory outbox outside Development (SF-CONFIG-013). +- Docs: `docs/audit.md` updated with at-least-once vs at-most-once delivery matrix and production-ready outbox patterns. + +### Security + +- Durable outbox prevents audit record loss on process restart or failure — critical for compliance. +- Transactional outbox pattern ensures audit writes and outbox enqueuing happen atomically. ## [1.0.0-preview.2] - 2026-05-10 diff --git a/docs/audit.md b/docs/audit.md index 7168e07..f5cb5ef 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -155,14 +155,58 @@ public string InternalNote { get; set; } = string.Empty; ## Audit outbox -`SensitiveFlow.Audit` ships a concrete in-memory outbox for tests/local development and an `OutboxAuditStore` decorator for production-owned durable outboxes: +`SensitiveFlow.Audit` ships a concrete in-memory outbox for tests/local development. Production systems must use a durable audit outbox to ensure reliable delivery to downstream systems (e.g., SIEM, compliance dashboards, data lakes). + +### In-memory outbox (tests only) ```csharp builder.Services.AddAuditStore(); -builder.Services.AddAuditOutbox(); +builder.Services.AddInMemoryAuditOutbox(); // Deprecated – use for tests/dev only +``` + +⚠️ `InMemoryAuditOutbox` is **deprecated for production**. It loses all enqueued records on process restart and is not suitable for compliance/audit scenarios. The `SensitiveFlowConfigurationValidator` will emit `SF-CONFIG-013` if it detects an in-memory outbox outside a Development environment. + +### Durable outbox with EF Core (production-ready) + +For production, use the EF Core-backed durable outbox with transactional guarantees: + +```bash +dotnet add package SensitiveFlow.Audit.EFCore.Outbox +``` + +```csharp +// Register durable audit store + durable outbox with automatic dispatcher +builder.Services.AddEfCoreAuditStore(opt => opt.UseSqlServer(...)); +builder.Services.AddEfCoreAuditOutbox(options => +{ + options.PollInterval = TimeSpan.FromSeconds(1); + options.BatchSize = 100; + options.MaxAttempts = 5; +}); + +// Register a publisher to deliver outbox records downstream +builder.Services.AddScoped(); ``` -Use `AddInMemoryAuditOutbox()` only for tests and local samples. It is not durable. +The durable outbox provides **at-least-once delivery** guarantees: +- Records enqueued and audit store writes happen in a single `SaveChanges` transaction +- Failed deliveries are retried with exponential backoff (configurable) +- Dead-lettered entries (max retries exceeded) are queryable for inspection +- The `AuditOutboxDispatcher` automatically detects and polls pending entries + +### Custom durable outbox + +If you need to integrate with a different backend (e.g. Apache Kafka, AWS SQS), implement `IDurableAuditOutbox`: + +```csharp +public sealed class KafkaAuditOutbox : IDurableAuditOutbox +{ + // Implement: EnqueueAsync, DequeueBatchAsync, MarkProcessedAsync, MarkFailedAsync +} + +builder.Services.AddAuditStore(); +builder.Services.AddAuditOutbox(); +``` ## Retrying transient failures diff --git a/docs/outbox-example.md b/docs/outbox-example.md new file mode 100644 index 0000000..4dfdd07 --- /dev/null +++ b/docs/outbox-example.md @@ -0,0 +1,239 @@ +# Audit Outbox Example + +This example demonstrates how to set up the durable audit outbox with EF Core in a production-ready application. + +## Installation + +```bash +dotnet add package SensitiveFlow.Audit.EFCore.Outbox +``` + +## Setup + +### 1. Enable Audit Store and Durable Outbox + +```csharp +using SensitiveFlow.Audit.EFCore.Extensions; +using SensitiveFlow.Audit.EFCore.Outbox.Extensions; + +var builder = WebApplication.CreateBuilder(args); + +var auditConnection = builder.Configuration.GetConnectionString("Audit") + ?? "Data Source=audit.db"; + +// Register durable audit store +builder.Services.AddEfCoreAuditStore(options => + options.UseSqlite(auditConnection)); // or UseSqlServer, UseNpgsql + +// Register durable outbox with background dispatcher +builder.Services.AddEfCoreAuditOutbox(options => +{ + // Polls every 2 seconds for pending entries + options.PollInterval = TimeSpan.FromSeconds(2); + + // Process 100 entries per batch + options.BatchSize = 100; + + // Retry failed deliveries up to 5 times + options.MaxAttempts = 5; + + // Use exponential backoff: 1s, 2s, 4s, 8s, 16s + options.BackoffStrategy = BackoffStrategy.Exponential; +}); +``` + +### 2. Implement Audit Outbox Publishers + +Publishers are responsible for delivering audit records to your downstream systems (SIEM, data lakes, Kafka, webhooks, etc.). + +#### Example: HTTP Publisher to SIEM + +```csharp +using SensitiveFlow.Core.Interfaces; +using SensitiveFlow.Core.Models; +using System.Text.Json; + +namespace YourApp.Audit; + +public sealed class SiemHttpPublisher : IAuditOutboxPublisher +{ + private readonly HttpClient _httpClient; + private readonly ILogger _logger; + + public SiemHttpPublisher(HttpClient httpClient, ILogger logger) + { + _httpClient = httpClient; + _logger = logger; + } + + public async Task PublishAsync(AuditOutboxEntry entry, CancellationToken cancellationToken) + { + var payload = JsonSerializer.Serialize(entry.Record); + + var request = new HttpRequestMessage(HttpMethod.Post, "/api/audit/receive") + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + Headers = { { "X-Audit-Attempt", entry.Attempts.ToString() } }, + }; + + var response = await _httpClient.SendAsync(request, cancellationToken); + + if (!response.IsSuccessStatusCode) + { + throw new HttpRequestException( + $"SIEM rejected audit record: {response.StatusCode}"); + } + + _logger.LogInformation("Delivered audit record {RecordId} to SIEM", + entry.Record.Id); + } +} +``` + +#### Register the Publisher + +```csharp +// Configure HttpClient for SIEM +builder.Services + .AddHttpClient(client => + { + client.BaseAddress = new Uri(builder.Configuration["SiemEndpoint"] + ?? "https://siem.example.com"); + client.DefaultRequestHeaders.Add("Authorization", + $"Bearer {builder.Configuration["SiemToken"]}"); + }); + +// Register as IAuditOutboxPublisher +builder.Services.AddScoped( + sp => sp.GetRequiredService()); +``` + +#### Example: Kafka Publisher + +```csharp +using Confluent.Kafka; + +public sealed class KafkaAuditPublisher : IAuditOutboxPublisher +{ + private readonly IProducer _producer; + private readonly string _topic; + + public KafkaAuditPublisher(IProducer producer, string topic) + { + _producer = producer; + _topic = topic; + } + + public async Task PublishAsync(AuditOutboxEntry entry, CancellationToken cancellationToken) + { + var json = JsonSerializer.Serialize(entry.Record); + + var message = new Message + { + Key = entry.Record.Id.ToString(), + Value = json, + }; + + await _producer.ProduceAsync(_topic, message, cancellationToken); + } +} +``` + +### 3. Create/Migrate Audit Database + +The outbox table is automatically created when you run migrations: + +```csharp +using var app = builder.Build(); + +// Apply migrations or create database +using (var scope = app.Services.CreateScope()) +{ + var auditDb = scope.ServiceProvider + .GetRequiredService>() + .CreateDbContext(); + + // For development only: + await auditDb.Database.EnsureCreatedAsync(); + + // For production, use EF Core migrations: + // await auditDb.Database.MigrateAsync(); +} + +app.Run(); +``` + +## Operations + +### Querying Outbox State + +```csharp +using (var scope = app.Services.CreateScope()) +{ + var factory = scope.ServiceProvider + .GetRequiredService>(); + + await using var ctx = await factory.CreateDbContextAsync(); + + // Pending entries (not yet processed) + var pending = await ctx.Set() + .Where(e => !e.IsProcessed && !e.IsDeadLettered) + .ToListAsync(); + + Console.WriteLine($"Pending: {pending.Count}"); + + // Dead-lettered entries (failed after max retries) + var deadLetters = await ctx.Set() + .Where(e => e.IsDeadLettered) + .ToListAsync(); + + foreach (var entry in deadLetters) + { + Console.WriteLine( + $"Dead Letter: {entry.AuditRecordId} - {entry.DeadLetterReason}"); + } + + // High-retry entries (approaching max attempts) + var risky = await ctx.Set() + .Where(e => !e.IsProcessed && e.Attempts >= 3) + .ToListAsync(); + + Console.WriteLine($"High-Retry: {risky.Count}"); +} +``` + +### Health Checks + +```csharp +builder.Services.AddSensitiveFlowHealthChecks() + .AddAuditStoreCheck(); + +app.MapHealthChecks("/health"); +``` + +## Guarantees + +- **Transactional Enqueue**: Audit record and outbox entry are persisted in one `SaveChanges` transaction +- **At-Least-Once Delivery**: Failed deliveries are retried with configurable backoff +- **Dead-Lettering**: Entries exceeding `MaxAttempts` are marked `IsDeadLettered` for manual inspection +- **Concurrent Safe**: Multiple app instances can run simultaneously; polling is coordinated via database state + +## Monitoring + +Monitor the `AuditOutboxEntry` table for: + +1. **Pending entries** — should be near zero in steady state +2. **Failed/dead-lettered entries** — indicates delivery issues +3. **Retry counts** — rising retry counts suggest unstable downstream systems +4. **Table size** — verify disk space is available; consider archiving old processed entries + +Example SQL Server query for monitoring: + +```sql +SELECT + COUNT(*) FILTER (WHERE IsProcessed = 0 AND IsDeadLettered = 0) as Pending, + COUNT(*) FILTER (WHERE IsDeadLettered = 1) as DeadLettered, + MAX(Attempts) as MaxAttempts, + AVG(Attempts) as AvgAttempts +FROM [sensitiveflow].[AuditOutboxEntries]; +``` diff --git a/docs/package-reference.md b/docs/package-reference.md index 2c392a9..ca0c97d 100644 --- a/docs/package-reference.md +++ b/docs/package-reference.md @@ -9,6 +9,7 @@ This document summarizes each SensitiveFlow package individually: purpose, prima | `SensitiveFlow.Core` | You annotate models or implement contracts. | Add attributes to models. | None by itself; it does not enforce behavior. | | `SensitiveFlow.Audit` | You register custom audit stores or decorators. | `AddAuditStore()` or first-party EF store plus optional retry/buffer. | In-memory/buffered data can be lost before durable write. | | `SensitiveFlow.Audit.EFCore` | You want first-party SQL audit storage. | `AddEfCoreAuditStore(...)`; create/migrate audit table. | Audit DB must be durable and backed up independently. | +| `SensitiveFlow.Audit.EFCore.Outbox` | You need durable, reliable delivery of audit records to downstream systems (SIEM, data lakes, compliance dashboards). | `AddEfCoreAuditOutbox()` and register `IAuditOutboxPublisher` implementations. | Outbox table must be monitored for dead-lettered entries (failed delivery after max retries). | | `SensitiveFlow.Audit.Snapshots.EFCore` | You want durable aggregate-level audit snapshots. | `AddEfCoreAuditSnapshotStore(...)`; create/migrate snapshot table. | Snapshots can be large; monitor storage growth. | | `SensitiveFlow.TokenStore.EFCore` | You want first-party SQL token storage for reversible pseudonymization. | `AddEfCoreTokenStore(...)`; create/migrate token table. | Losing token mappings makes pseudonymized data irrecoverable. | | `SensitiveFlow.EFCore` | You want automatic audit on `SaveChanges`. | `AddSensitiveFlowEFCore()` and `AddInterceptors(...)`. | Missing interceptor means no automatic audit. | @@ -123,6 +124,84 @@ Operational notes: - Implements `IBatchAuditStore`. - PostgreSQL and SQL Server container coverage lives in `tests/SensitiveFlow.Audit.EFCore.ContainerTests`. +## SensitiveFlow.Audit.EFCore.Outbox + +Purpose: + +- First-party EF Core-backed **durable audit outbox** for reliable, transactional delivery of audit records to downstream systems (SIEM, data lakes, compliance dashboards, Kafka, webhooks, etc.). + +Primary APIs: + +- `AddEfCoreAuditOutbox(options => ...)` +- `EfCoreAuditOutbox` +- `AuditOutboxEntry` (data model in Core) +- `IAuditOutboxPublisher` (interface for delivery logic — implemented by you) +- `AuditOutboxDispatcher` (background service that polls and delivers) +- `AuditOutboxDispatcherOptions` (configurable polling, retry backoff, max attempts) + +Install when: + +- You need **guaranteed, at-least-once delivery** of audit records to a remote system. +- Your business/compliance requirements demand that no audit record is lost if the application crashes. + +Recommended setup: + +```csharp +// Package installation +dotnet add package SensitiveFlow.Audit.EFCore.Outbox + +// DI setup +builder.Services.AddEfCoreAuditStore(opt => + opt.UseSqlServer(builder.Configuration.GetConnectionString("Audit"))); + +// Enable durable outbox with automatic background dispatcher +builder.Services.AddEfCoreAuditOutbox(options => +{ + options.PollInterval = TimeSpan.FromSeconds(2); + options.BatchSize = 100; + options.MaxAttempts = 5; + options.BackoffStrategy = BackoffStrategy.Exponential; // or Linear +}); + +// Register publishers (one or more) to deliver records downstream +builder.Services.AddScoped(); +builder.Services.AddScoped(); +``` + +Example publisher implementation: + +```csharp +public sealed class MySiemPublisher : IAuditOutboxPublisher +{ + private readonly HttpClient _http; + + public MySiemPublisher(HttpClient http) => _http = http; + + public async Task PublishAsync(AuditOutboxEntry entry, CancellationToken cancellationToken) + { + var json = JsonSerializer.Serialize(entry.Record); + var response = await _http.PostAsJsonAsync("/siem/audit", json, cancellationToken); + response.EnsureSuccessStatusCode(); + } +} +``` + +Delivery guarantee: + +- Records are enqueued and the audit store write happen in a **single `SaveChanges` transaction** — transactional outbox pattern. +- The `AuditOutboxDispatcher` background service polls periodically and calls all registered `IAuditOutboxPublisher` implementations. +- On successful publish, entries are marked `IsProcessed`. +- On exception, entries are marked `IsDeadLettered` or retried based on `MaxAttempts`. +- Dead-lettered entries and retry history are queryable for operational dashboards and alerting. + +Operational notes: + +- Audit and outbox data live in the same `AuditDbContext` (configurable persistence strategy). +- Dispatcher runs as a `HostedService` — activate after application startup. +- Multiple instances of your application can run simultaneously; polling is coordinated via outbox entry state transitions (no distributed locking required). +- Monitor the `AuditOutboxEntry` table for growth, dead-lettered entries, and retry counts in operational dashboards. +- SQL Server and PostgreSQL container coverage in `tests/SensitiveFlow.Audit.EFCore.Outbox.Tests`. + ## SensitiveFlow.Audit.Snapshots.EFCore Purpose: diff --git a/samples/WebApi.Sample/Infrastructure/SampleAuditOutboxPublisher.cs b/samples/WebApi.Sample/Infrastructure/SampleAuditOutboxPublisher.cs new file mode 100644 index 0000000..8bdf408 --- /dev/null +++ b/samples/WebApi.Sample/Infrastructure/SampleAuditOutboxPublisher.cs @@ -0,0 +1,47 @@ +using Microsoft.Extensions.Logging; +using SensitiveFlow.Core.Interfaces; +using SensitiveFlow.Core.Models; + +namespace WebApi.Sample.Infrastructure; + +/// +/// Sample IAuditOutboxPublisher implementation that logs audit entries using Serilog/ILogger. +/// In production, this would be replaced with implementations that send to: +/// - A SIEM system (e.g., Splunk, ELK, Datadog) +/// - A webhook/HTTP endpoint +/// - A message queue (e.g., RabbitMQ, Kafka) +/// - A cloud service (e.g., Azure Event Hubs, AWS Kinesis) +/// +public class SampleAuditOutboxPublisher : IAuditOutboxPublisher +{ + private readonly ILogger _logger; + + public SampleAuditOutboxPublisher(ILogger logger) + { + _logger = logger; + } + + public Task PublishAsync(AuditOutboxEntry entry, CancellationToken cancellationToken = default) + { + // Log the audit entry with full details + _logger.LogInformation( + "Publishing audit outbox entry: Id={OutboxId}, RecordId={RecordId}, " + + "DataSubjectId={DataSubjectId}, Entity={Entity}, Field={Field}, " + + "Operation={Operation}, Attempts={Attempts}", + entry.Id, + entry.Record.Id, + entry.Record.DataSubjectId, + entry.Record.Entity, + entry.Record.Field, + entry.Record.Operation, + entry.Attempts); + + // In production, here you would: + // - Send HTTP request to a webhook + // - Publish to a message queue + // - Stream to a SIEM system + // - etc. + + return Task.CompletedTask; + } +} diff --git a/samples/WebApi.Sample/Program.cs b/samples/WebApi.Sample/Program.cs index b179121..97f83d5 100644 --- a/samples/WebApi.Sample/Program.cs +++ b/samples/WebApi.Sample/Program.cs @@ -7,6 +7,7 @@ using SensitiveFlow.Anonymization.Extensions; using SensitiveFlow.Audit.EFCore; using SensitiveFlow.Audit.EFCore.Extensions; +using SensitiveFlow.Audit.EFCore.Outbox.Extensions; using SensitiveFlow.Audit.Extensions; using SensitiveFlow.AspNetCore.Extensions; using SensitiveFlow.Core.Diagnostics; @@ -60,6 +61,15 @@ .AddInterceptors(sp.GetRequiredService())); builder.Services.AddEfCoreAuditStore(options => options.UseSqlite(auditConnection)); + builder.Services.AddEfCoreAuditOutbox(options => + { + // Configure durable outbox with sensible defaults + options.PollInterval = TimeSpan.FromSeconds(5); // Poll every 5 seconds + options.BatchSize = 100; // Process up to 100 entries per batch + options.MaxAttempts = 5; // Retry up to 5 times before dead-lettering + }); + builder.Services.AddScoped(); + builder.Services.AddScoped(sp => sp.GetRequiredService()); builder.Services.AddAuditStoreRetry(); builder.Services.AddSensitiveFlowDiagnostics(); builder.Services.AddEfCoreTokenStore(options => options.UseSqlite(tokenConnection)); diff --git a/samples/WebApi.Sample/WebApi.Sample.csproj b/samples/WebApi.Sample/WebApi.Sample.csproj index 1d3595d..51ec071 100644 --- a/samples/WebApi.Sample/WebApi.Sample.csproj +++ b/samples/WebApi.Sample/WebApi.Sample.csproj @@ -24,6 +24,7 @@ + diff --git a/sensitiveflow.slnx b/sensitiveflow.slnx index a9e57c3..a493298 100644 --- a/sensitiveflow.slnx +++ b/sensitiveflow.slnx @@ -12,6 +12,7 @@ + @@ -31,6 +32,7 @@ + diff --git a/src/SensitiveFlow.Audit.EFCore.Outbox/Configuration/AuditOutboxEntryConfiguration.cs b/src/SensitiveFlow.Audit.EFCore.Outbox/Configuration/AuditOutboxEntryConfiguration.cs new file mode 100644 index 0000000..3805cdf --- /dev/null +++ b/src/SensitiveFlow.Audit.EFCore.Outbox/Configuration/AuditOutboxEntryConfiguration.cs @@ -0,0 +1,56 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using SensitiveFlow.Audit.EFCore.Outbox.Entities; + +namespace SensitiveFlow.Audit.EFCore.Outbox.Configuration; + +/// +/// EF Core model configuration for . +/// +public sealed class AuditOutboxEntryConfiguration : IEntityTypeConfiguration +{ + /// + public void Configure(EntityTypeBuilder builder) + { + builder.ToTable("AuditOutboxEntries", "sensitiveflow"); + + builder.HasKey(e => e.Id); + + builder.Property(e => e.Id) + .ValueGeneratedNever(); + + builder.Property(e => e.AuditRecordId) + .IsRequired() + .HasMaxLength(256); + + builder.Property(e => e.Payload) + .IsRequired(); + + builder.Property(e => e.Attempts) + .HasDefaultValue(0); + + builder.Property(e => e.EnqueuedAt) + .IsRequired(); + + builder.Property(e => e.LastAttemptAt); + + builder.Property(e => e.LastError) + .HasMaxLength(512); + + builder.Property(e => e.IsProcessed) + .HasDefaultValue(false); + + builder.Property(e => e.ProcessedAt); + + builder.Property(e => e.IsDeadLettered) + .HasDefaultValue(false); + + builder.Property(e => e.DeadLetterReason) + .HasMaxLength(512); + + // Indexes for querying pending/dead-lettered entries + builder.HasIndex(e => new { e.IsProcessed, e.IsDeadLettered }); + builder.HasIndex(e => e.IsDeadLettered); + builder.HasIndex(e => e.EnqueuedAt); + } +} diff --git a/src/SensitiveFlow.Audit.EFCore.Outbox/Entities/AuditOutboxEntryEntity.cs b/src/SensitiveFlow.Audit.EFCore.Outbox/Entities/AuditOutboxEntryEntity.cs new file mode 100644 index 0000000..f96e52a --- /dev/null +++ b/src/SensitiveFlow.Audit.EFCore.Outbox/Entities/AuditOutboxEntryEntity.cs @@ -0,0 +1,41 @@ +namespace SensitiveFlow.Audit.EFCore.Outbox.Entities; + +/// +/// EF Core persistence entity for durable audit outbox entries. +/// Tracks delivery state and attempts separately from the immutable Core model. +/// +public sealed class AuditOutboxEntryEntity +{ + /// Unique stable identifier for this outbox entry. + public Guid Id { get; set; } = Guid.NewGuid(); + + /// The audit record ID being delivered. + public string AuditRecordId { get; set; } = string.Empty; + + /// Serialized audit record payload (JSON). + public string Payload { get; set; } = string.Empty; + + /// How many times delivery has been attempted. + public int Attempts { get; set; } + + /// When the entry was enqueued (UTC). + public DateTimeOffset EnqueuedAt { get; set; } = DateTimeOffset.UtcNow; + + /// When the last delivery attempt was made (UTC), or null if never attempted. + public DateTimeOffset? LastAttemptAt { get; set; } + + /// The most recent error message from delivery, if any. + public string? LastError { get; set; } + + /// Whether this entry has been successfully delivered and marked processed. + public bool IsProcessed { get; set; } + + /// When the entry was marked as processed, or null if still pending. + public DateTimeOffset? ProcessedAt { get; set; } + + /// Whether this entry has been moved to dead-letter after max retries. + public bool IsDeadLettered { get; set; } + + /// Reason for dead-lettering, if applicable. + public string? DeadLetterReason { get; set; } +} diff --git a/src/SensitiveFlow.Audit.EFCore.Outbox/Extensions/OutboxEFCoreServiceCollectionExtensions.cs b/src/SensitiveFlow.Audit.EFCore.Outbox/Extensions/OutboxEFCoreServiceCollectionExtensions.cs new file mode 100644 index 0000000..8f333bc --- /dev/null +++ b/src/SensitiveFlow.Audit.EFCore.Outbox/Extensions/OutboxEFCoreServiceCollectionExtensions.cs @@ -0,0 +1,44 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using SensitiveFlow.Audit.EFCore; +using SensitiveFlow.Audit.EFCore.Outbox.Stores; +using SensitiveFlow.Audit.Outbox; +using SensitiveFlow.Core.Interfaces; + +namespace SensitiveFlow.Audit.EFCore.Outbox.Extensions; + +/// +/// DI extensions for the EF Core durable audit outbox. +/// +public static class OutboxEFCoreServiceCollectionExtensions +{ + /// + /// Registers as the durable + /// and wires it with for automatic background polling. + /// + /// + /// Call this after AddEfCoreAuditStore() or a similar audit store registration. + /// The outbox shares the same , + /// ensuring that audit writes and outbox enqueuing happen in a single transaction. + /// + public static IServiceCollection AddEfCoreAuditOutbox( + this IServiceCollection services, + Action? configureDispatcher = null) + { + ArgumentNullException.ThrowIfNull(services); + + // Register the durable outbox + services.AddSingleton(sp => + new EfCoreAuditOutbox(sp.GetRequiredService>())); + + services.AddSingleton(sp => (IAuditOutbox)sp.GetRequiredService()); + + // Register the dispatcher + var dispatcherOptions = new AuditOutboxDispatcherOptions(); + configureDispatcher?.Invoke(dispatcherOptions); + services.AddSingleton(dispatcherOptions); + services.AddHostedService(); + + return services; + } +} diff --git a/src/SensitiveFlow.Audit.EFCore.Outbox/SensitiveFlow.Audit.EFCore.Outbox.csproj b/src/SensitiveFlow.Audit.EFCore.Outbox/SensitiveFlow.Audit.EFCore.Outbox.csproj new file mode 100644 index 0000000..46ade6d --- /dev/null +++ b/src/SensitiveFlow.Audit.EFCore.Outbox/SensitiveFlow.Audit.EFCore.Outbox.csproj @@ -0,0 +1,16 @@ + + + + EF Core-backed durable audit outbox for SensitiveFlow.Audit. + + + + + + + + + + + + diff --git a/src/SensitiveFlow.Audit.EFCore.Outbox/Stores/EfCoreAuditOutbox.cs b/src/SensitiveFlow.Audit.EFCore.Outbox/Stores/EfCoreAuditOutbox.cs new file mode 100644 index 0000000..3255c7a --- /dev/null +++ b/src/SensitiveFlow.Audit.EFCore.Outbox/Stores/EfCoreAuditOutbox.cs @@ -0,0 +1,162 @@ +using System.Text.Json; +using Microsoft.EntityFrameworkCore; +using SensitiveFlow.Audit.EFCore; +using SensitiveFlow.Audit.EFCore.Outbox.Entities; +using SensitiveFlow.Core.Interfaces; +using SensitiveFlow.Core.Models; + +namespace SensitiveFlow.Audit.EFCore.Outbox.Stores; + +/// +/// EF Core-backed durable audit outbox. Persists outbox entries in the same transaction +/// as the audit record for at-least-once delivery semantics with transactional guarantee. +/// +public sealed class EfCoreAuditOutbox : IDurableAuditOutbox +{ + private readonly IDbContextFactory _factory; + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + + /// Initializes a new instance. + public EfCoreAuditOutbox(IDbContextFactory factory) + { + _factory = factory ?? throw new ArgumentNullException(nameof(factory)); + } + + /// + public async Task EnqueueAsync(AuditRecord record, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(record); + + await using var ctx = await _factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + var entry = new AuditOutboxEntryEntity + { + AuditRecordId = record.Id.ToString(), + Payload = JsonSerializer.Serialize(record, JsonOptions), + EnqueuedAt = DateTimeOffset.UtcNow, + }; + ctx.Set().Add(entry); + await ctx.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + } + + /// + public async Task> DequeueBatchAsync(int max, CancellationToken cancellationToken = default) + { + if (max <= 0) + { + throw new ArgumentException("Must be > 0", nameof(max)); + } + + await using var ctx = await _factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + var pending = await ctx.Set() + .Where(e => !e.IsProcessed && !e.IsDeadLettered) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + // Order by EnqueuedAt on client side (SQLite doesn't support DateTimeOffset in ORDER BY) + return pending + .OrderBy(e => e.EnqueuedAt) + .Take(max) + .Select(ToOutboxEntry) + .ToList(); + } + + /// + public async Task MarkProcessedAsync(IEnumerable ids, CancellationToken cancellationToken = default) + { + var idList = ids?.ToList() ?? throw new ArgumentNullException(nameof(ids)); + if (idList.Count == 0) + { + return; + } + + await using var ctx = await _factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + var entries = await ctx.Set() + .Where(e => idList.Contains(e.Id)) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + foreach (var entry in entries) + { + entry.IsProcessed = true; + entry.ProcessedAt = DateTimeOffset.UtcNow; + } + + await ctx.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + } + + /// + public async Task MarkFailedAsync(Guid id, string error, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(error); + + await using var ctx = await _factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + var entry = await ctx.Set() + .FirstOrDefaultAsync(e => e.Id == id, cancellationToken) + .ConfigureAwait(false); + + if (entry != null) + { + entry.Attempts++; + entry.LastAttemptAt = DateTimeOffset.UtcNow; + entry.LastError = error; + await ctx.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + } + } + + /// Mark an entry as dead-lettered after max retries. + public async Task MarkDeadLetteredAsync(Guid id, string reason, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(reason); + + await using var ctx = await _factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await ctx.Set() + .Where(e => e.Id == id) + .ExecuteUpdateAsync( + s => s + .SetProperty(e => e.IsDeadLettered, true) + .SetProperty(e => e.DeadLetterReason, reason), + cancellationToken) + .ConfigureAwait(false); + } + + /// Get pending entries count. + public async Task GetPendingCountAsync(CancellationToken cancellationToken = default) + { + await using var ctx = await _factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + return await ctx.Set() + .Where(e => !e.IsProcessed && !e.IsDeadLettered) + .CountAsync(cancellationToken) + .ConfigureAwait(false); + } + + /// Get dead-lettered entries for inspection. + public async Task> GetDeadLetteredAsync(int skip = 0, int take = 100, CancellationToken cancellationToken = default) + { + await using var ctx = await _factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + var deadLettered = await ctx.Set() + .Where(e => e.IsDeadLettered) + .OrderByDescending(e => e.EnqueuedAt) + .Skip(skip) + .Take(take) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return deadLettered.Select(ToOutboxEntry).ToList(); + } + + private static AuditOutboxEntry ToOutboxEntry(AuditOutboxEntryEntity entity) + { + var record = JsonSerializer.Deserialize(entity.Payload, JsonOptions) + ?? throw new InvalidOperationException($"Failed to deserialize audit record from outbox entry {entity.Id}"); + + return new AuditOutboxEntry + { + Id = entity.Id, + Record = record, + Attempts = entity.Attempts, + EnqueuedAt = entity.EnqueuedAt, + LastAttemptAt = entity.LastAttemptAt, + LastError = entity.LastError, + }; + } +} diff --git a/src/SensitiveFlow.Audit.EFCore/AuditDbContext.cs b/src/SensitiveFlow.Audit.EFCore/AuditDbContext.cs index b412b15..0608498 100644 --- a/src/SensitiveFlow.Audit.EFCore/AuditDbContext.cs +++ b/src/SensitiveFlow.Audit.EFCore/AuditDbContext.cs @@ -22,5 +22,16 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) { base.OnModelCreating(modelBuilder); modelBuilder.ApplyConfiguration(new AuditRecordEntityTypeConfiguration()); + + // Try to apply durable outbox configuration if the type is available + // This allows SensitiveFlow.Audit.EFCore.Outbox to register its entities without + // creating a hard dependency in the core package. + var outboxEntityType = Type.GetType("SensitiveFlow.Audit.EFCore.Outbox.Entities.AuditOutboxEntryEntity, SensitiveFlow.Audit.EFCore.Outbox"); + if (outboxEntityType is not null) + { + modelBuilder.ApplyConfigurationsFromAssembly( + outboxEntityType.Assembly, + t => t.Namespace?.StartsWith("SensitiveFlow.Audit.EFCore.Outbox") == true); + } } } diff --git a/src/SensitiveFlow.Audit.EFCore/Stores/EfCoreAuditStore.cs b/src/SensitiveFlow.Audit.EFCore/Stores/EfCoreAuditStore.cs index 930e581..17bce46 100644 --- a/src/SensitiveFlow.Audit.EFCore/Stores/EfCoreAuditStore.cs +++ b/src/SensitiveFlow.Audit.EFCore/Stores/EfCoreAuditStore.cs @@ -18,7 +18,7 @@ namespace SensitiveFlow.Audit.EFCore.Stores; /// exists to avoid. /// /// -public sealed class EfCoreAuditStore : IBatchAuditStore where TContext : DbContext +public sealed class EfCoreAuditStore : IBatchAuditStore, IAuditStoreTransaction where TContext : DbContext { private readonly IDbContextFactory _factory; private readonly Func> _setSelector; @@ -100,6 +100,29 @@ public async Task> QueryByDataSubjectAsync( return rows.ConvertAll(static e => e.ToRecord()); } + /// + public async Task ExecuteInTransactionAsync( + Func operation, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(operation); + + await using var ctx = await _factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); + await using var transaction = await ctx.Database.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + + try + { + await operation(cancellationToken).ConfigureAwait(false); + await ctx.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + } + catch + { + await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); + throw; + } + } + private static void ValidatePagination(int skip, int take) { if (skip < 0) diff --git a/src/SensitiveFlow.Audit/Decorators/OutboxAuditStore.cs b/src/SensitiveFlow.Audit/Decorators/OutboxAuditStore.cs index 561b5a5..4116736 100644 --- a/src/SensitiveFlow.Audit/Decorators/OutboxAuditStore.cs +++ b/src/SensitiveFlow.Audit/Decorators/OutboxAuditStore.cs @@ -1,5 +1,6 @@ using SensitiveFlow.Core.Interfaces; using SensitiveFlow.Core.Models; +using SensitiveFlow.Audit.Outbox; namespace SensitiveFlow.Audit.Decorators; @@ -22,8 +23,12 @@ public OutboxAuditStore(IAuditStore inner, IAuditOutbox outbox) /// public async Task AppendAsync(AuditRecord record, CancellationToken cancellationToken = default) { - await _inner.AppendAsync(record, cancellationToken); - await _outbox.EnqueueAsync(record, cancellationToken); + await ExecuteAsync(async ct => + { + await _inner.AppendAsync(record, ct); + await _outbox.EnqueueAsync(record, ct); + SensitiveFlowAuditDiagnostics.RecordEnqueued(); + }, cancellationToken); } /// @@ -31,22 +36,27 @@ public async Task AppendRangeAsync(IReadOnlyCollection records, Can { ArgumentNullException.ThrowIfNull(records); - if (_inner is IBatchAuditStore batch) - { - await batch.AppendRangeAsync(records, cancellationToken); - } - else + await ExecuteAsync(async ct => { + if (_inner is IBatchAuditStore batch) + { + await batch.AppendRangeAsync(records, ct); + } + else + { + foreach (var record in records) + { + await _inner.AppendAsync(record, ct); + } + } + foreach (var record in records) { - await _inner.AppendAsync(record, cancellationToken); + await _outbox.EnqueueAsync(record, ct); } - } - foreach (var record in records) - { - await _outbox.EnqueueAsync(record, cancellationToken); - } + SensitiveFlowAuditDiagnostics.RecordEnqueued(records.Count); + }, cancellationToken); } /// @@ -67,4 +77,14 @@ public Task> QueryByDataSubjectAsync( int take = 100, CancellationToken cancellationToken = default) => _inner.QueryByDataSubjectAsync(dataSubjectId, from, to, skip, take, cancellationToken); + + private Task ExecuteAsync(Func operation, CancellationToken cancellationToken) + { + if (_inner is IAuditStoreTransaction transaction) + { + return transaction.ExecuteInTransactionAsync(operation, cancellationToken); + } + + return operation(cancellationToken); + } } diff --git a/src/SensitiveFlow.Audit/Extensions/AuditServiceCollectionExtensions.cs b/src/SensitiveFlow.Audit/Extensions/AuditServiceCollectionExtensions.cs index 87ffa0e..ba86864 100644 --- a/src/SensitiveFlow.Audit/Extensions/AuditServiceCollectionExtensions.cs +++ b/src/SensitiveFlow.Audit/Extensions/AuditServiceCollectionExtensions.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using SensitiveFlow.Audit.Decorators; using SensitiveFlow.Audit.Outbox; @@ -170,6 +171,7 @@ public static IServiceCollection AddBufferedAuditStore( /// systems should provide a durable implementation /// and call . /// + [Obsolete("In-memory audit outbox is for tests/local development only. Use AddEfCoreAuditOutbox() or AddAuditOutbox() for durable production delivery.", error: false)] public static IServiceCollection AddInMemoryAuditOutbox(this IServiceCollection services) { services.AddSingleton(); @@ -190,6 +192,21 @@ public static IServiceCollection AddAuditOutbox(this IServiceCollection return services.AddAuditOutboxDecorator(); } + /// + /// Registers the hosted durable audit outbox dispatcher. + /// + public static IServiceCollection AddAuditOutboxDispatcher( + this IServiceCollection services, + Action? configure = null) + { + var options = new AuditOutboxDispatcherOptions(); + configure?.Invoke(options); + + services.AddSingleton(options); + services.AddHostedService(); + return services; + } + private static IServiceCollection AddAuditOutboxDecorator(this IServiceCollection services) { var existing = services.FirstOrDefault(d => d.ServiceType == typeof(IAuditStore)) diff --git a/src/SensitiveFlow.Audit/Outbox/AuditOutboxDispatcher.cs b/src/SensitiveFlow.Audit/Outbox/AuditOutboxDispatcher.cs new file mode 100644 index 0000000..881dc16 --- /dev/null +++ b/src/SensitiveFlow.Audit/Outbox/AuditOutboxDispatcher.cs @@ -0,0 +1,105 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using SensitiveFlow.Core.Interfaces; + +namespace SensitiveFlow.Audit.Outbox; + +/// +/// Hosted service that dispatches durable audit outbox entries through registered publishers. +/// +public sealed class AuditOutboxDispatcher : BackgroundService +{ + private readonly IDurableAuditOutbox? _outbox; + private readonly IEnumerable _publishers; + private readonly AuditOutboxDispatcherOptions _options; + private readonly ILogger? _logger; + + /// Initializes a new instance. + public AuditOutboxDispatcher( + IDurableAuditOutbox? outbox, + IEnumerable publishers, + AuditOutboxDispatcherOptions options, + ILogger? logger = null) + { + _outbox = outbox; + _publishers = publishers ?? []; + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger; + } + + /// + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + if (_outbox is null || !_publishers.Any()) + { + return; + } + + while (!stoppingToken.IsCancellationRequested) + { + await DispatchOnceAsync(stoppingToken); + await Task.Delay(_options.PollInterval, stoppingToken); + } + } + + /// Runs one dispatcher polling cycle. Exposed for tests. + public async Task DispatchOnceAsync(CancellationToken cancellationToken = default) + { + if (_outbox is null) + { + return; + } + + var publishers = _publishers.ToArray(); + if (publishers.Length == 0) + { + return; + } + + var batch = await _outbox.DequeueBatchAsync(_options.BatchSize, cancellationToken); + foreach (var entry in batch) + { + if (ShouldDeadLetter(entry.Attempts)) + { + await _outbox.MarkDeadLetteredAsync(entry.Id, "Max audit outbox attempts reached.", cancellationToken); + SensitiveFlowAuditDiagnostics.RecordDeadLettered(); + continue; + } + + try + { + await DelayForBackoffAsync(entry.Attempts, cancellationToken); + foreach (var publisher in publishers) + { + await publisher.PublishAsync(entry, cancellationToken); + } + + await _outbox.MarkProcessedAsync([entry.Id], cancellationToken); + SensitiveFlowAuditDiagnostics.RecordDispatched(); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger?.LogWarning(ex, "Audit outbox dispatch failed for entry {EntryId}.", entry.Id); + await _outbox.MarkFailedAsync(entry.Id, ex.Message, cancellationToken); + SensitiveFlowAuditDiagnostics.RecordFailed(); + } + } + } + + private bool ShouldDeadLetter(int attempts) + => _options.DeadLetterAfterMax && attempts >= _options.MaxAttempts; + + private Task DelayForBackoffAsync(int attempts, CancellationToken cancellationToken) + { + if (attempts <= 0 || _options.Backoff == BackoffStrategy.None) + { + return Task.CompletedTask; + } + + var multiplier = _options.Backoff == BackoffStrategy.Exponential + ? Math.Pow(2, Math.Min(attempts, 6)) + : attempts; + var delay = TimeSpan.FromMilliseconds(Math.Min(1000, 100 * multiplier)); + return Task.Delay(delay, cancellationToken); + } +} diff --git a/src/SensitiveFlow.Audit/Outbox/AuditOutboxDispatcherOptions.cs b/src/SensitiveFlow.Audit/Outbox/AuditOutboxDispatcherOptions.cs new file mode 100644 index 0000000..3d1cf19 --- /dev/null +++ b/src/SensitiveFlow.Audit/Outbox/AuditOutboxDispatcherOptions.cs @@ -0,0 +1,20 @@ +namespace SensitiveFlow.Audit.Outbox; + +/// Options for the hosted audit outbox dispatcher. +public sealed class AuditOutboxDispatcherOptions +{ + /// Polling interval. Defaults to one second. + public TimeSpan PollInterval { get; set; } = TimeSpan.FromSeconds(1); + + /// Maximum number of outbox entries dequeued per poll. Defaults to 100. + public int BatchSize { get; set; } = 100; + + /// Maximum number of delivery attempts before dead-lettering. Defaults to 5. + public int MaxAttempts { get; set; } = 5; + + /// Backoff strategy between failed attempts. + public BackoffStrategy Backoff { get; set; } = BackoffStrategy.Exponential; + + /// Whether entries should be treated as dead-lettered after max attempts. + public bool DeadLetterAfterMax { get; set; } = true; +} diff --git a/src/SensitiveFlow.Audit/Outbox/BackoffStrategy.cs b/src/SensitiveFlow.Audit/Outbox/BackoffStrategy.cs new file mode 100644 index 0000000..38c85c9 --- /dev/null +++ b/src/SensitiveFlow.Audit/Outbox/BackoffStrategy.cs @@ -0,0 +1,14 @@ +namespace SensitiveFlow.Audit.Outbox; + +/// Backoff strategy used by the audit outbox dispatcher. +public enum BackoffStrategy +{ + /// No delay beyond the polling interval. + None, + + /// Linear delay based on attempt count. + Linear, + + /// Exponential delay based on attempt count. + Exponential, +} diff --git a/src/SensitiveFlow.Audit/Outbox/InMemoryAuditOutbox.cs b/src/SensitiveFlow.Audit/Outbox/InMemoryAuditOutbox.cs index 8169521..292bce4 100644 --- a/src/SensitiveFlow.Audit/Outbox/InMemoryAuditOutbox.cs +++ b/src/SensitiveFlow.Audit/Outbox/InMemoryAuditOutbox.cs @@ -8,6 +8,7 @@ namespace SensitiveFlow.Audit.Outbox; /// In-memory audit outbox implementation for tests, samples, and local development. /// It is not durable and should not be used as the only production outbox. /// +[Obsolete("InMemoryAuditOutbox is for tests/local development only. Use AddEfCoreAuditOutbox() or AddAuditOutbox() for durable production delivery.", error: false)] public sealed class InMemoryAuditOutbox : IAuditOutbox { private readonly ConcurrentQueue _records = new(); diff --git a/src/SensitiveFlow.Audit/Outbox/SensitiveFlowAuditDiagnostics.cs b/src/SensitiveFlow.Audit/Outbox/SensitiveFlowAuditDiagnostics.cs new file mode 100644 index 0000000..c85143d --- /dev/null +++ b/src/SensitiveFlow.Audit/Outbox/SensitiveFlowAuditDiagnostics.cs @@ -0,0 +1,46 @@ +using System.Diagnostics.Metrics; +using SensitiveFlow.Core.Diagnostics; + +namespace SensitiveFlow.Audit.Outbox; + +/// Metrics emitted by SensitiveFlow audit infrastructure. +public static class SensitiveFlowAuditDiagnostics +{ + private static readonly Meter Meter = new(SensitiveFlowDiagnostics.MeterName); + + /// Number of audit outbox records enqueued. + public static readonly Counter OutboxEnqueued = Meter.CreateCounter( + "sensitiveflow.audit.outbox.enqueued", + unit: "records", + description: "Audit records enqueued in an outbox."); + + /// Number of audit outbox records dispatched successfully. + public static readonly Counter OutboxDispatched = Meter.CreateCounter( + "sensitiveflow.audit.outbox.dispatched", + unit: "records", + description: "Audit outbox records dispatched successfully."); + + /// Number of audit outbox dispatch failures. + public static readonly Counter OutboxFailed = Meter.CreateCounter( + "sensitiveflow.audit.outbox.failed", + unit: "records", + description: "Audit outbox dispatch failures."); + + /// Number of audit outbox entries dead-lettered. + public static readonly Counter OutboxDeadLettered = Meter.CreateCounter( + "sensitiveflow.audit.outbox.deadlettered", + unit: "records", + description: "Audit outbox records that reached max delivery attempts."); + + /// Records one enqueue. + public static void RecordEnqueued(long count = 1) => OutboxEnqueued.Add(count); + + /// Records one successful dispatch. + public static void RecordDispatched(long count = 1) => OutboxDispatched.Add(count); + + /// Records one dispatch failure. + public static void RecordFailed(long count = 1) => OutboxFailed.Add(count); + + /// Records one dead-letter event. + public static void RecordDeadLettered(long count = 1) => OutboxDeadLettered.Add(count); +} diff --git a/src/SensitiveFlow.Audit/SensitiveFlow.Audit.csproj b/src/SensitiveFlow.Audit/SensitiveFlow.Audit.csproj index a0241b6..0cf90fe 100644 --- a/src/SensitiveFlow.Audit/SensitiveFlow.Audit.csproj +++ b/src/SensitiveFlow.Audit/SensitiveFlow.Audit.csproj @@ -6,6 +6,7 @@ + diff --git a/src/SensitiveFlow.Core/Interfaces/IAuditOutboxPublisher.cs b/src/SensitiveFlow.Core/Interfaces/IAuditOutboxPublisher.cs new file mode 100644 index 0000000..35ca8c4 --- /dev/null +++ b/src/SensitiveFlow.Core/Interfaces/IAuditOutboxPublisher.cs @@ -0,0 +1,13 @@ +using SensitiveFlow.Core.Models; + +namespace SensitiveFlow.Core.Interfaces; + +/// +/// Delivers audit outbox entries to an application-owned destination such as a +/// webhook, queue, event bus, or stream. +/// +public interface IAuditOutboxPublisher +{ + /// Publishes one audit outbox entry. + Task PublishAsync(AuditOutboxEntry entry, CancellationToken cancellationToken = default); +} diff --git a/src/SensitiveFlow.Core/Interfaces/IAuditStoreTransaction.cs b/src/SensitiveFlow.Core/Interfaces/IAuditStoreTransaction.cs new file mode 100644 index 0000000..f586e07 --- /dev/null +++ b/src/SensitiveFlow.Core/Interfaces/IAuditStoreTransaction.cs @@ -0,0 +1,13 @@ +namespace SensitiveFlow.Core.Interfaces; + +/// +/// Optional audit-store capability for executing related audit operations in the +/// same durable transaction. +/// +public interface IAuditStoreTransaction +{ + /// Executes the supplied operation in the store's transaction boundary. + Task ExecuteInTransactionAsync( + Func operation, + CancellationToken cancellationToken = default); +} diff --git a/src/SensitiveFlow.Core/Interfaces/IDurableAuditOutbox.cs b/src/SensitiveFlow.Core/Interfaces/IDurableAuditOutbox.cs new file mode 100644 index 0000000..c85235c --- /dev/null +++ b/src/SensitiveFlow.Core/Interfaces/IDurableAuditOutbox.cs @@ -0,0 +1,31 @@ +using SensitiveFlow.Core.Models; + +namespace SensitiveFlow.Core.Interfaces; + +/// +/// Durable audit outbox that supports polling, acknowledgements, and failure tracking. +/// +public interface IDurableAuditOutbox : IAuditOutbox +{ + /// Dequeues a batch of pending entries for dispatch. + Task> DequeueBatchAsync( + int max, + CancellationToken cancellationToken = default); + + /// Marks entries as processed after successful dispatch. + Task MarkProcessedAsync( + IEnumerable ids, + CancellationToken cancellationToken = default); + + /// Marks one entry as failed with a safe diagnostic message. + Task MarkFailedAsync( + Guid id, + string error, + CancellationToken cancellationToken = default); + + /// Marks one entry as dead-lettered after max retries. + Task MarkDeadLetteredAsync( + Guid id, + string reason, + CancellationToken cancellationToken = default); +} diff --git a/src/SensitiveFlow.Core/Models/AuditOutboxEntry.cs b/src/SensitiveFlow.Core/Models/AuditOutboxEntry.cs new file mode 100644 index 0000000..8d20e2d --- /dev/null +++ b/src/SensitiveFlow.Core/Models/AuditOutboxEntry.cs @@ -0,0 +1,25 @@ +namespace SensitiveFlow.Core.Models; + +/// +/// Durable outbox entry that tracks delivery attempts for an audit record. +/// +public sealed record AuditOutboxEntry +{ + /// Unique identifier of the outbox entry. + public Guid Id { get; init; } = Guid.NewGuid(); + + /// Audit record to deliver. + public required AuditRecord Record { get; init; } + + /// Number of dispatch attempts. + public int Attempts { get; init; } + + /// Timestamp when the entry was enqueued. + public DateTimeOffset EnqueuedAt { get; init; } = DateTimeOffset.UtcNow; + + /// Timestamp of the latest dispatch attempt, when any. + public DateTimeOffset? LastAttemptAt { get; init; } + + /// Latest dispatch error, when any. + public string? LastError { get; init; } +} diff --git a/src/SensitiveFlow.Core/Profiles/SensitiveFlowDefaults.cs b/src/SensitiveFlow.Core/Profiles/SensitiveFlowDefaults.cs index 304fad2..ea4c2a2 100644 --- a/src/SensitiveFlow.Core/Profiles/SensitiveFlowDefaults.cs +++ b/src/SensitiveFlow.Core/Profiles/SensitiveFlowDefaults.cs @@ -33,4 +33,9 @@ public static class SensitiveFlowDefaults /// Default health-check registration name for token-store checks. /// public const string TokenStoreHealthCheckName = "sensitiveflow-token-store"; + + /// + /// Default health-check registration name for audit-outbox checks. + /// + public const string AuditOutboxHealthCheckName = "sensitiveflow-audit-outbox"; } diff --git a/src/SensitiveFlow.Diagnostics/SensitiveFlow.Diagnostics.csproj b/src/SensitiveFlow.Diagnostics/SensitiveFlow.Diagnostics.csproj index e2a2c99..0de716e 100644 --- a/src/SensitiveFlow.Diagnostics/SensitiveFlow.Diagnostics.csproj +++ b/src/SensitiveFlow.Diagnostics/SensitiveFlow.Diagnostics.csproj @@ -6,6 +6,7 @@ + diff --git a/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs index 1e3700e..851abc2 100644 --- a/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs +++ b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; using SensitiveFlow.Core.Discovery; using SensitiveFlow.Core.Interfaces; using SensitiveFlow.Core.Policies; @@ -56,6 +57,7 @@ public SensitiveFlowConfigurationReport Validate(IServiceProvider services) AddEfCoreDiagnostics(provider, diagnostics); AddRetentionDiagnostics(provider, diagnostics); AddAspNetCoreDiagnostics(provider, diagnostics); + AddOutboxDiagnostics(provider, diagnostics); var sensitiveFlowOptions = provider.GetService(); if (sensitiveFlowOptions is not null) @@ -148,6 +150,30 @@ private static void AddAspNetCoreDiagnostics( } } + private static void AddOutboxDiagnostics( + IServiceProvider provider, + ICollection diagnostics) + { + var outbox = provider.GetService(); + if (outbox is null) + { + return; + } + + var environment = provider.GetService(); + if (outbox.GetType().FullName == "SensitiveFlow.Audit.Outbox.InMemoryAuditOutbox" + && environment?.IsDevelopment() == false) + { + diagnostics.Add(Warning("SF-CONFIG-013", "In-memory audit outbox is configured outside Development. Use AddEfCoreAuditOutbox() or AddAuditOutbox().")); + } + + var durableOutbox = provider.GetService(); + if (durableOutbox is not null && !provider.GetServices().Any()) + { + diagnostics.Add(Warning("SF-CONFIG-014", "IDurableAuditOutbox is registered, but no IAuditOutboxPublisher registration was found.")); + } + } + private static void AddPolicyDiagnostics( IServiceProvider provider, SensitiveFlowOptions options, diff --git a/src/SensitiveFlow.HealthChecks/Checks/AuditOutboxHealthCheck.cs b/src/SensitiveFlow.HealthChecks/Checks/AuditOutboxHealthCheck.cs new file mode 100644 index 0000000..c68a24e --- /dev/null +++ b/src/SensitiveFlow.HealthChecks/Checks/AuditOutboxHealthCheck.cs @@ -0,0 +1,35 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Hosting; +using SensitiveFlow.Core.Interfaces; + +namespace SensitiveFlow.HealthChecks.Checks; + +/// Health check for audit outbox configuration. +public sealed class AuditOutboxHealthCheck : IHealthCheck +{ + private readonly IAuditOutbox _outbox; + private readonly IHostEnvironment? _environment; + + /// Initializes a new instance. + public AuditOutboxHealthCheck(IAuditOutbox outbox, IHostEnvironment? environment = null) + { + _outbox = outbox; + _environment = environment; + } + + /// + public Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + var isInMemory = _outbox.GetType().FullName == "SensitiveFlow.Audit.Outbox.InMemoryAuditOutbox"; + if (isInMemory && _environment?.IsDevelopment() == false) + { + return Task.FromResult(HealthCheckResult.Degraded("In-memory audit outbox is configured outside Development.")); + } + + return Task.FromResult(HealthCheckResult.Healthy("Audit outbox resolved.")); + } +} diff --git a/src/SensitiveFlow.HealthChecks/Extensions/SensitiveFlowHealthCheckBuilderExtensions.cs b/src/SensitiveFlow.HealthChecks/Extensions/SensitiveFlowHealthCheckBuilderExtensions.cs index cc33f15..3963a46 100644 --- a/src/SensitiveFlow.HealthChecks/Extensions/SensitiveFlowHealthCheckBuilderExtensions.cs +++ b/src/SensitiveFlow.HealthChecks/Extensions/SensitiveFlowHealthCheckBuilderExtensions.cs @@ -38,4 +38,15 @@ public static IHealthChecksBuilder AddTokenStoreCheck( ArgumentNullException.ThrowIfNull(builder); return builder.AddCheck(name, failureStatus, tags ?? Array.Empty()); } + + /// Adds an audit outbox health check. + public static IHealthChecksBuilder AddAuditOutboxCheck( + this IHealthChecksBuilder builder, + string name = SensitiveFlowDefaults.AuditOutboxHealthCheckName, + HealthStatus? failureStatus = HealthStatus.Degraded, + IEnumerable? tags = null) + { + ArgumentNullException.ThrowIfNull(builder); + return builder.AddCheck(name, failureStatus, tags ?? Array.Empty()); + } } diff --git a/src/SensitiveFlow.HealthChecks/SensitiveFlow.HealthChecks.csproj b/src/SensitiveFlow.HealthChecks/SensitiveFlow.HealthChecks.csproj index d40b0e1..6634ff1 100644 --- a/src/SensitiveFlow.HealthChecks/SensitiveFlow.HealthChecks.csproj +++ b/src/SensitiveFlow.HealthChecks/SensitiveFlow.HealthChecks.csproj @@ -6,6 +6,7 @@ + diff --git a/tests/SensitiveFlow.Audit.EFCore.Outbox.Tests/EfCoreAuditOutboxTests.cs b/tests/SensitiveFlow.Audit.EFCore.Outbox.Tests/EfCoreAuditOutboxTests.cs new file mode 100644 index 0000000..006f476 --- /dev/null +++ b/tests/SensitiveFlow.Audit.EFCore.Outbox.Tests/EfCoreAuditOutboxTests.cs @@ -0,0 +1,205 @@ +using FluentAssertions; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using SensitiveFlow.Audit.EFCore; +using SensitiveFlow.Audit.EFCore.Outbox.Extensions; +using SensitiveFlow.Audit.EFCore.Outbox.Stores; +using SensitiveFlow.Core.Enums; +using SensitiveFlow.Core.Interfaces; +using SensitiveFlow.Core.Models; + +namespace SensitiveFlow.Audit.EFCore.Outbox.Tests; + +public sealed class EfCoreAuditOutboxTests : IAsyncLifetime +{ + private DbContextFactory? _factory; + + public async Task InitializeAsync() + { + _factory = new DbContextFactory(); + using var ctx = await _factory.CreateDbContextAsync(); + await ctx.Database.EnsureCreatedAsync(); + } + + public async Task DisposeAsync() + { + if (_factory != null) + { + await _factory.DisposeAsync(); + } + } + + [Fact] + public async Task EnqueueAsync_AddsEntryToDatabase() + { + var outbox = new EfCoreAuditOutbox(_factory!); + var record = SampleRecord(); + + await outbox.EnqueueAsync(record); + + using var ctx = await _factory!.CreateDbContextAsync(); + var entries = await ctx.Set() + .Where(e => e.AuditRecordId == record.Id.ToString()) + .ToListAsync(); + + entries.Should().HaveCount(1); + entries[0].Payload.Should().NotBeNullOrEmpty(); + entries[0].IsProcessed.Should().BeFalse(); + entries[0].IsDeadLettered.Should().BeFalse(); + entries[0].Attempts.Should().Be(0); + } + + [Fact] + public async Task DequeueBatchAsync_ReturnsPendingEntries() + { + var outbox = new EfCoreAuditOutbox(_factory!); + + // Enqueue 3 records + for (int i = 0; i < 3; i++) + { + await outbox.EnqueueAsync(SampleRecord()); + } + + // Dequeue first 2 + var batch = await outbox.DequeueBatchAsync(2); + + batch.Should().HaveCount(2); + batch.Should().AllSatisfy(e => e.Attempts.Should().Be(0)); + } + + [Fact] + public async Task DequeueBatchAsync_IgnoresProcessedEntries() + { + var outbox = new EfCoreAuditOutbox(_factory!); + var record1 = SampleRecord(); + var record2 = SampleRecord(); + + await outbox.EnqueueAsync(record1); + await outbox.EnqueueAsync(record2); + + // Get first batch + var batch1 = await outbox.DequeueBatchAsync(10); + var processedId = batch1.First().Id; + + // Mark as processed + await outbox.MarkProcessedAsync([processedId]); + + // Dequeue again - should only get the unprocessed one + var batch2 = await outbox.DequeueBatchAsync(10); + + batch2.Should().HaveCount(1); + batch2[0].Id.Should().NotBe(processedId); + } + + [Fact] + public async Task MarkProcessedAsync_UpdatesEntry() + { + var outbox = new EfCoreAuditOutbox(_factory!); + await outbox.EnqueueAsync(SampleRecord()); + + var batch = await outbox.DequeueBatchAsync(10); + await outbox.MarkProcessedAsync([batch[0].Id]); + + using var ctx = await _factory!.CreateDbContextAsync(); + var entry = await ctx.Set() + .FirstAsync(e => e.Id == batch[0].Id); + + entry.IsProcessed.Should().BeTrue(); + entry.ProcessedAt.Should().NotBeNull(); + } + + [Fact] + public async Task MarkFailedAsync_IncrementsAttemptsAndSetsError() + { + var outbox = new EfCoreAuditOutbox(_factory!); + await outbox.EnqueueAsync(SampleRecord()); + + var batch = await outbox.DequeueBatchAsync(10); + var entryId = batch[0].Id; + var errorMsg = "Network timeout"; + + await outbox.MarkFailedAsync(entryId, errorMsg); + + using var ctx = await _factory!.CreateDbContextAsync(); + var entry = await ctx.Set() + .FirstAsync(e => e.Id == entryId); + + entry.Attempts.Should().Be(1); + entry.LastError.Should().Be(errorMsg); + entry.LastAttemptAt.Should().NotBeNull(); + } + + [Fact] + public async Task DequeueBatchAsync_IgnoresDeadLetteredEntries() + { + var outbox = new EfCoreAuditOutbox(_factory!); + var record1 = SampleRecord(); + var record2 = SampleRecord(); + + await outbox.EnqueueAsync(record1); + await outbox.EnqueueAsync(record2); + + var batch = await outbox.DequeueBatchAsync(10); + var deadLetterId = batch[0].Id; + + // Mark as dead-lettered + using (var ctx = await _factory!.CreateDbContextAsync()) + { + var entry = await ctx.Set() + .FirstAsync(e => e.Id == deadLetterId); + entry.IsDeadLettered = true; + entry.DeadLetterReason = "Max retries exceeded"; + await ctx.SaveChangesAsync(); + } + + // Dequeue again - should only get the non-dead-lettered one + var batch2 = await outbox.DequeueBatchAsync(10); + + batch2.Should().HaveCount(1); + batch2[0].Id.Should().NotBe(deadLetterId); + } + + private static AuditRecord SampleRecord() => new() + { + DataSubjectId = $"subject-{Guid.NewGuid()}", + Entity = "Customer", + Field = "Email", + Operation = AuditOperation.Update, + }; + + // Test fixture for in-memory SQLite with shared connection + private sealed class DbContextFactory : IDbContextFactory, IAsyncDisposable + { + private SqliteConnection? _connection; + + public AuditDbContext CreateDbContext() + { + _connection ??= new SqliteConnection("Data Source=:memory:"); + if (_connection.State == System.Data.ConnectionState.Closed) + { + _connection.Open(); + } + + var options = new DbContextOptionsBuilder() + .UseSqlite(_connection) + .Options; + return new AuditDbContext(options); + } + + public async Task CreateDbContextAsync(CancellationToken cancellationToken = default) + { + var ctx = CreateDbContext(); + await ctx.Database.EnsureCreatedAsync(cancellationToken); + return ctx; + } + + public async ValueTask DisposeAsync() + { + if (_connection != null) + { + await _connection.CloseAsync(); + await _connection.DisposeAsync(); + } + } + } +} diff --git a/tests/SensitiveFlow.Audit.EFCore.Outbox.Tests/SensitiveFlow.Audit.EFCore.Outbox.Tests.csproj b/tests/SensitiveFlow.Audit.EFCore.Outbox.Tests/SensitiveFlow.Audit.EFCore.Outbox.Tests.csproj new file mode 100644 index 0000000..ef08c0e --- /dev/null +++ b/tests/SensitiveFlow.Audit.EFCore.Outbox.Tests/SensitiveFlow.Audit.EFCore.Outbox.Tests.csproj @@ -0,0 +1,17 @@ + + + false + false + + + + + + + + + + + + + diff --git a/tests/SensitiveFlow.Audit.Tests/AuditOutboxTests.cs b/tests/SensitiveFlow.Audit.Tests/AuditOutboxTests.cs index 1c049cb..88345c2 100644 --- a/tests/SensitiveFlow.Audit.Tests/AuditOutboxTests.cs +++ b/tests/SensitiveFlow.Audit.Tests/AuditOutboxTests.cs @@ -8,6 +8,7 @@ namespace SensitiveFlow.Audit.Tests; +#pragma warning disable CS0618 // Type or member is obsolete public sealed class AuditOutboxTests { [Fact] @@ -89,3 +90,4 @@ public Task> QueryByDataSubjectAsync( _records.Where(r => r.DataSubjectId == dataSubjectId).ToArray()); } } +#pragma warning restore CS0618 // Type or member is obsolete From 364fe9a860b249cb289227bfd49b470036748908 Mon Sep 17 00:00:00 2001 From: Mateus Silva Date: Mon, 11 May 2026 11:44:06 -0300 Subject: [PATCH 07/23] feat: Enhance logging by masking sensitive email information and adding sanitization for log entries --- samples/MinimalApi.Sample/Program.cs | 4 ++-- .../Controllers/CustomersController.cs | 4 ++-- .../Extensions/StringAnonymizationExtensions.cs | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/samples/MinimalApi.Sample/Program.cs b/samples/MinimalApi.Sample/Program.cs index f7569bc..5608293 100644 --- a/samples/MinimalApi.Sample/Program.cs +++ b/samples/MinimalApi.Sample/Program.cs @@ -178,7 +178,7 @@ await scope.ServiceProvider.GetRequiredService() logger.LogInformation("Customer created - name: {Name}, email: {[Sensitive]Email}", customer.Name.MaskName(), - customer.Email); + customer.Email.MaskEmail()); return Results.Created($"/customers/{customer.DataSubjectId}", ToResponse(customer)); }) @@ -211,7 +211,7 @@ await auditStore.AppendAsync(new AuditRecord }, ct); logger.LogInformation("Customer {Id} accessed - email: {[Sensitive]Email}", - id, customer.Email); + id.SanitizeForLog(), customer.Email.MaskEmail()); return Results.Ok(ToResponse(customer)); }) diff --git a/samples/WebApi.Sample/Controllers/CustomersController.cs b/samples/WebApi.Sample/Controllers/CustomersController.cs index 9996823..ab184e1 100644 --- a/samples/WebApi.Sample/Controllers/CustomersController.cs +++ b/samples/WebApi.Sample/Controllers/CustomersController.cs @@ -59,7 +59,7 @@ public async Task Create([FromBody] CreateCustomerRequest request _logger.LogInformation("Customer created - name: {Name}, email: {[Sensitive]Email}", customer.Name.MaskName(), - customer.Email); + customer.Email.MaskEmail()); return CreatedAtAction(nameof(Get), new { id = customer.DataSubjectId }, ToResponse(customer)); } @@ -86,7 +86,7 @@ await _auditStore.AppendAsync(new AuditRecord }, ct); _logger.LogInformation("Customer {Id} accessed - email: {[Sensitive]Email}", - id, customer.Email); + id.SanitizeForLog(), customer.Email.MaskEmail()); return Ok(ToResponse(customer)); } diff --git a/src/SensitiveFlow.Anonymization/Extensions/StringAnonymizationExtensions.cs b/src/SensitiveFlow.Anonymization/Extensions/StringAnonymizationExtensions.cs index 5dec44f..2f3feea 100644 --- a/src/SensitiveFlow.Anonymization/Extensions/StringAnonymizationExtensions.cs +++ b/src/SensitiveFlow.Anonymization/Extensions/StringAnonymizationExtensions.cs @@ -70,6 +70,20 @@ public static string MaskPhone(this string value) => public static string MaskName(this string value) => NameMasker.Mask(value); + /// + /// Sanitizes a string for safe use in log messages by stripping carriage returns + /// and line feeds. This prevents log-forging attacks where malicious input + /// containing newlines could forge additional log entries. + /// + /// + /// Call this on any user-provided value before passing it to a logger + /// when the value is not already masked or pseudonymized. + /// + public static string SanitizeForLog(this string? value) => + string.IsNullOrEmpty(value) + ? string.Empty + : value.Replace("\r", string.Empty).Replace("\n", string.Empty); + // ── Pseudonymization (data remains personal) ─────────────────────────────── /// From b6329fecaa2a6dfc9c4e7a1885a5ccb580f83994 Mon Sep 17 00:00:00 2001 From: Mateus Silva Date: Mon, 11 May 2026 12:34:55 -0300 Subject: [PATCH 08/23] feat: Add unit tests for Audit Outbox and Health Check functionalities --- .../Extensions/SanitizeForLogTests.cs | 58 +++++ ...xEFCoreServiceCollectionExtensionsTests.cs | 75 ++++++ .../AuditOutboxDispatcherTests.cs | 237 ++++++++++++++++++ .../OutboxAuditStoreTests.cs | 121 +++++++++ .../AuditOutboxHealthCheckTests.cs | 86 +++++++ .../SensitiveFlow.HealthChecks.Tests.csproj | 2 + 6 files changed, 579 insertions(+) create mode 100644 tests/SensitiveFlow.Anonymization.Tests/Extensions/SanitizeForLogTests.cs create mode 100644 tests/SensitiveFlow.Audit.EFCore.Outbox.Tests/OutboxEFCoreServiceCollectionExtensionsTests.cs create mode 100644 tests/SensitiveFlow.Audit.Tests/AuditOutboxDispatcherTests.cs create mode 100644 tests/SensitiveFlow.Audit.Tests/OutboxAuditStoreTests.cs create mode 100644 tests/SensitiveFlow.HealthChecks.Tests/AuditOutboxHealthCheckTests.cs diff --git a/tests/SensitiveFlow.Anonymization.Tests/Extensions/SanitizeForLogTests.cs b/tests/SensitiveFlow.Anonymization.Tests/Extensions/SanitizeForLogTests.cs new file mode 100644 index 0000000..ea6ae16 --- /dev/null +++ b/tests/SensitiveFlow.Anonymization.Tests/Extensions/SanitizeForLogTests.cs @@ -0,0 +1,58 @@ +using FluentAssertions; +using SensitiveFlow.Anonymization.Extensions; + +namespace SensitiveFlow.Anonymization.Tests.Extensions; + +public sealed class SanitizeForLogTests +{ + [Fact] + public void SanitizeForLog_RemovesCarriageReturn() + { + var result = "hello\rworld".SanitizeForLog(); + result.Should().Be("helloworld"); + } + + [Fact] + public void SanitizeForLog_RemovesLineFeed() + { + var result = "hello\nworld".SanitizeForLog(); + result.Should().Be("helloworld"); + } + + [Fact] + public void SanitizeForLog_RemovesBothCrAndLf() + { + var result = "a\r\nb\nc\r".SanitizeForLog(); + result.Should().Be("abc"); + } + + [Fact] + public void SanitizeForLog_Null_ReturnsEmpty() + { + string? input = null; + var result = input.SanitizeForLog(); + result.Should().BeEmpty(); + } + + [Fact] + public void SanitizeForLog_Empty_ReturnsEmpty() + { + var result = "".SanitizeForLog(); + result.Should().BeEmpty(); + } + + [Fact] + public void SanitizeForLog_CleanString_Unchanged() + { + var result = "clean-string_123".SanitizeForLog(); + result.Should().Be("clean-string_123"); + } + + [Fact] + public void SanitizeForLog_PreventsLogForging() + { + var malicious = "user\n[ERROR] Fake alert\n[INFO] Compromised"; + var result = malicious.SanitizeForLog(); + result.Should().NotContain("\n").And.NotContain("\r"); + } +} diff --git a/tests/SensitiveFlow.Audit.EFCore.Outbox.Tests/OutboxEFCoreServiceCollectionExtensionsTests.cs b/tests/SensitiveFlow.Audit.EFCore.Outbox.Tests/OutboxEFCoreServiceCollectionExtensionsTests.cs new file mode 100644 index 0000000..c40995c --- /dev/null +++ b/tests/SensitiveFlow.Audit.EFCore.Outbox.Tests/OutboxEFCoreServiceCollectionExtensionsTests.cs @@ -0,0 +1,75 @@ +using FluentAssertions; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using SensitiveFlow.Audit.EFCore; +using SensitiveFlow.Audit.EFCore.Outbox.Extensions; +using SensitiveFlow.Audit.Outbox; +using SensitiveFlow.Core.Interfaces; + +namespace SensitiveFlow.Audit.EFCore.Outbox.Tests; + +public sealed class OutboxEFCoreServiceCollectionExtensionsTests +{ + [Fact] + public void AddEfCoreAuditOutbox_RegistersDurableOutbox() + { + var services = new ServiceCollection(); + services.AddDbContextFactory(options => options.UseSqlite("Data Source=:memory:")); + + services.AddEfCoreAuditOutbox(); + var provider = services.BuildServiceProvider(); + + provider.GetService().Should().NotBeNull(); + } + + [Fact] + public void AddEfCoreAuditOutbox_RegistersIAuditOutbox() + { + var services = new ServiceCollection(); + services.AddDbContextFactory(options => options.UseSqlite("Data Source=:memory:")); + + services.AddEfCoreAuditOutbox(); + var provider = services.BuildServiceProvider(); + + provider.GetService().Should().NotBeNull(); + } + + [Fact] + public void AddEfCoreAuditOutbox_RegistersDispatcherOptions() + { + var services = new ServiceCollection(); + services.AddDbContextFactory(options => options.UseSqlite("Data Source=:memory:")); + + services.AddEfCoreAuditOutbox(options => options.BatchSize = 42); + var provider = services.BuildServiceProvider(); + + var options = provider.GetRequiredService(); + options.BatchSize.Should().Be(42); + } + + [Fact] + public void AddEfCoreAuditOutbox_RegistersHostedService() + { + var services = new ServiceCollection(); + services.AddDbContextFactory(options => options.UseSqlite("Data Source=:memory:")); + + services.AddEfCoreAuditOutbox(); + + var hostedServices = services + .Where(d => d.ServiceType == typeof(IHostedService)) + .ToList(); + + hostedServices.Should().ContainSingle(d => d.ImplementationType == typeof(AuditOutboxDispatcher)); + } + + [Fact] + public void AddEfCoreAuditOutbox_NullServices_ThrowsArgumentNullException() + { + IServiceCollection? services = null; + + var act = () => services!.AddEfCoreAuditOutbox(); + + act.Should().Throw().WithParameterName("services"); + } +} diff --git a/tests/SensitiveFlow.Audit.Tests/AuditOutboxDispatcherTests.cs b/tests/SensitiveFlow.Audit.Tests/AuditOutboxDispatcherTests.cs new file mode 100644 index 0000000..17be0a6 --- /dev/null +++ b/tests/SensitiveFlow.Audit.Tests/AuditOutboxDispatcherTests.cs @@ -0,0 +1,237 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging; +using NSubstitute; +using SensitiveFlow.Audit.Outbox; +using SensitiveFlow.Core.Enums; +using SensitiveFlow.Core.Interfaces; +using SensitiveFlow.Core.Models; + +namespace SensitiveFlow.Audit.Tests; + +public sealed class AuditOutboxDispatcherTests +{ + [Fact] + public async Task DispatchOnceAsync_WithNoOutbox_ReturnsImmediately() + { + var dispatcher = new AuditOutboxDispatcher( + null, + [Substitute.For()], + new AuditOutboxDispatcherOptions()); + + await dispatcher.DispatchOnceAsync(); + } + + [Fact] + public async Task DispatchOnceAsync_WithNoPublishers_ReturnsImmediately() + { + var outbox = Substitute.For(); + var dispatcher = new AuditOutboxDispatcher( + outbox, + [], + new AuditOutboxDispatcherOptions()); + + await dispatcher.DispatchOnceAsync(); + + await outbox.DidNotReceive().DequeueBatchAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task DispatchOnceAsync_DispatchesEntryToPublisher() + { + var outbox = Substitute.For(); + var publisher = Substitute.For(); + var entry = SampleEntry(); + + outbox.DequeueBatchAsync(100, Arg.Any()) + .Returns([entry]); + + var dispatcher = new AuditOutboxDispatcher( + outbox, + [publisher], + new AuditOutboxDispatcherOptions { BatchSize = 100 }); + + await dispatcher.DispatchOnceAsync(); + + await publisher.Received(1).PublishAsync(entry, Arg.Any()); + await outbox.Received(1).MarkProcessedAsync( + Arg.Is>(ids => ids.Contains(entry.Id)), + Arg.Any()); + } + + [Fact] + public async Task DispatchOnceAsync_MarksFailed_WhenPublisherThrows() + { + var outbox = Substitute.For(); + var publisher = Substitute.For(); + var entry = SampleEntry(); + + outbox.DequeueBatchAsync(100, Arg.Any()) + .Returns([entry]); + publisher.PublishAsync(entry, Arg.Any()) + .Returns(Task.FromException(new InvalidOperationException("boom"))); + + var dispatcher = new AuditOutboxDispatcher( + outbox, + [publisher], + new AuditOutboxDispatcherOptions { BatchSize = 100 }); + + await dispatcher.DispatchOnceAsync(); + + await outbox.Received(1).MarkFailedAsync( + entry.Id, + "boom", + Arg.Any()); + } + + [Fact] + public async Task DispatchOnceAsync_DeadLetters_WhenMaxAttemptsReached() + { + var outbox = Substitute.For(); + var publisher = Substitute.For(); + var entry = SampleEntry(attempts: 5); + + outbox.DequeueBatchAsync(100, Arg.Any()) + .Returns([entry]); + + var dispatcher = new AuditOutboxDispatcher( + outbox, + [publisher], + new AuditOutboxDispatcherOptions + { + BatchSize = 100, + MaxAttempts = 5, + DeadLetterAfterMax = true, + }); + + await dispatcher.DispatchOnceAsync(); + + await outbox.Received(1).MarkDeadLetteredAsync( + entry.Id, + "Max audit outbox attempts reached.", + Arg.Any()); + await publisher.DidNotReceive().PublishAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task DispatchOnceAsync_DoesNotDeadLetter_WhenDeadLetterAfterMaxIsFalse() + { + var outbox = Substitute.For(); + var publisher = Substitute.For(); + var entry = SampleEntry(attempts: 5); + + outbox.DequeueBatchAsync(100, Arg.Any()) + .Returns([entry]); + + var dispatcher = new AuditOutboxDispatcher( + outbox, + [publisher], + new AuditOutboxDispatcherOptions + { + BatchSize = 100, + MaxAttempts = 5, + DeadLetterAfterMax = false, + }); + + await dispatcher.DispatchOnceAsync(); + + await outbox.DidNotReceive().MarkDeadLetteredAsync( + Arg.Any(), + Arg.Any(), + Arg.Any()); + await publisher.Received(1).PublishAsync(entry, Arg.Any()); + } + + [Fact] + public async Task DispatchOnceAsync_CallsMultiplePublishers() + { + var outbox = Substitute.For(); + var pub1 = Substitute.For(); + var pub2 = Substitute.For(); + var entry = SampleEntry(); + + outbox.DequeueBatchAsync(100, Arg.Any()) + .Returns([entry]); + + var dispatcher = new AuditOutboxDispatcher( + outbox, + [pub1, pub2], + new AuditOutboxDispatcherOptions { BatchSize = 100 }); + + await dispatcher.DispatchOnceAsync(); + + await pub1.Received(1).PublishAsync(entry, Arg.Any()); + await pub2.Received(1).PublishAsync(entry, Arg.Any()); + } + + [Fact] + public async Task DispatchOnceAsync_StopsOnCancellation() + { + var outbox = Substitute.For(); + var publisher = Substitute.For(); + + var cts = new CancellationTokenSource(); + cts.Cancel(); + + var dispatcher = new AuditOutboxDispatcher( + outbox, + [publisher], + new AuditOutboxDispatcherOptions { PollInterval = TimeSpan.FromMilliseconds(10) }); + + await dispatcher.StartAsync(cts.Token); + + // Should complete quickly without throwing + using var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + await dispatcher.StopAsync(timeoutCts.Token); + } + + [Theory] + [InlineData(BackoffStrategy.None, 1, 0)] + [InlineData(BackoffStrategy.Linear, 1, 100)] + [InlineData(BackoffStrategy.Linear, 3, 300)] + [InlineData(BackoffStrategy.Exponential, 1, 200)] + [InlineData(BackoffStrategy.Exponential, 3, 800)] + public async Task DispatchOnceAsync_AppliesBackoffDelay(BackoffStrategy strategy, int attempts, int expectedDelayMs) + { + var outbox = Substitute.For(); + var publisher = Substitute.For(); + var entry = SampleEntry(attempts); + + outbox.DequeueBatchAsync(100, Arg.Any()) + .Returns([entry]); + + var dispatcher = new AuditOutboxDispatcher( + outbox, + [publisher], + new AuditOutboxDispatcherOptions + { + BatchSize = 100, + Backoff = strategy, + }); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + await dispatcher.DispatchOnceAsync(); + sw.Stop(); + + if (expectedDelayMs == 0) + { + sw.ElapsedMilliseconds.Should().BeLessThan(50); + } + else + { + sw.ElapsedMilliseconds.Should().BeGreaterThanOrEqualTo(expectedDelayMs - 50); + } + } + + private static AuditOutboxEntry SampleEntry(int attempts = 0) => new() + { + Id = Guid.NewGuid(), + Record = new AuditRecord + { + DataSubjectId = "subject-1", + Entity = "Customer", + Field = "Email", + Operation = AuditOperation.Update, + }, + Attempts = attempts, + }; +} diff --git a/tests/SensitiveFlow.Audit.Tests/OutboxAuditStoreTests.cs b/tests/SensitiveFlow.Audit.Tests/OutboxAuditStoreTests.cs new file mode 100644 index 0000000..1ca692a --- /dev/null +++ b/tests/SensitiveFlow.Audit.Tests/OutboxAuditStoreTests.cs @@ -0,0 +1,121 @@ +using FluentAssertions; +using NSubstitute; +using SensitiveFlow.Audit.Decorators; +using SensitiveFlow.Core.Enums; +using SensitiveFlow.Core.Interfaces; +using SensitiveFlow.Core.Models; + +namespace SensitiveFlow.Audit.Tests; + +public sealed class OutboxAuditStoreTests +{ + [Fact] + public async Task AppendAsync_WhenInnerSupportsTransaction_ExecutesInTransaction() + { + var inner = Substitute.For(); + var outbox = Substitute.For(); + var sut = new OutboxAuditStore(inner, outbox); + var record = SampleRecord(); + + await sut.AppendAsync(record); + + await ((IAuditStoreTransaction)inner).Received(1).ExecuteInTransactionAsync( + Arg.Any>(), + Arg.Any()); + } + + [Fact] + public async Task AppendAsync_WhenInnerDoesNotSupportTransaction_ExecutesDirectly() + { + var inner = Substitute.For(); + var outbox = Substitute.For(); + var sut = new OutboxAuditStore(inner, outbox); + var record = SampleRecord(); + + await sut.AppendAsync(record); + + await inner.Received(1).AppendAsync(record, Arg.Any()); + await outbox.Received(1).EnqueueAsync(record, Arg.Any()); + } + + [Fact] + public async Task AppendRangeAsync_WhenInnerIsBatchAndSupportsTransaction_ExecutesInTransaction() + { + var inner = Substitute.For(); + var outbox = Substitute.For(); + var sut = new OutboxAuditStore(inner, outbox); + var records = new[] { SampleRecord(), SampleRecord() }; + + await sut.AppendRangeAsync(records); + + await ((IAuditStoreTransaction)inner).Received(1).ExecuteInTransactionAsync( + Arg.Any>(), + Arg.Any()); + } + + [Fact] + public async Task AppendRangeAsync_WhenInnerIsBatchWithoutTransaction_DelegatesBatch() + { + var inner = Substitute.For(); + var outbox = Substitute.For(); + var sut = new OutboxAuditStore(inner, outbox); + var records = new[] { SampleRecord(), SampleRecord() }; + + await sut.AppendRangeAsync(records); + + await inner.Received(1).AppendRangeAsync(records, Arg.Any()); + await outbox.Received(2).EnqueueAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task AppendRangeAsync_WhenInnerIsNotBatchWithoutTransaction_FallsBackToIndividualAppends() + { + var inner = Substitute.For(); + var outbox = Substitute.For(); + var sut = new OutboxAuditStore(inner, outbox); + var records = new[] { SampleRecord(), SampleRecord() }; + + await sut.AppendRangeAsync(records); + + await inner.Received(2).AppendAsync(Arg.Any(), Arg.Any()); + await outbox.Received(2).EnqueueAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task QueryAsync_DelegatesToInner() + { + var inner = Substitute.For(); + var outbox = Substitute.For(); + var sut = new OutboxAuditStore(inner, outbox); + var expected = new[] { SampleRecord() }; + inner.QueryAsync(null, null, 0, 100, Arg.Any()) + .Returns(expected); + + var result = await sut.QueryAsync(); + + result.Should().BeEquivalentTo(expected); + } + + [Fact] + public async Task QueryByDataSubjectAsync_DelegatesToInner() + { + var inner = Substitute.For(); + var outbox = Substitute.For(); + var sut = new OutboxAuditStore(inner, outbox); + var expected = new[] { SampleRecord() }; + inner.QueryByDataSubjectAsync("subject-1", null, null, 0, 100, Arg.Any()) + .Returns(expected); + + var result = await sut.QueryByDataSubjectAsync("subject-1"); + + result.Should().BeEquivalentTo(expected); + } + + private static AuditRecord SampleRecord() => new() + { + DataSubjectId = "subject-1", + Entity = "Customer", + Field = "Email", + Operation = AuditOperation.Update, + }; +} diff --git a/tests/SensitiveFlow.HealthChecks.Tests/AuditOutboxHealthCheckTests.cs b/tests/SensitiveFlow.HealthChecks.Tests/AuditOutboxHealthCheckTests.cs new file mode 100644 index 0000000..9ec7183 --- /dev/null +++ b/tests/SensitiveFlow.HealthChecks.Tests/AuditOutboxHealthCheckTests.cs @@ -0,0 +1,86 @@ +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using NSubstitute; +using SensitiveFlow.Audit.Outbox; +using SensitiveFlow.Core.Interfaces; +using SensitiveFlow.HealthChecks.Checks; +using SensitiveFlow.HealthChecks.Extensions; + +namespace SensitiveFlow.HealthChecks.Tests; + +#pragma warning disable CS0618 // Type or member is obsolete +public sealed class AuditOutboxHealthCheckTests +{ + [Fact] + public async Task CheckHealthAsync_WithDurableOutbox_ReturnsHealthy() + { + var outbox = Substitute.For(); + var check = new AuditOutboxHealthCheck(outbox); + + var result = await check.CheckHealthAsync(new HealthCheckContext()); + + result.Status.Should().Be(HealthStatus.Healthy); + result.Description.Should().Contain("Audit outbox resolved"); + } + + [Fact] + public async Task CheckHealthAsync_WithInMemoryOutbox_InDevelopment_ReturnsHealthy() + { + var outbox = new InMemoryAuditOutbox(); + var env = Substitute.For(); + env.EnvironmentName.Returns("Development"); + + var check = new AuditOutboxHealthCheck(outbox, env); + + var result = await check.CheckHealthAsync(new HealthCheckContext()); + + result.Status.Should().Be(HealthStatus.Healthy); + } + + [Fact] + public async Task CheckHealthAsync_WithInMemoryOutbox_InProduction_ReturnsDegraded() + { + var outbox = new InMemoryAuditOutbox(); + var env = Substitute.For(); + env.EnvironmentName.Returns("Production"); + + var check = new AuditOutboxHealthCheck(outbox, env); + + var result = await check.CheckHealthAsync(new HealthCheckContext()); + + result.Status.Should().Be(HealthStatus.Degraded); + result.Description.Should().Contain("In-memory audit outbox"); + } + + [Fact] + public async Task CheckHealthAsync_WithInMemoryOutbox_NoEnvironment_ReturnsHealthy() + { + var outbox = new InMemoryAuditOutbox(); + + var check = new AuditOutboxHealthCheck(outbox, null); + + var result = await check.CheckHealthAsync(new HealthCheckContext()); + + result.Status.Should().Be(HealthStatus.Healthy); + } + + [Fact] + public void AddAuditOutboxCheck_RegistersCheck() + { + var services = new ServiceCollection(); + services.AddSingleton(new InMemoryAuditOutbox()); + services.AddLogging(); + + services.AddHealthChecks() + .AddAuditOutboxCheck(); + + var provider = services.BuildServiceProvider(); + var options = provider.GetRequiredService>().Value; + + options.Registrations.Should().Contain(r => r.Name == "sensitiveflow-audit-outbox"); + } +} +#pragma warning restore CS0618 // Type or member is obsolete diff --git a/tests/SensitiveFlow.HealthChecks.Tests/SensitiveFlow.HealthChecks.Tests.csproj b/tests/SensitiveFlow.HealthChecks.Tests/SensitiveFlow.HealthChecks.Tests.csproj index 3187b05..9b154a4 100644 --- a/tests/SensitiveFlow.HealthChecks.Tests/SensitiveFlow.HealthChecks.Tests.csproj +++ b/tests/SensitiveFlow.HealthChecks.Tests/SensitiveFlow.HealthChecks.Tests.csproj @@ -7,7 +7,9 @@ + + From e0fb0ab788403120b420bb5cf713258fd29d145d Mon Sep 17 00:00:00 2001 From: Mateus Silva Date: Mon, 11 May 2026 13:31:55 -0300 Subject: [PATCH 09/23] feat: Enhance SensitiveFlow with new features and improvements - Added `sensitiveflow-audit-outbox` health check to documentation. - Updated CLI scan input options to include project/solution files. - Implemented warning for `AddInMemoryAuditOutbox()` usage outside of debug mode. - Improved `EfCoreAuditOutbox` to correctly apply pagination in `GetDeadLetteredAsync`. - Enhanced `SensitiveFlowExceptionSanitizer` to include regex for long digit masking. - Added validation warnings for in-memory audit outbox and durable outbox configurations. - Introduced new tests for data subject export masking and exception sanitization. - Updated `SensitiveFlowToolRunner` to emit warnings for in-memory outbox usage. - Added structured logging tests to ensure sensitive data is handled correctly. --- README.md | 5 +- docs/audit.md | 24 ++- docs/package-reference.md | 19 +- docs/policies-discovery-health.md | 12 +- .../Stores/EfCoreAuditOutbox.cs | 10 +- .../SensitiveFlowExceptionSanitizer.cs | 19 +- .../SensitiveFlowConfigurationValidator.cs | 7 +- .../SensitiveFlowToolRunner.cs | 64 +++++++ .../Export/DataSubjectExporterTests.cs | 65 +++++++ .../EfCoreAuditOutboxTests.cs | 109 ++++++++++++ .../OutputBehaviorAttributeTests.cs | 68 ++++++++ .../CoreContractTests.cs | 65 +++++++ .../SensitiveFlowExceptionSanitizerTests.cs | 49 ++++++ .../SensitiveFlowToolRunnerTests.cs | 122 +++++++++++++ .../SensitiveFlow.Diagnostics.Tests.csproj | 2 + .../SensitiveFlowValidationTests.cs | 120 +++++++++++++ .../RedactingLoggerStructuredTests.cs | 162 ++++++++++++++++++ 17 files changed, 892 insertions(+), 30 deletions(-) create mode 100644 tests/SensitiveFlow.Core.Tests/Attributes/OutputBehaviorAttributeTests.cs create mode 100644 tests/SensitiveFlow.Core.Tests/Exceptions/SensitiveFlowExceptionSanitizerTests.cs diff --git a/README.md b/README.md index eee45c1..02c6917 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,10 @@ Defaults: - redaction marker: `[REDACTED]` - logging: `[Sensitive]` markers are redacted; annotated structured object members are redacted unless policies request `MaskInLogs()` - retention anonymization marker: `[ANONYMIZED]` -- health checks: `sensitiveflow-audit-store`, `sensitiveflow-token-store` +- health checks: `sensitiveflow-audit-store`, `sensitiveflow-token-store`, `sensitiveflow-audit-outbox` +- audit outbox dispatcher: `PollInterval = 1s`, `BatchSize = 100`, `MaxAttempts = 5`, `BackoffStrategy = Exponential`, `DeadLetterAfterMax = true` +- data-subject export: raw annotated values by default; use `[Redaction(Export = ...)]` to mask, redact, or omit specific fields +- CLI: project/solution inputs are built first; `SF-CLI-001` warns when `AddInMemoryAuditOutbox()` is outside `#if DEBUG` Every `SaveChanges` on a field annotated with `[PersonalData]` or `[SensitiveData]` now produces an `AuditRecord` automatically. diff --git a/docs/audit.md b/docs/audit.md index f5cb5ef..1ea7e40 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -50,7 +50,7 @@ public interface IAuditStore DateTimeOffset? from = null, DateTimeOffset? to = null, int skip = 0, - int take = int.MaxValue, + int take = 100, CancellationToken cancellationToken = default); Task> QueryByDataSubjectAsync( @@ -58,7 +58,7 @@ public interface IAuditStore DateTimeOffset? from = null, DateTimeOffset? to = null, int skip = 0, - int take = int.MaxValue, + int take = 100, CancellationToken cancellationToken = default); } ``` @@ -208,6 +208,26 @@ builder.Services.AddAuditStore(); builder.Services.AddAuditOutbox(); ``` +### Audit outbox guarantees and defaults + +| Outbox | Intended use | Delivery guarantee | Survives restart | Notes | +| --- | --- | --- | --- | --- | +| `InMemoryAuditOutbox` | tests/local development | best effort / at-most-once | no | Deprecated for production. Diagnostics emit `SF-CONFIG-013` outside Development, and the audit-outbox health check reports `Degraded`. | +| `SensitiveFlow.Audit.EFCore.Outbox` | production durable delivery | at-least-once | yes | EF Core storage, dispatcher retries, and dead-letter state. | +| custom `IDurableAuditOutbox` | application-specific backend | defined by your implementation | should be yes | Register at least one `IAuditOutboxPublisher`; otherwise diagnostics emit `SF-CONFIG-014`. | + +Dispatcher defaults: + +| Option | Default | +| --- | --- | +| `PollInterval` | `1s` | +| `BatchSize` | `100` | +| `MaxAttempts` | `5` | +| `BackoffStrategy` | `Exponential` | +| `DeadLetterAfterMax` | `true` | + +Custom durable outboxes must implement `MarkDeadLetteredAsync(...)` in addition to enqueue, dequeue, processed, and failed acknowledgement methods. Register at least one `IAuditOutboxPublisher` so the dispatcher has a delivery target. + ## Retrying transient failures Audit appends sit in the hot path of `SaveChanges`. A brief network blip or lock contention against the audit store would otherwise cascade into a `SaveChangesAsync` failure. Wrap your store with the bundled `RetryingAuditStore`: diff --git a/docs/package-reference.md b/docs/package-reference.md index ca0c97d..8b1efd6 100644 --- a/docs/package-reference.md +++ b/docs/package-reference.md @@ -18,7 +18,7 @@ This document summarizes each SensitiveFlow package individually: purpose, prima | `SensitiveFlow.Json` | You need automatic response serialization redaction. | `WithSensitiveDataRedaction()` on `System.Text.Json`. | Does not cover Newtonsoft.Json. | | `SensitiveFlow.Logging` | You need sensitive log value redaction. | `AddSensitiveFlowLogging()` and/or provider wrapper. | Not semantic PII detection; scalar values still need explicit markers or structured metadata. | | `SensitiveFlow.Diagnostics` | You want OpenTelemetry spans/metrics. | `AddSensitiveFlowDiagnostics()` after audit store/decorators. | Decorator order changes what latency is measured. | -| `SensitiveFlow.HealthChecks` | You want ASP.NET Core health checks for SensitiveFlow infrastructure. | `AddSensitiveFlowHealthChecks().AddAuditStoreCheck().AddTokenStoreCheck()`. | Token stores without `IHealthProbe` are resolution-only checks to avoid mutating data. | +| `SensitiveFlow.HealthChecks` | You want ASP.NET Core health checks for SensitiveFlow infrastructure. | `AddSensitiveFlowHealthChecks().AddAuditStoreCheck().AddTokenStoreCheck().AddAuditOutboxCheck()`. | Token stores without `IHealthProbe` are resolution-only checks to avoid mutating data; in-memory audit outbox is `Degraded` outside Development. | | `SensitiveFlow.Retention` | You evaluate retention policies. | `AddRetention()` / `AddRetentionExecutor()` and run a scheduled job. | It will not delete database rows automatically. | | `SensitiveFlow.Analyzers` | You want compile-time guardrails. | Add analyzer package to application projects. | Warnings still require engineering judgment. | | `SensitiveFlow.SourceGenerators` | You want generated sensitive metadata. | Add source generator package. | Keep generator tests aligned with reflection fallback. | @@ -88,7 +88,8 @@ Operational notes: - `RetryingAuditStore` does not swallow exhausted failures. - `BufferedAuditStore` is advanced. It can lose records on process crash before flush. Avoid using it with scoped stores until lifetime semantics are hardened. -- `AddInMemoryAuditOutbox()` is for tests/local development. Production outboxes should implement `IAuditOutbox` durably and use `AddAuditOutbox()`. +- `AddInMemoryAuditOutbox()` is obsolete for production and should only be used in tests/local development. +- Production outboxes should implement `IDurableAuditOutbox` and use `AddAuditOutbox()` plus at least one `IAuditOutboxPublisher`. ## SensitiveFlow.Audit.EFCore @@ -194,13 +195,23 @@ Delivery guarantee: - On exception, entries are marked `IsDeadLettered` or retried based on `MaxAttempts`. - Dead-lettered entries and retry history are queryable for operational dashboards and alerting. +Defaults: + +| Option | Default | +| --- | --- | +| `PollInterval` | `1s` | +| `BatchSize` | `100` | +| `MaxAttempts` | `5` | +| `BackoffStrategy` | `Exponential` | +| `DeadLetterAfterMax` | `true` | + Operational notes: - Audit and outbox data live in the same `AuditDbContext` (configurable persistence strategy). - Dispatcher runs as a `HostedService` — activate after application startup. -- Multiple instances of your application can run simultaneously; polling is coordinated via outbox entry state transitions (no distributed locking required). +- Multiple instances of your application can run simultaneously, but you should validate duplicate-delivery tolerance because outbox delivery is at-least-once. - Monitor the `AuditOutboxEntry` table for growth, dead-lettered entries, and retry counts in operational dashboards. -- SQL Server and PostgreSQL container coverage in `tests/SensitiveFlow.Audit.EFCore.Outbox.Tests`. +- SQLite coverage lives in `tests/SensitiveFlow.Audit.EFCore.Outbox.Tests`; provider-specific container coverage should be added when SQL Server/PostgreSQL migrations are introduced. ## SensitiveFlow.Audit.Snapshots.EFCore diff --git a/docs/policies-discovery-health.md b/docs/policies-discovery-health.md index bfc4b6a..1b504c7 100644 --- a/docs/policies-discovery-health.md +++ b/docs/policies-discovery-health.md @@ -39,8 +39,9 @@ SensitiveFlow defaults are intentionally conservative and documented in `Sensiti | Retention anonymization marker | `[ANONYMIZED]` | | Audit health check name | `sensitiveflow-audit-store` | | Token health check name | `sensitiveflow-token-store` | +| Audit outbox health check name | `sensitiveflow-audit-outbox` | | Logging redactor | redacts marked values to `[REDACTED]` | -| CLI scan input | compiled assembly file or directory containing `.dll` files | +| CLI scan input | compiled assembly, project/solution file, or directory containing a single project/solution or `.dll` files | Precedence for JSON output is: @@ -96,6 +97,8 @@ The tool writes: - `sensitiveflow-report.json` - `sensitiveflow-report.md` +The CLI also scans source files for production footguns. `SF-CLI-001` is emitted when `AddInMemoryAuditOutbox()` appears outside a `#if DEBUG` branch. The warning does not fail the scan, but it should be treated as release-blocking for production apps. + ## Health checks `SensitiveFlow.HealthChecks` integrates with `Microsoft.Extensions.Diagnostics.HealthChecks`. @@ -103,10 +106,11 @@ The tool writes: ```csharp builder.Services.AddSensitiveFlowHealthChecks() .AddAuditStoreCheck() - .AddTokenStoreCheck(); + .AddTokenStoreCheck() + .AddAuditOutboxCheck(); ``` -`IAuditStore` is checked with a read-only `QueryAsync(take: 1)` probe. `ITokenStore` is resolution-only by default because the contract has no read-only ping; stores can implement `IHealthProbe` for a real non-mutating probe. +`IAuditStore` is checked with a read-only `QueryAsync(take: 1)` probe. `ITokenStore` is resolution-only by default because the contract has no read-only ping; stores can implement `IHealthProbe` for a real non-mutating probe. `IAuditOutbox` is healthy when a durable or custom outbox is registered, and reports `Degraded` when `InMemoryAuditOutbox` is used outside Development. ## Startup validation @@ -131,3 +135,5 @@ Example warnings: - `SF-CONFIG-010`: retention annotations found without `RetentionExecutor` or handlers. - `SF-CONFIG-011`: ASP.NET Core services registered but `UseSensitiveFlowAudit()` was not marked. - `SF-CONFIG-012`: middleware observed an authenticated user before it ran, which can indicate it was placed after authentication. +- `SF-CONFIG-013`: in-memory audit outbox registered outside Development. +- `SF-CONFIG-014`: durable audit outbox registered without any `IAuditOutboxPublisher`. diff --git a/src/SensitiveFlow.Audit.EFCore.Outbox/Stores/EfCoreAuditOutbox.cs b/src/SensitiveFlow.Audit.EFCore.Outbox/Stores/EfCoreAuditOutbox.cs index 3255c7a..8bc16cf 100644 --- a/src/SensitiveFlow.Audit.EFCore.Outbox/Stores/EfCoreAuditOutbox.cs +++ b/src/SensitiveFlow.Audit.EFCore.Outbox/Stores/EfCoreAuditOutbox.cs @@ -135,13 +135,15 @@ public async Task> GetDeadLetteredAsync(int skip await using var ctx = await _factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); var deadLettered = await ctx.Set() .Where(e => e.IsDeadLettered) - .OrderByDescending(e => e.EnqueuedAt) - .Skip(skip) - .Take(take) .ToListAsync(cancellationToken) .ConfigureAwait(false); - return deadLettered.Select(ToOutboxEntry).ToList(); + return deadLettered + .OrderByDescending(e => e.EnqueuedAt) + .Skip(skip) + .Take(take) + .Select(ToOutboxEntry) + .ToList(); } private static AuditOutboxEntry ToOutboxEntry(AuditOutboxEntryEntity entity) diff --git a/src/SensitiveFlow.Core/Exceptions/SensitiveFlowExceptionSanitizer.cs b/src/SensitiveFlow.Core/Exceptions/SensitiveFlowExceptionSanitizer.cs index 7f8feac..b31f66d 100644 --- a/src/SensitiveFlow.Core/Exceptions/SensitiveFlowExceptionSanitizer.cs +++ b/src/SensitiveFlow.Core/Exceptions/SensitiveFlowExceptionSanitizer.cs @@ -5,8 +5,14 @@ namespace SensitiveFlow.Core.Exceptions; /// /// Produces safe exception messages by removing common high-risk raw values. /// -public static partial class SensitiveFlowExceptionSanitizer +public static class SensitiveFlowExceptionSanitizer { + private static readonly Regex EmailRegex = + new(@"[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, TimeSpan.FromSeconds(1)); + + private static readonly Regex LongDigitRegex = + new(@"\b\d[\d.\-\/\s]{5,}\d\b", RegexOptions.CultureInvariant, TimeSpan.FromSeconds(1)); + /// Creates a sanitized exception view suitable for logs and diagnostics. public static SensitiveFlowSanitizedException Sanitize(Exception exception) { @@ -28,15 +34,8 @@ public static SensitiveFlowSanitizedException Sanitize(Exception exception) public static string SanitizeMessage(string message) { ArgumentNullException.ThrowIfNull(message); - var sanitized = EmailRegex().Replace(message, "[email]"); - sanitized = LongDigitRegex().Replace(sanitized, "[number]"); + var sanitized = EmailRegex.Replace(message, "[email]"); + sanitized = LongDigitRegex.Replace(sanitized, "[number]"); return sanitized; } - - [GeneratedRegex(@"[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)] - private static partial Regex EmailRegex(); - - [GeneratedRegex(@"\b\d[\d.\-\/\s]{5,}\d\b", RegexOptions.CultureInvariant)] - private static partial Regex LongDigitRegex(); } - diff --git a/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs index 851abc2..8075cc0 100644 --- a/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs +++ b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs @@ -155,13 +155,8 @@ private static void AddOutboxDiagnostics( ICollection diagnostics) { var outbox = provider.GetService(); - if (outbox is null) - { - return; - } - var environment = provider.GetService(); - if (outbox.GetType().FullName == "SensitiveFlow.Audit.Outbox.InMemoryAuditOutbox" + if (outbox?.GetType().FullName == "SensitiveFlow.Audit.Outbox.InMemoryAuditOutbox" && environment?.IsDevelopment() == false) { diagnostics.Add(Warning("SF-CONFIG-013", "In-memory audit outbox is configured outside Development. Use AddEfCoreAuditOutbox() or AddAuditOutbox().")); diff --git a/src/SensitiveFlow.Tool/SensitiveFlowToolRunner.cs b/src/SensitiveFlow.Tool/SensitiveFlowToolRunner.cs index 6c88ba9..293df8a 100644 --- a/src/SensitiveFlow.Tool/SensitiveFlowToolRunner.cs +++ b/src/SensitiveFlow.Tool/SensitiveFlowToolRunner.cs @@ -34,6 +34,8 @@ public static int Run(string[] args, TextWriter output, TextWriter error) : Directory.GetCurrentDirectory(); Directory.CreateDirectory(outputDirectory); + EmitSourceWarnings(inputPath, error); + var buildResult = TryBuildSourceInput(inputPath, output, error); if (buildResult.ExitCode != 0) { @@ -146,6 +148,68 @@ private static BuildSourceResult TryBuildSourceInput(string inputPath, TextWrite return projects.Length == 1 ? projects[0] : null; } + private static void EmitSourceWarnings(string inputPath, TextWriter error) + { + foreach (var sourceFile in ResolveSourceFiles(inputPath)) + { + WarnIfInMemoryOutboxIsNotDebugOnly(sourceFile, error); + } + } + + private static IEnumerable ResolveSourceFiles(string inputPath) + { + var root = File.Exists(inputPath) + ? Path.GetDirectoryName(inputPath) + : inputPath; + + if (root is null || !Directory.Exists(root)) + { + yield break; + } + + foreach (var file in Directory.EnumerateFiles(root, "*.cs", SearchOption.AllDirectories)) + { + if (file.Contains($"{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}", StringComparison.OrdinalIgnoreCase) + || file.Contains($"{Path.DirectorySeparatorChar}obj{Path.DirectorySeparatorChar}", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + yield return file; + } + } + + private static void WarnIfInMemoryOutboxIsNotDebugOnly(string sourceFile, TextWriter error) + { + var conditionalStack = new Stack(); + var lines = File.ReadLines(sourceFile); + var lineNumber = 0; + + foreach (var rawLine in lines) + { + lineNumber++; + var line = rawLine.Trim(); + + if (line.StartsWith("#if", StringComparison.Ordinal)) + { + conditionalStack.Push(line.Contains("DEBUG", StringComparison.Ordinal) && !line.Contains("!DEBUG", StringComparison.Ordinal)); + } + else if (line.StartsWith("#else", StringComparison.Ordinal) && conditionalStack.Count > 0) + { + conditionalStack.Push(!conditionalStack.Pop()); + } + else if (line.StartsWith("#endif", StringComparison.Ordinal) && conditionalStack.Count > 0) + { + conditionalStack.Pop(); + } + + if (!conditionalStack.Contains(true) && rawLine.Contains("AddInMemoryAuditOutbox(", StringComparison.Ordinal)) + { + error.WriteLine($"SF-CLI-001: AddInMemoryAuditOutbox() found outside #if DEBUG: {sourceFile}:{lineNumber}"); + } + } + } + private static IEnumerable ResolveAssemblies(string inputPath) { if (File.Exists(inputPath)) diff --git a/tests/SensitiveFlow.Anonymization.Tests/Export/DataSubjectExporterTests.cs b/tests/SensitiveFlow.Anonymization.Tests/Export/DataSubjectExporterTests.cs index 200c3c3..eb61a29 100644 --- a/tests/SensitiveFlow.Anonymization.Tests/Export/DataSubjectExporterTests.cs +++ b/tests/SensitiveFlow.Anonymization.Tests/Export/DataSubjectExporterTests.cs @@ -63,6 +63,34 @@ public void Export_RespectsContextualExportRedaction() exported.Should().NotContainKey("SecretNote"); } + [Fact] + public void Export_MasksPhoneNameGenericAndEmptyValues() + { + var exported = _exporter.Export(new MaskedExportShape()); + + exported["Phone"].Should().Be("(**) *****-**89"); + exported["Name"].Should().Be("M**** S****"); + exported["Code"].Should().Be("A***"); + exported["Single"].Should().Be("*"); + exported["Empty"].Should().Be(string.Empty); + } + + [Fact] + public void Export_MaskOnNonStringValue_ReturnsNull() + { + var exported = _exporter.Export(new NonStringMaskedShape()); + + exported.Should().ContainKey("Score").WhoseValue.Should().BeNull(); + } + + [Fact] + public void Export_EmailWithoutUsablePrefix_UsesGenericMask() + { + var exported = _exporter.Export(new ShortEmailShape()); + + exported["Email"].Should().Be("a**"); + } + [Fact] public void Export_OnNullEntity_Throws() { @@ -119,4 +147,41 @@ public class ContextualExportCustomer [Redaction(Export = OutputRedactionAction.Omit)] public string SecretNote { get; set; } = "hide me"; } + + public class MaskedExportShape + { + [PersonalData] + [Redaction(Export = OutputRedactionAction.Mask)] + public string Phone { get; set; } = "(11) 99999-8889"; + + [PersonalData] + [Redaction(Export = OutputRedactionAction.Mask)] + public string Name { get; set; } = "Maria Silva"; + + [PersonalData] + [Redaction(Export = OutputRedactionAction.Mask)] + public string Code { get; set; } = "ABCD"; + + [PersonalData] + [Redaction(Export = OutputRedactionAction.Mask)] + public string Single { get; set; } = "Z"; + + [PersonalData] + [Redaction(Export = OutputRedactionAction.Mask)] + public string Empty { get; set; } = string.Empty; + } + + public class NonStringMaskedShape + { + [PersonalData] + [Redaction(Export = OutputRedactionAction.Mask)] + public int Score { get; set; } = 42; + } + + public class ShortEmailShape + { + [PersonalData] + [Redaction(Export = OutputRedactionAction.Mask)] + public string Email { get; set; } = "a@x"; + } } diff --git a/tests/SensitiveFlow.Audit.EFCore.Outbox.Tests/EfCoreAuditOutboxTests.cs b/tests/SensitiveFlow.Audit.EFCore.Outbox.Tests/EfCoreAuditOutboxTests.cs index 006f476..8ee14e6 100644 --- a/tests/SensitiveFlow.Audit.EFCore.Outbox.Tests/EfCoreAuditOutboxTests.cs +++ b/tests/SensitiveFlow.Audit.EFCore.Outbox.Tests/EfCoreAuditOutboxTests.cs @@ -67,6 +67,17 @@ public async Task DequeueBatchAsync_ReturnsPendingEntries() batch.Should().AllSatisfy(e => e.Attempts.Should().Be(0)); } + [Fact] + public async Task DequeueBatchAsync_WithNonPositiveMax_Throws() + { + var outbox = new EfCoreAuditOutbox(_factory!); + + var act = () => outbox.DequeueBatchAsync(0); + + await act.Should().ThrowAsync() + .WithParameterName("max"); + } + [Fact] public async Task DequeueBatchAsync_IgnoresProcessedEntries() { @@ -108,6 +119,29 @@ public async Task MarkProcessedAsync_UpdatesEntry() entry.ProcessedAt.Should().NotBeNull(); } + [Fact] + public async Task MarkProcessedAsync_WithEmptyIds_ReturnsWithoutChangingPendingCount() + { + var outbox = new EfCoreAuditOutbox(_factory!); + await outbox.EnqueueAsync(SampleRecord()); + + await outbox.MarkProcessedAsync([]); + + var count = await outbox.GetPendingCountAsync(); + count.Should().Be(1); + } + + [Fact] + public async Task MarkProcessedAsync_WithNullIds_Throws() + { + var outbox = new EfCoreAuditOutbox(_factory!); + + var act = () => outbox.MarkProcessedAsync(null!); + + await act.Should().ThrowAsync() + .WithParameterName("ids"); + } + [Fact] public async Task MarkFailedAsync_IncrementsAttemptsAndSetsError() { @@ -129,6 +163,26 @@ public async Task MarkFailedAsync_IncrementsAttemptsAndSetsError() entry.LastAttemptAt.Should().NotBeNull(); } + [Fact] + public async Task MarkFailedAsync_WithBlankError_Throws() + { + var outbox = new EfCoreAuditOutbox(_factory!); + + var act = () => outbox.MarkFailedAsync(Guid.NewGuid(), " "); + + await act.Should().ThrowAsync(); + } + + [Fact] + public async Task MarkFailedAsync_ForMissingEntry_DoesNotThrow() + { + var outbox = new EfCoreAuditOutbox(_factory!); + + await outbox.MarkFailedAsync(Guid.NewGuid(), "timeout"); + + (await outbox.GetPendingCountAsync()).Should().Be(0); + } + [Fact] public async Task DequeueBatchAsync_IgnoresDeadLetteredEntries() { @@ -159,6 +213,61 @@ public async Task DequeueBatchAsync_IgnoresDeadLetteredEntries() batch2[0].Id.Should().NotBe(deadLetterId); } + [Fact] + public async Task MarkDeadLetteredAsync_SetsDeadLetterStateAndRemovesFromPending() + { + var outbox = new EfCoreAuditOutbox(_factory!); + await outbox.EnqueueAsync(SampleRecord()); + var entry = (await outbox.DequeueBatchAsync(10)).Single(); + + await outbox.MarkDeadLetteredAsync(entry.Id, "publisher failed too many times"); + + (await outbox.GetPendingCountAsync()).Should().Be(0); + var deadLetters = await outbox.GetDeadLetteredAsync(); + deadLetters.Should().ContainSingle(e => e.Id == entry.Id); + } + + [Fact] + public async Task MarkDeadLetteredAsync_WithBlankReason_Throws() + { + var outbox = new EfCoreAuditOutbox(_factory!); + + var act = () => outbox.MarkDeadLetteredAsync(Guid.NewGuid(), ""); + + await act.Should().ThrowAsync(); + } + + [Fact] + public async Task GetDeadLetteredAsync_AppliesSkipAndTake() + { + var outbox = new EfCoreAuditOutbox(_factory!); + for (var i = 0; i < 3; i++) + { + await outbox.EnqueueAsync(SampleRecord()); + } + + var entries = await outbox.DequeueBatchAsync(10); + foreach (var entry in entries) + { + await outbox.MarkDeadLetteredAsync(entry.Id, "failed"); + } + + var page = await outbox.GetDeadLetteredAsync(skip: 1, take: 1); + + page.Should().ContainSingle(); + } + + [Fact] + public async Task EnqueueAsync_WithNullRecord_Throws() + { + var outbox = new EfCoreAuditOutbox(_factory!); + + var act = () => outbox.EnqueueAsync(null!); + + await act.Should().ThrowAsync() + .WithParameterName("record"); + } + private static AuditRecord SampleRecord() => new() { DataSubjectId = $"subject-{Guid.NewGuid()}", diff --git a/tests/SensitiveFlow.Core.Tests/Attributes/OutputBehaviorAttributeTests.cs b/tests/SensitiveFlow.Core.Tests/Attributes/OutputBehaviorAttributeTests.cs new file mode 100644 index 0000000..e86c6cc --- /dev/null +++ b/tests/SensitiveFlow.Core.Tests/Attributes/OutputBehaviorAttributeTests.cs @@ -0,0 +1,68 @@ +using FluentAssertions; +using SensitiveFlow.Core.Attributes; +using SensitiveFlow.Core.Enums; + +namespace SensitiveFlow.Core.Tests.Attributes; + +public sealed class OutputBehaviorAttributeTests +{ + [Fact] + public void AllowSensitiveLogging_StoresJustificationAndRejectsNull() + { + var attribute = new AllowSensitiveLoggingAttribute("required for operational correlation"); + + attribute.Justification.Should().Be("required for operational correlation"); + var act = () => new AllowSensitiveLoggingAttribute(null!); + act.Should().Throw() + .WithParameterName("justification"); + } + + [Fact] + public void AllowSensitiveReturn_StoresJustificationAndRejectsBlank() + { + var attribute = new AllowSensitiveReturnAttribute("explicit data subject export"); + + attribute.Justification.Should().Be("explicit data subject export"); + var act = () => new AllowSensitiveReturnAttribute(" "); + act.Should().Throw(); + } + + [Fact] + public void RedactMaskAndOmitAttributes_ExposeOutputActions() + { + new RedactAttribute().Action.Should().Be(OutputRedactionAction.Redact); + new OmitAttribute().Action.Should().Be(OutputRedactionAction.Omit); + + var mask = new MaskAttribute(MaskKind.Email); + mask.Action.Should().Be(OutputRedactionAction.Mask); + mask.Kind.Should().Be(MaskKind.Email); + } + + [Fact] + public void RedactionAttribute_ResolvesEveryContextAndUnknownAsNone() + { + var attribute = new RedactionAttribute + { + ApiResponse = OutputRedactionAction.Redact, + Logs = OutputRedactionAction.Mask, + Audit = OutputRedactionAction.Pseudonymize, + Export = OutputRedactionAction.Omit, + }; + + attribute.ForContext(RedactionContext.ApiResponse).Should().Be(OutputRedactionAction.Redact); + attribute.ForContext(RedactionContext.Log).Should().Be(OutputRedactionAction.Mask); + attribute.ForContext(RedactionContext.Audit).Should().Be(OutputRedactionAction.Pseudonymize); + attribute.ForContext(RedactionContext.Export).Should().Be(OutputRedactionAction.Omit); + attribute.ForContext((RedactionContext)999).Should().Be(OutputRedactionAction.None); + } + + [Fact] + public void PersonalAndSensitiveData_DefaultSensitivityIsDocumentedByBehavior() + { + new PersonalDataAttribute().Sensitivity.Should().Be(DataSensitivity.Medium); + new PersonalDataAttribute().Category.Should().Be(DataCategory.Other); + + new SensitiveDataAttribute().Sensitivity.Should().Be(DataSensitivity.High); + new SensitiveDataAttribute().Category.Should().Be(SensitiveDataCategory.Other); + } +} diff --git a/tests/SensitiveFlow.Core.Tests/CoreContractTests.cs b/tests/SensitiveFlow.Core.Tests/CoreContractTests.cs index ecb1f9c..220be32 100644 --- a/tests/SensitiveFlow.Core.Tests/CoreContractTests.cs +++ b/tests/SensitiveFlow.Core.Tests/CoreContractTests.cs @@ -236,6 +236,7 @@ public void SensitiveFlowDefaults_AreDocumented() SensitiveFlowDefaults.AnonymousValue.Should().Be("[ANONYMIZED]"); SensitiveFlowDefaults.AuditStoreHealthCheckName.Should().Be("sensitiveflow-audit-store"); SensitiveFlowDefaults.TokenStoreHealthCheckName.Should().Be("sensitiveflow-token-store"); + SensitiveFlowDefaults.AuditOutboxHealthCheckName.Should().Be("sensitiveflow-audit-outbox"); } [Theory] @@ -251,6 +252,15 @@ public void SensitiveFlowOptions_AllBuiltInProfiles_AddRules(SensitiveFlowProfil options.Policies.Rules.Should().NotBeEmpty(); } + [Fact] + public void SensitiveFlowOptions_UnknownProfile_Throws() + { + var act = () => new SensitiveFlowOptions().UseProfile((SensitiveFlowProfile)999); + + act.Should().Throw() + .WithParameterName("profile"); + } + [Fact] public void SensitiveFlowPolicyConfiguration_Create_AppliesCallback() { @@ -335,6 +345,61 @@ public void CsvDataExportFormatter_PrefixesFormulaValues() csv.Should().Contain("\"'=cmd\""); } + [Fact] + public void CsvDataExportFormatter_HandlesEmptyNullAndQuotedValues() + { + var formatter = new CsvDataExportFormatter(); + + formatter.Format([]).Should().BeEmpty(); + var csv = formatter.Format([ + new Dictionary + { + ["Name"] = "Maria \"M\"", + ["Phone"] = null, + }, + new Dictionary + { + ["Extra"] = "+formula", + }, + ]); + + csv.Should().Contain("\"Maria \"\"M\"\"\""); + csv.Should().Contain("\"Phone\""); + csv.Should().Contain("\"'+formula\""); + } + + [Fact] + public void CsvDataExportFormatter_NullRows_Throws() + { + var formatter = new CsvDataExportFormatter(); + + var act = () => formatter.Format(null!); + + act.Should().Throw() + .WithParameterName("rows"); + } + + [Fact] + public void AuditOutboxEntry_DefaultsAreStable() + { + var before = DateTimeOffset.UtcNow.AddSeconds(-1); + var record = new AuditRecord + { + DataSubjectId = "subject-1", + Entity = "Customer", + Field = "Email", + }; + + var entry = new AuditOutboxEntry { Record = record }; + + entry.Id.Should().NotBeEmpty(); + entry.Record.Should().Be(record); + entry.Attempts.Should().Be(0); + entry.EnqueuedAt.Should().BeOnOrAfter(before); + entry.LastAttemptAt.Should().BeNull(); + entry.LastError.Should().BeNull(); + } + private sealed class DiscoveryCustomer { [PersonalData(Category = DataCategory.Contact)] diff --git a/tests/SensitiveFlow.Core.Tests/Exceptions/SensitiveFlowExceptionSanitizerTests.cs b/tests/SensitiveFlow.Core.Tests/Exceptions/SensitiveFlowExceptionSanitizerTests.cs new file mode 100644 index 0000000..bc544fa --- /dev/null +++ b/tests/SensitiveFlow.Core.Tests/Exceptions/SensitiveFlowExceptionSanitizerTests.cs @@ -0,0 +1,49 @@ +using FluentAssertions; +using SensitiveFlow.Core.Exceptions; + +namespace SensitiveFlow.Core.Tests.Exceptions; + +public sealed class SensitiveFlowExceptionSanitizerTests +{ + [Fact] + public void Sanitize_ForSensitiveFlowException_PreservesTypeAndCodeButSanitizesMessage() + { + var exception = new SensitiveFlowConfigurationException( + "Failed for maria@example.com and tax id 123.456.789-00.", + "SF_TEST_001"); + + var sanitized = SensitiveFlowExceptionSanitizer.Sanitize(exception); + + sanitized.Type.Should().Be(nameof(SensitiveFlowConfigurationException)); + sanitized.Code.Should().Be("SF_TEST_001"); + sanitized.Message.Should().Be("Failed for [email] and tax id [number]."); + } + + [Fact] + public void Sanitize_ForRegularException_UsesNullCode() + { + var sanitized = SensitiveFlowExceptionSanitizer.Sanitize(new InvalidOperationException("email joao@example.com")); + + sanitized.Type.Should().Be(nameof(InvalidOperationException)); + sanitized.Code.Should().BeNull(); + sanitized.Message.Should().Be("email [email]"); + } + + [Fact] + public void Sanitize_WithNullException_Throws() + { + var act = () => SensitiveFlowExceptionSanitizer.Sanitize(null!); + + act.Should().Throw() + .WithParameterName("exception"); + } + + [Fact] + public void SanitizeMessage_WithNullMessage_Throws() + { + var act = () => SensitiveFlowExceptionSanitizer.SanitizeMessage(null!); + + act.Should().Throw() + .WithParameterName("message"); + } +} diff --git a/tests/SensitiveFlow.Core.Tests/SensitiveFlowToolRunnerTests.cs b/tests/SensitiveFlow.Core.Tests/SensitiveFlowToolRunnerTests.cs index a8f46a9..aeb8a58 100644 --- a/tests/SensitiveFlow.Core.Tests/SensitiveFlowToolRunnerTests.cs +++ b/tests/SensitiveFlow.Core.Tests/SensitiveFlowToolRunnerTests.cs @@ -72,5 +72,127 @@ public void Run_ScanInvalidProject_ReturnsBuildError() exitCode.Should().Be(5); error.ToString().Should().Contain("dotnet build failed"); } + + [Fact] + public void Run_ScanDirectoryWithSingleInvalidProject_BuildsThatProject() + { + var root = Path.Combine(Path.GetTempPath(), "sensitiveflow-tool-project-dir-tests", Guid.NewGuid().ToString("N")); + var outputDirectory = Path.Combine(root, "report"); + Directory.CreateDirectory(root); + File.WriteAllText(Path.Combine(root, "AnnotatedProject.csproj"), """ + + + not-a-real-tfm + + + """); + using var output = new StringWriter(); + using var error = new StringWriter(); + + var exitCode = SensitiveFlowToolRunner.Run(["scan", root, outputDirectory], output, error); + + exitCode.Should().Be(5); + output.ToString().Should().Contain("Building source project before scan"); + error.ToString().Should().Contain("dotnet build failed"); + } + + [Fact] + public void Run_ScanDirectoryWithInvalidSolution_BuildsThatSolution() + { + var root = Path.Combine(Path.GetTempPath(), "sensitiveflow-tool-sln-dir-tests", Guid.NewGuid().ToString("N")); + var outputDirectory = Path.Combine(root, "report"); + Directory.CreateDirectory(root); + File.WriteAllText(Path.Combine(root, "Broken.slnx"), ""); + using var output = new StringWriter(); + using var error = new StringWriter(); + + var exitCode = SensitiveFlowToolRunner.Run(["scan", root, outputDirectory], output, error); + + exitCode.Should().Be(5); + output.ToString().Should().Contain("Broken.slnx"); + error.ToString().Should().Contain("dotnet build failed"); + } + + [Fact] + public void Run_ScanMissingInput_ReturnsNotFoundError() + { + using var output = new StringWriter(); + using var error = new StringWriter(); + var missingPath = Path.Combine(Path.GetTempPath(), "sensitiveflow-missing", Guid.NewGuid().ToString("N")); + + var exitCode = SensitiveFlowToolRunner.Run(["scan", missingPath], output, error); + + exitCode.Should().Be(3); + error.ToString().Should().Contain("not found"); + } + + [Fact] + public void Run_ScanDirectoryWithoutAssemblies_ReturnsNoAssembliesError() + { + var directory = Path.Combine(Path.GetTempPath(), "sensitiveflow-tool-empty", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(directory); + using var output = new StringWriter(); + using var error = new StringWriter(); + + var exitCode = SensitiveFlowToolRunner.Run(["scan", directory], output, error); + + exitCode.Should().Be(4); + error.ToString().Should().Contain("No assemblies found"); + } + + [Fact] + public void Run_ScanSourceInput_WarnsWhenInMemoryOutboxIsNotDebugOnly() + { + var root = Path.Combine(Path.GetTempPath(), "sensitiveflow-tool-source-warning", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + File.WriteAllText(Path.Combine(root, "Program.cs"), "services.AddInMemoryAuditOutbox();"); + using var output = new StringWriter(); + using var error = new StringWriter(); + + var exitCode = SensitiveFlowToolRunner.Run(["scan", root], output, error); + + exitCode.Should().Be(4); + error.ToString().Should().Contain("SF-CLI-001"); + } + + [Fact] + public void Run_ScanSourceInput_DoesNotWarnWhenInMemoryOutboxIsDebugOnly() + { + var root = Path.Combine(Path.GetTempPath(), "sensitiveflow-tool-source-debug", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + File.WriteAllText(Path.Combine(root, "Program.cs"), """ + #if DEBUG + services.AddInMemoryAuditOutbox(); + #endif + """); + using var output = new StringWriter(); + using var error = new StringWriter(); + + var exitCode = SensitiveFlowToolRunner.Run(["scan", root], output, error); + + exitCode.Should().Be(4); + error.ToString().Should().NotContain("SF-CLI-001"); + } + + [Fact] + public void Run_ScanSourceInput_WarnsWhenInMemoryOutboxIsInElseBranch() + { + var root = Path.Combine(Path.GetTempPath(), "sensitiveflow-tool-source-else", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + File.WriteAllText(Path.Combine(root, "Program.cs"), """ + #if DEBUG + services.AddEfCoreAuditOutbox(); + #else + services.AddInMemoryAuditOutbox(); + #endif + """); + using var output = new StringWriter(); + using var error = new StringWriter(); + + var exitCode = SensitiveFlowToolRunner.Run(["scan", root], output, error); + + exitCode.Should().Be(4); + error.ToString().Should().Contain("SF-CLI-001"); + } } #endif diff --git a/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlow.Diagnostics.Tests.csproj b/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlow.Diagnostics.Tests.csproj index e34a20f..a283134 100644 --- a/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlow.Diagnostics.Tests.csproj +++ b/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlow.Diagnostics.Tests.csproj @@ -8,9 +8,11 @@ + + diff --git a/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs b/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs index e20f083..ba08ecd 100644 --- a/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs +++ b/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs @@ -1,7 +1,10 @@ using FluentAssertions; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.FileProviders; +using Microsoft.Extensions.Hosting; using SensitiveFlow.AspNetCore.Diagnostics; using SensitiveFlow.AspNetCore.Extensions; +using SensitiveFlow.Audit.Outbox; using SensitiveFlow.Core.Enums; using SensitiveFlow.Core.Attributes; using SensitiveFlow.Core.Interfaces; @@ -10,6 +13,7 @@ using SensitiveFlow.Diagnostics.Extensions; using SensitiveFlow.Diagnostics.Validation; using SensitiveFlow.EFCore.Extensions; +using SensitiveFlow.Json.Configuration; namespace SensitiveFlow.Diagnostics.Tests; @@ -49,6 +53,37 @@ public void ValidateSensitiveFlow_WhenTokenStoreRequiredAndMissing_ReportsWarnin report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-002"); } + [Fact] + public void ValidateSensitiveFlow_WhenJsonRedactionRequiredAndOptionsMissing_ReportsWarning() + { + _ = typeof(JsonRedactionOptions).Assembly; + var services = new ServiceCollection(); + services.AddSensitiveFlowValidation(o => + { + o.RequireAuditStore = false; + o.RequireJsonRedaction = true; + }); + + var report = services.BuildServiceProvider().ValidateSensitiveFlow(); + + report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-004"); + } + + [Fact] + public void ValidateSensitiveFlow_WhenRetentionRequiredAndServicesMissing_ReportsWarning() + { + var services = new ServiceCollection(); + services.AddSensitiveFlowValidation(o => + { + o.RequireAuditStore = false; + o.RequireRetention = true; + }); + + var report = services.BuildServiceProvider().ValidateSensitiveFlow(); + + report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-005"); + } + [Fact] public void ValidateSensitiveFlow_WhenPseudonymizerWithoutTokenStore_ReportsWarning() { @@ -163,6 +198,61 @@ public void ValidateSensitiveFlow_WhenAuditMiddlewareObservedAuthenticatedUser_R report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-012"); } + [Fact] + public void ValidateSensitiveFlow_WhenInMemoryOutboxRegisteredOutsideDevelopment_ReportsWarning() + { + var services = new ServiceCollection(); +#pragma warning disable CS0618 + services.AddSingleton(); +#pragma warning restore CS0618 + services.AddSingleton(new TestHostEnvironment("Production")); + services.AddSensitiveFlowValidation(o => o.RequireAuditStore = false); + + var report = services.BuildServiceProvider().ValidateSensitiveFlow(); + + report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-013"); + } + + [Fact] + public void ValidateSensitiveFlow_WhenInMemoryOutboxRegisteredInDevelopment_DoesNotReportProductionWarning() + { + var services = new ServiceCollection(); +#pragma warning disable CS0618 + services.AddSingleton(); +#pragma warning restore CS0618 + services.AddSingleton(new TestHostEnvironment(Environments.Development)); + services.AddSensitiveFlowValidation(o => o.RequireAuditStore = false); + + var report = services.BuildServiceProvider().ValidateSensitiveFlow(); + + report.Diagnostics.Should().NotContain(d => d.Code == "SF-CONFIG-013"); + } + + [Fact] + public void ValidateSensitiveFlow_WhenDurableOutboxHasNoPublisher_ReportsWarning() + { + var services = new ServiceCollection(); + services.AddSingleton(); + services.AddSensitiveFlowValidation(o => o.RequireAuditStore = false); + + var report = services.BuildServiceProvider().ValidateSensitiveFlow(); + + report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-014"); + } + + [Fact] + public void ValidateSensitiveFlow_WhenDurableOutboxHasPublisher_DoesNotReportPublisherWarning() + { + var services = new ServiceCollection(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSensitiveFlowValidation(o => o.RequireAuditStore = false); + + var report = services.BuildServiceProvider().ValidateSensitiveFlow(); + + report.Diagnostics.Should().NotContain(d => d.Code == "SF-CONFIG-014"); + } + [Fact] public void AddSensitiveFlow_RegistersSharedOptions() { @@ -216,4 +306,34 @@ private sealed class RetentionAnnotatedShape [RetentionData(Years = 1)] public string Email { get; set; } = string.Empty; } + + private sealed class FakeDurableOutbox : IDurableAuditOutbox + { + public Task EnqueueAsync(AuditRecord record, CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task> DequeueBatchAsync(int max, CancellationToken cancellationToken = default) + => Task.FromResult>([]); + + public Task MarkProcessedAsync(IEnumerable ids, CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task MarkFailedAsync(Guid id, string error, CancellationToken cancellationToken = default) => Task.CompletedTask; + + public Task MarkDeadLetteredAsync(Guid id, string reason, CancellationToken cancellationToken = default) => Task.CompletedTask; + } + + private sealed class FakePublisher : IAuditOutboxPublisher + { + public Task PublishAsync(AuditOutboxEntry entry, CancellationToken cancellationToken = default) => Task.CompletedTask; + } + + private sealed class TestHostEnvironment(string environmentName) : IHostEnvironment + { + public string EnvironmentName { get; set; } = environmentName; + + public string ApplicationName { get; set; } = "SensitiveFlow.Tests"; + + public string ContentRootPath { get; set; } = Directory.GetCurrentDirectory(); + + public IFileProvider ContentRootFileProvider { get; set; } = new NullFileProvider(); + } } diff --git a/tests/SensitiveFlow.Logging.Tests/RedactingLoggerStructuredTests.cs b/tests/SensitiveFlow.Logging.Tests/RedactingLoggerStructuredTests.cs index fc2acf3..8900a20 100644 --- a/tests/SensitiveFlow.Logging.Tests/RedactingLoggerStructuredTests.cs +++ b/tests/SensitiveFlow.Logging.Tests/RedactingLoggerStructuredTests.cs @@ -246,6 +246,97 @@ public void Log_StructuredObject_ContextualLogActionCanOmit() projected["Email"].Should().Be("m****@example.com"); } + [Fact] + public void Log_StructuredObject_AllowSensitiveLoggingPreservesValue() + { + var spy = new StateSpyLogger(); + var logger = new RedactingLogger(spy, new DefaultSensitiveValueRedactor()); + + logger.Log(LogLevel.Information, new EventId(0), + new List> + { + new("Customer", new AllowedLogShape()), + }, + null, + (s, _) => string.Empty); + + var projected = spy.LastState.Single(kv => kv.Key == "Customer").Value + .Should().BeAssignableTo>().Subject; + projected["Email"].Should().Be("maria@example.com"); + projected["TaxId"].Should().Be("[REDACTED]"); + } + + [Fact] + public void Log_StructuredObject_ExplicitAttributesCoverOmitRedactAndMaskKinds() + { + var spy = new StateSpyLogger(); + var logger = new RedactingLogger(spy, new DefaultSensitiveValueRedactor()); + + logger.Log(LogLevel.Information, new EventId(0), + new List> + { + new("Shape", new AttributeLogShape()), + }, + null, + (s, _) => string.Empty); + + var projected = spy.LastState.Single(kv => kv.Key == "Shape").Value + .Should().BeAssignableTo>().Subject; + projected.Should().NotContainKey("Omitted"); + projected["Redacted"].Should().Be("[REDACTED]"); + projected["Phone"].Should().Be("(**) *****-**89"); + projected["Name"].Should().Be("M**** S****"); + projected["Code"].Should().Be("A***"); + projected["Single"].Should().Be("*"); + projected["Number"].Should().Be("[REDACTED]"); + } + + [Fact] + public void Log_StructuredObject_ContextualMaskInfersPhoneNameGenericAndShortEmail() + { + var spy = new StateSpyLogger(); + var logger = new RedactingLogger(spy, new DefaultSensitiveValueRedactor()); + + logger.Log(LogLevel.Information, new EventId(0), + new List> + { + new("Shape", new InferredMaskLogShape()), + }, + null, + (s, _) => string.Empty); + + var projected = spy.LastState.Single(kv => kv.Key == "Shape").Value + .Should().BeAssignableTo>().Subject; + projected["PhoneNumber"].Should().Be("(**) *****-**89"); + projected["FullName"].Should().Be("M**** S****"); + projected["SecretCode"].Should().Be("A***"); + projected["Email"].Should().Be("a**"); + projected["Empty"].Should().Be(string.Empty); + } + + [Fact] + public void Log_StructuredObject_IgnoresNullStringValueTypeAndUnannotatedObjects() + { + var spy = new StateSpyLogger(); + var logger = new RedactingLogger(spy, new DefaultSensitiveValueRedactor()); + + logger.Log(LogLevel.Information, new EventId(0), + new List> + { + new("Null", null), + new("String", "raw"), + new("Number", 123), + new("Object", new UnannotatedLogShape()), + }, + null, + (s, _) => string.Empty); + + spy.LastState.Single(kv => kv.Key == "Null").Value.Should().BeNull(); + spy.LastState.Should().Contain(kv => kv.Key == "String" && (string?)kv.Value == "raw"); + spy.LastState.Should().Contain(kv => kv.Key == "Number" && (int?)kv.Value == 123); + spy.LastState.Single(kv => kv.Key == "Object").Value.Should().BeOfType(); + } + private sealed class SpyLogger : ILogger { public string? LastMessage { get; private set; } @@ -305,4 +396,75 @@ private sealed class ContextualLogShape [Redaction(Logs = OutputRedactionAction.Omit)] public string TaxId { get; set; } = "12345678900"; } + + private sealed class AllowedLogShape + { + [PersonalData(Category = DataCategory.Contact)] + [AllowSensitiveLogging("Diagnostic correlation value in restricted logs.")] + public string Email { get; set; } = "maria@example.com"; + + [SensitiveData(Category = SensitiveDataCategory.Other)] + public string TaxId { get; set; } = "12345678900"; + } + + private sealed class AttributeLogShape + { + [PersonalData] + [Omit] + public string Omitted { get; set; } = "hide"; + + [PersonalData] + [Redact] + public string Redacted { get; set; } = "secret"; + + [PersonalData] + [Mask(MaskKind.Phone)] + public string Phone { get; set; } = "(11) 99999-8889"; + + [PersonalData] + [Mask(MaskKind.Name)] + public string Name { get; set; } = "Maria Silva"; + + [PersonalData] + [Mask] + public string Code { get; set; } = "ABCD"; + + [PersonalData] + [Mask] + public string Single { get; set; } = "Z"; + + [PersonalData] + [Mask] + public int Number { get; set; } = 42; + + public string Public { get; set; } = "visible"; + } + + private sealed class InferredMaskLogShape + { + [PersonalData] + [Redaction(Logs = OutputRedactionAction.Mask)] + public string PhoneNumber { get; set; } = "(11) 99999-8889"; + + [PersonalData] + [Redaction(Logs = OutputRedactionAction.Mask)] + public string FullName { get; set; } = "Maria Silva"; + + [PersonalData] + [Redaction(Logs = OutputRedactionAction.Mask)] + public string SecretCode { get; set; } = "ABCD"; + + [PersonalData] + [Redaction(Logs = OutputRedactionAction.Mask)] + public string Email { get; set; } = "a@x"; + + [PersonalData] + [Redaction(Logs = OutputRedactionAction.Mask)] + public string Empty { get; set; } = string.Empty; + } + + private sealed class UnannotatedLogShape + { + public string Value { get; set; } = "plain"; + } } From 5c5914edb2b5fd0f0f680fe6257af5447d6e3bf5 Mon Sep 17 00:00:00 2001 From: Mateus Silva Date: Mon, 11 May 2026 14:36:18 -0300 Subject: [PATCH 10/23] feat: Refactor SensitiveFlowToolRunner to use ISensitiveFlowToolHost for improved testability and abstraction --- .../Properties/AssemblyInfo.cs | 3 + .../SensitiveFlowToolRunner.cs | 187 ++++++++++++------ .../SensitiveFlowToolRunnerTests.cs | 123 ++++++++++++ 3 files changed, 256 insertions(+), 57 deletions(-) create mode 100644 src/SensitiveFlow.Tool/Properties/AssemblyInfo.cs diff --git a/src/SensitiveFlow.Tool/Properties/AssemblyInfo.cs b/src/SensitiveFlow.Tool/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..70a8c66 --- /dev/null +++ b/src/SensitiveFlow.Tool/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("SensitiveFlow.Core.Tests")] diff --git a/src/SensitiveFlow.Tool/SensitiveFlowToolRunner.cs b/src/SensitiveFlow.Tool/SensitiveFlowToolRunner.cs index 293df8a..4eb6cd5 100644 --- a/src/SensitiveFlow.Tool/SensitiveFlowToolRunner.cs +++ b/src/SensitiveFlow.Tool/SensitiveFlowToolRunner.cs @@ -11,10 +11,14 @@ public static class SensitiveFlowToolRunner { /// Runs the CLI command. public static int Run(string[] args, TextWriter output, TextWriter error) + => Run(args, output, error, SensitiveFlowToolHost.Instance); + + internal static int Run(string[] args, TextWriter output, TextWriter error, ISensitiveFlowToolHost host) { ArgumentNullException.ThrowIfNull(args); ArgumentNullException.ThrowIfNull(output); ArgumentNullException.ThrowIfNull(error); + ArgumentNullException.ThrowIfNull(host); if (args.Length < 2 || !string.Equals(args[0], "scan", StringComparison.OrdinalIgnoreCase)) { @@ -23,7 +27,7 @@ public static int Run(string[] args, TextWriter output, TextWriter error) } var inputPath = Path.GetFullPath(args[1]); - if (!File.Exists(inputPath) && !Directory.Exists(inputPath)) + if (!host.FileExists(inputPath) && !host.DirectoryExists(inputPath)) { error.WriteLine($"Assembly, project, solution, or directory not found: {inputPath}"); return 3; @@ -31,19 +35,19 @@ public static int Run(string[] args, TextWriter output, TextWriter error) var outputDirectory = args.Length >= 3 ? Path.GetFullPath(args[2]) - : Directory.GetCurrentDirectory(); - Directory.CreateDirectory(outputDirectory); + : host.CurrentDirectory; + host.CreateDirectory(outputDirectory); - EmitSourceWarnings(inputPath, error); + EmitSourceWarnings(inputPath, error, host); - var buildResult = TryBuildSourceInput(inputPath, output, error); + var buildResult = TryBuildSourceInput(inputPath, output, error, host); if (buildResult.ExitCode != 0) { return buildResult.ExitCode; } var scanPath = buildResult.ScanPath ?? inputPath; - var assemblies = ResolveAssemblies(scanPath).ToArray(); + var assemblies = ResolveAssemblies(scanPath, host).ToArray(); if (assemblies.Length == 0) { error.WriteLine($"No assemblies found to scan: {scanPath}"); @@ -55,73 +59,54 @@ public static int Run(string[] args, TextWriter output, TextWriter error) var jsonPath = Path.Combine(outputDirectory, "sensitiveflow-report.json"); var markdownPath = Path.Combine(outputDirectory, "sensitiveflow-report.md"); - File.WriteAllText(jsonPath, report.ToJson()); - File.WriteAllText(markdownPath, report.ToMarkdown()); + host.WriteAllText(jsonPath, report.ToJson()); + host.WriteAllText(markdownPath, report.ToMarkdown()); output.WriteLine($"SensitiveFlow report written: {jsonPath}"); output.WriteLine($"SensitiveFlow report written: {markdownPath}"); return 0; } - private static BuildSourceResult TryBuildSourceInput(string inputPath, TextWriter output, TextWriter error) + private static BuildSourceResult TryBuildSourceInput(string inputPath, TextWriter output, TextWriter error, ISensitiveFlowToolHost host) { - var buildTarget = ResolveBuildTarget(inputPath); + var buildTarget = ResolveBuildTarget(inputPath, host); if (buildTarget is null) { return new BuildSourceResult(0, null); } output.WriteLine($"Building source project before scan: {buildTarget}"); - var psi = new ProcessStartInfo("dotnet", $"build \"{buildTarget}\" -c Release --nologo") - { - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - }; - - using var process = Process.Start(psi); - if (process is null) + var result = host.Build(buildTarget, TimeSpan.FromMinutes(2)); + if (!result.Started) { error.WriteLine("Failed to start dotnet build."); return new BuildSourceResult(5, null); } - var stdoutTask = process.StandardOutput.ReadToEndAsync(); - var stderrTask = process.StandardError.ReadToEndAsync(); - if (!process.WaitForExit(TimeSpan.FromMinutes(2))) + if (result.TimedOut) { - try - { - process.Kill(entireProcessTree: true); - } - catch (InvalidOperationException) - { - } - error.WriteLine("dotnet build timed out."); return new BuildSourceResult(5, null); } - var stdout = stdoutTask.GetAwaiter().GetResult(); - var stderr = stderrTask.GetAwaiter().GetResult(); - if (process.ExitCode != 0) + if (result.ExitCode != 0) { - error.WriteLine(stdout); - error.WriteLine(stderr); - error.WriteLine($"dotnet build failed with exit code {process.ExitCode}."); + error.WriteLine(result.StandardOutput); + error.WriteLine(result.StandardError); + error.WriteLine($"dotnet build failed with exit code {result.ExitCode}."); return new BuildSourceResult(5, null); } - var scanPath = Directory.Exists(inputPath) + var scanPath = host.DirectoryExists(inputPath) ? inputPath : Path.GetDirectoryName(buildTarget); return new BuildSourceResult(0, scanPath); } - private static string? ResolveBuildTarget(string inputPath) + private static string? ResolveBuildTarget(string inputPath, ISensitiveFlowToolHost host) { - if (File.Exists(inputPath)) + if (host.FileExists(inputPath)) { var extension = Path.GetExtension(inputPath); return extension.Equals(".csproj", StringComparison.OrdinalIgnoreCase) @@ -131,43 +116,43 @@ private static BuildSourceResult TryBuildSourceInput(string inputPath, TextWrite : null; } - if (!Directory.Exists(inputPath)) + if (!host.DirectoryExists(inputPath)) { return null; } - var solution = Directory.EnumerateFiles(inputPath, "*.slnx", SearchOption.TopDirectoryOnly) - .Concat(Directory.EnumerateFiles(inputPath, "*.sln", SearchOption.TopDirectoryOnly)) + var solution = host.EnumerateFiles(inputPath, "*.slnx", SearchOption.TopDirectoryOnly) + .Concat(host.EnumerateFiles(inputPath, "*.sln", SearchOption.TopDirectoryOnly)) .FirstOrDefault(); if (solution is not null) { return solution; } - var projects = Directory.EnumerateFiles(inputPath, "*.csproj", SearchOption.TopDirectoryOnly).ToArray(); + var projects = host.EnumerateFiles(inputPath, "*.csproj", SearchOption.TopDirectoryOnly).ToArray(); return projects.Length == 1 ? projects[0] : null; } - private static void EmitSourceWarnings(string inputPath, TextWriter error) + private static void EmitSourceWarnings(string inputPath, TextWriter error, ISensitiveFlowToolHost host) { - foreach (var sourceFile in ResolveSourceFiles(inputPath)) + foreach (var sourceFile in ResolveSourceFiles(inputPath, host)) { - WarnIfInMemoryOutboxIsNotDebugOnly(sourceFile, error); + WarnIfInMemoryOutboxIsNotDebugOnly(sourceFile, error, host); } } - private static IEnumerable ResolveSourceFiles(string inputPath) + private static IEnumerable ResolveSourceFiles(string inputPath, ISensitiveFlowToolHost host) { - var root = File.Exists(inputPath) + var root = host.FileExists(inputPath) ? Path.GetDirectoryName(inputPath) : inputPath; - if (root is null || !Directory.Exists(root)) + if (root is null || !host.DirectoryExists(root)) { yield break; } - foreach (var file in Directory.EnumerateFiles(root, "*.cs", SearchOption.AllDirectories)) + foreach (var file in host.EnumerateFiles(root, "*.cs", SearchOption.AllDirectories)) { if (file.Contains($"{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}", StringComparison.OrdinalIgnoreCase) || file.Contains($"{Path.DirectorySeparatorChar}obj{Path.DirectorySeparatorChar}", StringComparison.OrdinalIgnoreCase)) @@ -179,10 +164,10 @@ private static IEnumerable ResolveSourceFiles(string inputPath) } } - private static void WarnIfInMemoryOutboxIsNotDebugOnly(string sourceFile, TextWriter error) + private static void WarnIfInMemoryOutboxIsNotDebugOnly(string sourceFile, TextWriter error, ISensitiveFlowToolHost host) { var conditionalStack = new Stack(); - var lines = File.ReadLines(sourceFile); + var lines = host.ReadLines(sourceFile); var lineNumber = 0; foreach (var rawLine in lines) @@ -210,21 +195,21 @@ private static void WarnIfInMemoryOutboxIsNotDebugOnly(string sourceFile, TextWr } } - private static IEnumerable ResolveAssemblies(string inputPath) + private static IEnumerable ResolveAssemblies(string inputPath, ISensitiveFlowToolHost host) { - if (File.Exists(inputPath)) + if (host.FileExists(inputPath)) { - yield return Assembly.LoadFrom(inputPath); + yield return host.LoadAssembly(inputPath); yield break; } - foreach (var file in Directory.EnumerateFiles(inputPath, "*.dll", SearchOption.AllDirectories) + foreach (var file in host.EnumerateFiles(inputPath, "*.dll", SearchOption.AllDirectories) .Where(static p => !p.Contains($"{Path.DirectorySeparatorChar}obj{Path.DirectorySeparatorChar}", StringComparison.OrdinalIgnoreCase))) { Assembly assembly; try { - assembly = Assembly.LoadFrom(file); + assembly = host.LoadAssembly(file); } catch (BadImageFormatException) { @@ -237,3 +222,91 @@ private static IEnumerable ResolveAssemblies(string inputPath) private sealed record BuildSourceResult(int ExitCode, string? ScanPath); } + +internal interface ISensitiveFlowToolHost +{ + string CurrentDirectory { get; } + + bool FileExists(string path); + + bool DirectoryExists(string path); + + void CreateDirectory(string path); + + IEnumerable EnumerateFiles(string path, string searchPattern, SearchOption searchOption); + + IEnumerable ReadLines(string path); + + void WriteAllText(string path, string contents); + + Assembly LoadAssembly(string path); + + SensitiveFlowToolBuildResult Build(string buildTarget, TimeSpan timeout); +} + +internal sealed record SensitiveFlowToolBuildResult( + bool Started, + bool TimedOut, + int ExitCode, + string StandardOutput, + string StandardError); + +internal sealed class SensitiveFlowToolHost : ISensitiveFlowToolHost +{ + public static SensitiveFlowToolHost Instance { get; } = new(); + + public string CurrentDirectory => Directory.GetCurrentDirectory(); + + public bool FileExists(string path) => File.Exists(path); + + public bool DirectoryExists(string path) => Directory.Exists(path); + + public void CreateDirectory(string path) => Directory.CreateDirectory(path); + + public IEnumerable EnumerateFiles(string path, string searchPattern, SearchOption searchOption) + => Directory.EnumerateFiles(path, searchPattern, searchOption); + + public IEnumerable ReadLines(string path) => File.ReadLines(path); + + public void WriteAllText(string path, string contents) => File.WriteAllText(path, contents); + + public Assembly LoadAssembly(string path) => Assembly.LoadFrom(path); + + public SensitiveFlowToolBuildResult Build(string buildTarget, TimeSpan timeout) + { + var psi = new ProcessStartInfo("dotnet", $"build \"{buildTarget}\" -c Release --nologo") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }; + + using var process = Process.Start(psi); + if (process is null) + { + return new SensitiveFlowToolBuildResult(false, false, 5, string.Empty, string.Empty); + } + + var stdoutTask = process.StandardOutput.ReadToEndAsync(); + var stderrTask = process.StandardError.ReadToEndAsync(); + if (!process.WaitForExit(timeout)) + { + try + { + process.Kill(entireProcessTree: true); + } + catch (InvalidOperationException) + { + } + + return new SensitiveFlowToolBuildResult(true, true, 5, string.Empty, string.Empty); + } + + return new SensitiveFlowToolBuildResult( + true, + false, + process.ExitCode, + stdoutTask.GetAwaiter().GetResult(), + stderrTask.GetAwaiter().GetResult()); + } +} diff --git a/tests/SensitiveFlow.Core.Tests/SensitiveFlowToolRunnerTests.cs b/tests/SensitiveFlow.Core.Tests/SensitiveFlowToolRunnerTests.cs index aeb8a58..1890d5a 100644 --- a/tests/SensitiveFlow.Core.Tests/SensitiveFlowToolRunnerTests.cs +++ b/tests/SensitiveFlow.Core.Tests/SensitiveFlowToolRunnerTests.cs @@ -1,4 +1,5 @@ #if NET10_0_OR_GREATER +using System.Reflection; using FluentAssertions; using SensitiveFlow.Tool; @@ -194,5 +195,127 @@ public void Run_ScanSourceInput_WarnsWhenInMemoryOutboxIsInElseBranch() exitCode.Should().Be(4); error.ToString().Should().Contain("SF-CLI-001"); } + + [Fact] + public void Run_WhenBuildProcessCannotStart_ReturnsBuildError() + { + var host = FakeHost.ForProject("C:\\app\\App.csproj"); + host.BuildResult = new SensitiveFlowToolBuildResult(false, false, 5, string.Empty, string.Empty); + using var output = new StringWriter(); + using var error = new StringWriter(); + + var exitCode = SensitiveFlowToolRunner.Run(["scan", "C:\\app\\App.csproj", "C:\\out"], output, error, host); + + exitCode.Should().Be(5); + error.ToString().Should().Contain("Failed to start dotnet build"); + } + + [Fact] + public void Run_WhenBuildTimesOut_ReturnsBuildError() + { + var host = FakeHost.ForProject("C:\\app\\App.csproj"); + host.BuildResult = new SensitiveFlowToolBuildResult(true, true, 5, string.Empty, string.Empty); + using var output = new StringWriter(); + using var error = new StringWriter(); + + var exitCode = SensitiveFlowToolRunner.Run(["scan", "C:\\app\\App.csproj", "C:\\out"], output, error, host); + + exitCode.Should().Be(5); + error.ToString().Should().Contain("timed out"); + } + + [Fact] + public void Run_WhenBuildSucceeds_ScansBuildOutputAndWritesReports() + { + var projectPath = "C:\\app\\App.csproj"; + var outputAssembly = "C:\\app\\bin\\Release\\net10.0\\App.dll"; + var host = FakeHost.ForProject(projectPath); + host.Files.Add(outputAssembly); + host.EnumeratedFiles[("C:\\app", "*.dll", SearchOption.AllDirectories)] = [outputAssembly]; + host.BuildResult = new SensitiveFlowToolBuildResult(true, false, 0, "ok", string.Empty); + host.Assemblies[outputAssembly] = typeof(SensitiveFlowToolRunnerTests).Assembly; + using var output = new StringWriter(); + using var error = new StringWriter(); + + var exitCode = SensitiveFlowToolRunner.Run(["scan", projectPath, "C:\\out"], output, error, host); + + exitCode.Should().Be(0); + host.Writes.Keys.Should().Contain("C:\\out\\sensitiveflow-report.json"); + host.Writes.Keys.Should().Contain("C:\\out\\sensitiveflow-report.md"); + } + + [Fact] + public void Run_ScanDirectory_SkipsInvalidDlls() + { + var host = new FakeHost(); + host.Directories.Add("C:\\bin"); + host.Files.Add("C:\\bin\\bad.dll"); + host.Files.Add("C:\\bin\\good.dll"); + host.EnumeratedFiles[("C:\\bin", "*.dll", SearchOption.AllDirectories)] = ["C:\\bin\\bad.dll", "C:\\bin\\good.dll"]; + host.BadImageFiles.Add("C:\\bin\\bad.dll"); + host.Assemblies["C:\\bin\\good.dll"] = typeof(SensitiveFlowToolRunnerTests).Assembly; + using var output = new StringWriter(); + using var error = new StringWriter(); + + var exitCode = SensitiveFlowToolRunner.Run(["scan", "C:\\bin", "C:\\out"], output, error, host); + + exitCode.Should().Be(0); + host.Writes.Should().ContainKey("C:\\out\\sensitiveflow-report.json"); + } + + private sealed class FakeHost : ISensitiveFlowToolHost + { + public HashSet Files { get; } = new(StringComparer.OrdinalIgnoreCase); + + public HashSet Directories { get; } = new(StringComparer.OrdinalIgnoreCase); + + public Dictionary<(string Path, string Pattern, SearchOption Option), string[]> EnumeratedFiles { get; } = []; + + public Dictionary FileLines { get; } = new(StringComparer.OrdinalIgnoreCase); + + public Dictionary Writes { get; } = new(StringComparer.OrdinalIgnoreCase); + + public Dictionary Assemblies { get; } = new(StringComparer.OrdinalIgnoreCase); + + public HashSet BadImageFiles { get; } = new(StringComparer.OrdinalIgnoreCase); + + public SensitiveFlowToolBuildResult BuildResult { get; set; } = new(true, false, 0, string.Empty, string.Empty); + + public string CurrentDirectory => "C:\\work"; + + public static FakeHost ForProject(string projectPath) + { + var host = new FakeHost(); + host.Files.Add(projectPath); + host.Directories.Add(Path.GetDirectoryName(projectPath)!); + return host; + } + + public bool FileExists(string path) => Files.Contains(path); + + public bool DirectoryExists(string path) => Directories.Contains(path); + + public void CreateDirectory(string path) => Directories.Add(path); + + public IEnumerable EnumerateFiles(string path, string searchPattern, SearchOption searchOption) + => EnumeratedFiles.TryGetValue((path, searchPattern, searchOption), out var files) ? files : []; + + public IEnumerable ReadLines(string path) + => FileLines.TryGetValue(path, out var lines) ? lines : []; + + public void WriteAllText(string path, string contents) => Writes[path] = contents; + + public Assembly LoadAssembly(string path) + { + if (BadImageFiles.Contains(path)) + { + throw new BadImageFormatException(); + } + + return Assemblies[path]; + } + + public SensitiveFlowToolBuildResult Build(string buildTarget, TimeSpan timeout) => BuildResult; + } } #endif From f538ecffca18f3e83e7bf5591d9c3acb36cc6316 Mon Sep 17 00:00:00 2001 From: Mateus Silva Date: Mon, 11 May 2026 19:38:50 -0300 Subject: [PATCH 11/23] feat: Introduce SensitiveFlow.AspNetCore.EFCore for ASP.NET Core and EF Core integration - Added SensitiveFlowWebOptions for configuring SensitiveFlow features. - Implemented SensitiveFlowWebServiceCollectionExtensions for DI registration. - Created SensitiveFlowDatabaseInitializer for automatic database initialization. - Introduced Outbox pattern for audit store with EF Core. - Updated solution and project files to include new components. - Added tests for new features and configurations. --- README.md | 152 +++--- docs/README.md | 37 +- docs/getting-started.md | 127 ++--- docs/package-reference.md | 59 +++ samples/Console.Sample/Program.cs | 61 +-- .../MinimalApi.Sample.csproj | 22 +- samples/MinimalApi.Sample/Program.cs | 433 ++++-------------- samples/QuickStart.Sample/AppDbContext.cs | 29 ++ samples/QuickStart.Sample/Program.cs | 69 +++ .../QuickStart.Sample.csproj | 21 + samples/WebApi.Sample/Program.cs | 135 ++---- samples/WebApi.Sample/WebApi.Sample.csproj | 14 +- sensitiveflow.slnx | 3 + .../Extensions/SensitiveFlowWebOptions.cs | 346 ++++++++++++++ ...itiveFlowWebServiceCollectionExtensions.cs | 320 +++++++++++++ .../SensitiveFlow.AspNetCore.EFCore.csproj | 32 ++ ...OutboxEFCoreServiceCollectionExtensions.cs | 39 ++ ...nsitiveFlow.AspNetCore.EFCore.Tests.csproj | 18 + ...FlowWebServiceCollectionExtensionsTests.cs | 153 +++++++ ...xEFCoreServiceCollectionExtensionsTests.cs | 14 + 20 files changed, 1447 insertions(+), 637 deletions(-) create mode 100644 samples/QuickStart.Sample/AppDbContext.cs create mode 100644 samples/QuickStart.Sample/Program.cs create mode 100644 samples/QuickStart.Sample/QuickStart.Sample.csproj create mode 100644 src/SensitiveFlow.AspNetCore.EFCore/Extensions/SensitiveFlowWebOptions.cs create mode 100644 src/SensitiveFlow.AspNetCore.EFCore/Extensions/SensitiveFlowWebServiceCollectionExtensions.cs create mode 100644 src/SensitiveFlow.AspNetCore.EFCore/SensitiveFlow.AspNetCore.EFCore.csproj create mode 100644 tests/SensitiveFlow.AspNetCore.EFCore.Tests/SensitiveFlow.AspNetCore.EFCore.Tests.csproj create mode 100644 tests/SensitiveFlow.AspNetCore.EFCore.Tests/SensitiveFlowWebServiceCollectionExtensionsTests.cs diff --git a/README.md b/README.md index 02c6917..dd3064c 100644 --- a/README.md +++ b/README.md @@ -45,16 +45,18 @@ Sensitive data flows through your application on every request: EF Core saves, H ## Quick Start -### 1. Install packages +### 1. Install the composition package ```bash -dotnet add package SensitiveFlow.Core -dotnet add package SensitiveFlow.Audit -dotnet add package SensitiveFlow.Audit.EFCore -dotnet add package SensitiveFlow.EFCore -dotnet add package SensitiveFlow.AspNetCore +dotnet add package SensitiveFlow.AspNetCore.EFCore +dotnet add package Microsoft.EntityFrameworkCore.SqlServer # or your EF Core provider ``` +This single package brings in the full recommended stack: Core, Audit, EF Core audit +stores and outbox, Token store, Anonymization, JSON and logging redaction, Retention, +Diagnostics, and Health Checks. Database provider packages stay app-owned, so the +same setup works with SQL Server, PostgreSQL, SQLite, MySQL, or any EF Core provider. + ### 2. Annotate your model ```csharp @@ -78,90 +80,112 @@ public class Customer } ``` -### 3. Register a durable audit store - -`IAuditStore` is the persistence contract -- you own the implementation so audit records go -exactly where your infrastructure requires (SQL, MongoDB, Azure Table Storage, etc.). +`DataSubjectId` or `UserId` is required for EF Core audit correlation. -For EF Core-backed audit storage, use `SensitiveFlow.Audit.EFCore`. It registers an -`IAuditStore` that also implements `IBatchAuditStore`, avoiding one database roundtrip -per sensitive field. +### 3. Register SensitiveFlow ```csharp -builder.Services.AddEfCoreAuditStore(options => - options.UseSqlServer(builder.Configuration.GetConnectionString("AuditStorage"))); -builder.Services.AddAuditStoreRetry(); -``` - -> **Do not use an in-memory store in production.** Audit records must survive process -> restarts. Losing audit history defeats the accountability the audit trail is meant -> to provide. +using SensitiveFlow.AspNetCore.EFCore.Extensions; -### 4. Register services - -```csharp -builder.Services.AddEfCoreAuditStore(options => - options.UseSqlServer(builder.Configuration.GetConnectionString("AuditStorage"))); -builder.Services.AddAuditStoreRetry(); -builder.Services.AddSensitiveFlowEFCore(); // SensitiveFlow.EFCore -builder.Services.AddSensitiveFlowAspNetCore(); // SensitiveFlow.AspNetCore -builder.Services.AddSensitiveFlowLogging(); // SensitiveFlow.Logging -builder.Services.AddSensitiveFlowValidation(o => +builder.Services.AddSensitiveFlowWeb(options => { - o.RequireAuditStore = true; - o.RequireTokenStore = true; + options.UseProfile(SensitiveFlowProfile.Balanced); + + // Provider-agnostic: use any EF Core provider referenced by your app. + options.UseEfCoreStores( + audit => audit.UseSqlServer(builder.Configuration.GetConnectionString("Audit")!), + tokens => tokens.UseSqlServer(builder.Configuration.GetConnectionString("Tokens")!)); + + // You can also configure each store independently: + // options.UseEfCoreAuditStore(audit => audit.UseNpgsql(...)); + // options.UseEfCoreTokenStore(tokens => tokens.UseSqlite(...)); + + // Enable the features your app needs + options.EnableEfCoreAudit(); + options.EnableAspNetCoreContext(); + options.EnableJsonRedaction(); + options.EnableLoggingRedaction(); + options.EnableValidation(); + options.EnableHealthChecks(); + + // Production-grade features (opt-in) + // options.EnableOutbox(); + // options.EnableDiagnostics(); + // options.EnableAuditStoreRetry(); + // options.EnableCachingTokenStore(); + // options.EnableRetention().EnableRetentionExecutor(); + // options.EnableDataSubjectExport().EnableDataSubjectErasure(); }); ``` -### 5. Add the middleware +### 4. Wire your DbContext and middleware ```csharp -// Before UseAuthentication -- makes the pseudonymized IP token available to all handlers. -app.UseSensitiveFlowAudit(); -``` +// DbContext setup +builder.Services.AddDbContext((sp, options) => + options.UseSqlServer(connectionString) + .AddInterceptors(sp.GetRequiredService())); -### 6. Wire the interceptor into your DbContext +var app = builder.Build(); -```csharp -optionsBuilder.AddInterceptors(app.Services.GetRequiredService()); +// Place before UseAuthentication so the pseudonymized IP token +// is available throughout the request pipeline. +app.UseSensitiveFlow(); + +// Map health checks (registers under /health by default) +app.MapHealthChecks("/health"); ``` -### Optional: policies, reports, and health checks +That's it. Every `SaveChanges` on a field annotated with `[PersonalData]` or +`[SensitiveData]` now produces an `AuditRecord` automatically, HTTP responses are +JSON-redacted, logs are scrubbed, and health checks monitor the infrastructure. + +Schema is explicit: databases must already contain the app tables and the +SensitiveFlow audit/token/outbox tables before the first write. SensitiveFlow does +not create tables automatically, even in samples, because schema creation belongs +to migrations or deployment tooling. + +### Optional: policies, reports, and diagnostics ```csharp -builder.Services.AddSensitiveFlow(options => +// Custom policy overrides +builder.Services.AddSensitiveFlowWeb(options => { options.UseProfile(SensitiveFlowProfile.Strict); - options.Policies.ForCategory(DataCategory.Contact) - .MaskInLogs() - .RedactInJson() - .AuditOnChange(); + // ... store config, features ... }); +// Discovery report (CLI or at startup) var report = SensitiveDataDiscovery.Scan(typeof(Customer).Assembly); File.WriteAllText("sensitiveflow-report.md", report.ToMarkdown()); -builder.Services.AddSensitiveFlowHealthChecks() - .AddAuditStoreCheck() - .AddTokenStoreCheck(); - +// Startup validation var startupReport = app.Services.ValidateSensitiveFlow(); ``` -Defaults: - -- profile: `SensitiveFlowProfile.Balanced` -- JSON redaction: `JsonRedactionMode.Mask` -- redaction marker: `[REDACTED]` -- logging: `[Sensitive]` markers are redacted; annotated structured object members are redacted unless policies request `MaskInLogs()` -- retention anonymization marker: `[ANONYMIZED]` -- health checks: `sensitiveflow-audit-store`, `sensitiveflow-token-store`, `sensitiveflow-audit-outbox` -- audit outbox dispatcher: `PollInterval = 1s`, `BatchSize = 100`, `MaxAttempts = 5`, `BackoffStrategy = Exponential`, `DeadLetterAfterMax = true` -- data-subject export: raw annotated values by default; use `[Redaction(Export = ...)]` to mask, redact, or omit specific fields -- CLI: project/solution inputs are built first; `SF-CLI-001` warns when `AddInMemoryAuditOutbox()` is outside `#if DEBUG` - -Every `SaveChanges` on a field annotated with `[PersonalData]` or `[SensitiveData]` now -produces an `AuditRecord` automatically. +### Defaults + +| Area | Default | +|------|---------| +| Profile | `SensitiveFlowProfile.Balanced` | +| JSON redaction mode | `JsonRedactionMode.Mask` | +| Redaction marker | `[REDACTED]` | +| Logging | `[Sensitive]` markers redacted; annotated members redacted unless `MaskInLogs()` | +| Retention anonymization | `[ANONYMIZED]` | +| Audit outbox (when enabled) | `PollInterval=1s`, `BatchSize=100`, `MaxAttempts=5`, `BackoffStrategy=Exponential` | +| Data-subject export | Raw annotated values by default; use `[Redaction(Export = ...)]` to override | +| Health checks | `sensitiveflow-audit-store`, `sensitiveflow-token-store`, `sensitiveflow-audit-outbox` | + +### Advanced: fine-grained package-by-package setup + +The composition layer is the recommended path. If you need precise control over every +registration, install individual packages and register each service manually. See the +[Advanced Composition](docs/package-reference.md) guide for the full package-by-package +setup matrix and individual registration calls. + +> **Do not use in-memory stores in production.** Audit records and token mappings must +> survive process restarts. Losing audit history defeats accountability; losing token +> mappings makes pseudonymized data irrecoverable. ## Documentation diff --git a/docs/README.md b/docs/README.md index e0b5e8c..e571164 100644 --- a/docs/README.md +++ b/docs/README.md @@ -37,24 +37,35 @@ Use this page as the documentation index. The files are grouped by task so contr ## Recommended reading paths -For a web API using EF Core: +**Quick start (recommended):** + +1. [Getting started](getting-started.md) — composition package + EF provider + `AddSensitiveFlowWeb()`. +2. [Package reference / `SensitiveFlow.AspNetCore.EFCore`](package-reference.md#sensitiveflowaspnetcoreefcore) — composition layer reference. +3. [Attributes](attributes.md) + +**Full production web API:** 1. [Getting started](getting-started.md) 2. [Package reference](package-reference.md) 3. [Attributes](attributes.md) 4. [Audit](audit.md) -5. [EF Core](efcore.md) -6. [ASP.NET Core](aspnetcore.md) -7. [JSON redaction](json.md) -8. [TestKit](testkit.md) - -For production hardening: - -1. [Package reference](package-reference.md) -2. [Diagnostics](diagnostics.md) -3. [Audit](audit.md) -4. [Retention](retention.md) -5. [TestKit](testkit.md) +5. [Outbox](outbox-example.md) +6. [EF Core](efcore.md) +7. [ASP.NET Core](aspnetcore.md) +8. [JSON redaction](json.md) +9. [Logging](logging.md) +10. [Diagnostics](diagnostics.md) +11. [Policies, discovery, and health](policies-discovery-health.md) +12. [TestKit](testkit.md) + +For data protection workflows: + +1. [Anonymization](anonymization.md) +2. [Retention](retention.md) + +For advanced composition (per-package control): + +1. [Package reference](package-reference.md) — per-package setup matrix with granular `Add*()` calls. For AI-assisted implementation: diff --git a/docs/getting-started.md b/docs/getting-started.md index c883b74..f5ae294 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -2,25 +2,20 @@ SensitiveFlow is a .NET library that brings observability and control to sensitive data at runtime. It audits EF Core changes, redacts sensitive JSON/log output, and provides masking, pseudonymization, retention, export, and erasure utilities. -## Installation +## Quick Start (recommended) -Install only the packages used by the app: +Install the single composition package: ```bash -dotnet add package SensitiveFlow.Core -dotnet add package SensitiveFlow.Audit -dotnet add package SensitiveFlow.Audit.EFCore -dotnet add package SensitiveFlow.TokenStore.EFCore -dotnet add package SensitiveFlow.EFCore -dotnet add package SensitiveFlow.AspNetCore -dotnet add package SensitiveFlow.Anonymization -dotnet add package SensitiveFlow.Json -dotnet add package SensitiveFlow.Logging -dotnet add package SensitiveFlow.Retention -dotnet add package SensitiveFlow.Diagnostics +dotnet add package SensitiveFlow.AspNetCore.EFCore +dotnet add package Microsoft.EntityFrameworkCore.SqlServer # or your EF Core provider ``` -## Step 1 - Annotate your model +This brings in the full recommended ASP.NET Core + EF Core stack. The database +provider package remains app-owned, so the same SensitiveFlow setup works with any +EF Core provider. + +### Step 1 - Annotate your model ```csharp using SensitiveFlow.Core.Attributes; @@ -45,40 +40,37 @@ public class Customer `DataSubjectId` or `UserId` is required for EF Core audit correlation. -## Step 2 - Register the recommended web stack - -This is the recommended ASP.NET Core + EF Core setup. For a smaller app, keep only the -features you need: `Core` + `Anonymization` works without DI, EF audit needs an -`IAuditStore`, and ASP.NET Core IP pseudonymization needs an `IPseudonymizer`. +### Step 2 - Register SensitiveFlow ```csharp -builder.Services.AddEfCoreAuditStore(options => - options.UseSqlServer(builder.Configuration.GetConnectionString("Audit"))); -builder.Services.AddAuditStoreRetry(); -builder.Services.AddSensitiveFlowDiagnostics(); - -builder.Services.AddEfCoreTokenStore(options => - options.UseSqlServer(builder.Configuration.GetConnectionString("Tokens"))); -builder.Services.AddCachingTokenStore(); - -builder.Services.AddSensitiveFlowEFCore(); -builder.Services.AddSensitiveFlowAspNetCore(); -builder.Services.AddSensitiveFlowLogging(); -builder.Services.AddDataSubjectExport(); -builder.Services.AddDataSubjectErasure(); -builder.Services.AddRetention(); -builder.Services.AddRetentionExecutor(); - -builder.Services.AddControllers() - .AddJsonOptions(options => - options.JsonSerializerOptions.WithSensitiveDataRedaction()); -``` +using SensitiveFlow.AspNetCore.EFCore.Extensions; -`AddEfCoreTokenStore(...)` registers both `ITokenStore` and the default -`IPseudonymizer`. Keep that store durable because losing token mappings makes reversible -pseudonymization impossible. +builder.Services.AddSensitiveFlowWeb(options => +{ + options.UseProfile(SensitiveFlowProfile.Balanced); + + options.UseEfCoreStores( + audit => audit.UseSqlServer(builder.Configuration.GetConnectionString("Audit")), + tokens => tokens.UseSqlServer(builder.Configuration.GetConnectionString("Tokens"))); + + options.EnableEfCoreAudit(); + options.EnableAspNetCoreContext(); + options.EnableLoggingRedaction(); + options.EnableJsonRedaction(); + options.EnableValidation(); + options.EnableHealthChecks(); + + // Enable as needed: + // options.EnableOutbox(); + // options.EnableDiagnostics(); + // options.EnableAuditStoreRetry(); + // options.EnableCachingTokenStore(); + // options.EnableRetention().EnableRetentionExecutor(); + // options.EnableDataSubjectExport().EnableDataSubjectErasure(); +}); +``` -## Step 3 - Wire your DbContext +### Step 3 - Wire your DbContext and middleware ```csharp builder.Services.AddDbContext((provider, options) => @@ -86,28 +78,53 @@ builder.Services.AddDbContext((provider, options) => options.UseSqlServer(builder.Configuration.GetConnectionString("App")); options.AddInterceptors(provider.GetRequiredService()); }); -``` -## Step 4 - Add the middleware +var app = builder.Build(); -```csharp -app.UseSensitiveFlowAudit(); +// Place before UseAuthentication. +app.UseSensitiveFlow(); app.UseAuthentication(); app.UseAuthorization(); +app.MapHealthChecks("/health"); ``` -Configure forwarded headers before `UseSensitiveFlowAudit` when the app runs behind a proxy/load balancer. +### What you get -## Runtime behavior +| Feature | How it works | +|---------|-------------| +| EF Core audit | Every `SaveChangesAsync` on `[PersonalData]`/`[SensitiveData]` fields writes `AuditRecord` | +| HTTP context | Middleware fills `ActorId`/`IpAddressToken` (pseudonymized) | +| JSON redaction | `System.Text.Json` output is masked/redacted automatically | +| Log redaction | `[Sensitive]` markers and annotated members are scrubbed | +| Startup validation | Verifies configured infrastructure at startup | +| Health checks | `/health` endpoint monitors audit and token stores | -Every `SaveChangesAsync` call scans changed entities for `[PersonalData]` or `[SensitiveData]` and writes audit records through `IAuditStore`. HTTP middleware fills `ActorId`/`IpAddressToken`, JSON redaction protects serialized output, and retention/export/erasure services are called explicitly by your jobs or endpoints. +### Next steps -## Next Steps - -- [Package reference](package-reference.md): package-by-package setup matrix. -- [Audit](audit.md): retry, buffering, query, retention, and snapshot concepts. +- [Package reference](package-reference.md): full package-by-package reference and the `SensitiveFlow.AspNetCore.EFCore` composition layer. +- [Audit](audit.md): retry, buffering, outbox, snapshots, and query concepts. - [EF Core](efcore.md): interceptor behavior and entity requirements. - [ASP.NET Core](aspnetcore.md): request context and IP pseudonymization. - [JSON redaction](json.md): `System.Text.Json` output protection. - [Anonymization](anonymization.md): token stores, masking, export, erasure, fingerprints. - [Retention](retention.md): scheduled retention evaluation and execution. + +## Advanced: per-package setup + +If you need precise control over which packages are installed and how each service +is registered, see the [Package Reference](package-reference.md) for a full +package-by-package setup matrix with individual registration calls. + +The composition package is the recommended path for new adopters. The granular +extension methods (e.g. `AddEfCoreAuditStore`, `AddSensitiveFlowEFCore`) continue +to work and are documented in the package reference for teams that need them. + +## Database schema + +SensitiveFlow does not create tables automatically. The app tables and the +SensitiveFlow audit/token/outbox tables must exist before the first write. + +Use EF Core migrations, checked-in SQL scripts, or your deployment tooling to +create schema in every environment. This is true for local development too, so +local behavior matches production and missing schema fails early instead of being +hidden by `EnsureCreated`. diff --git a/docs/package-reference.md b/docs/package-reference.md index 8b1efd6..59a6e38 100644 --- a/docs/package-reference.md +++ b/docs/package-reference.md @@ -6,6 +6,7 @@ This document summarizes each SensitiveFlow package individually: purpose, prima | Package | Use when | Minimum setup | Main risk | | --- | --- | --- | --- | +| `SensitiveFlow.AspNetCore.EFCore` | **Recommended first package.** You're building an ASP.NET Core + EF Core app and want one composition entry point. | `builder.Services.AddSensitiveFlowWeb(options => { ... })`. | Hides individual registrations; switch to per-package setup when you need full control. | | `SensitiveFlow.Core` | You annotate models or implement contracts. | Add attributes to models. | None by itself; it does not enforce behavior. | | `SensitiveFlow.Audit` | You register custom audit stores or decorators. | `AddAuditStore()` or first-party EF store plus optional retry/buffer. | In-memory/buffered data can be lost before durable write. | | `SensitiveFlow.Audit.EFCore` | You want first-party SQL audit storage. | `AddEfCoreAuditStore(...)`; create/migrate audit table. | Audit DB must be durable and backed up independently. | @@ -25,6 +26,64 @@ This document summarizes each SensitiveFlow package individually: purpose, prima | `SensitiveFlow.TestKit` | You implement custom stores or leak tests. | Inherit contract tests. | Contract tests need isolated fresh stores. | | `SensitiveFlow.Tool` | You want CI/documentation reports from annotated assemblies. | `dotnet tool install SensitiveFlow.Tool`; run `sensitiveflow scan `. | Project/source inputs are built first, then compiled assemblies are scanned. | +## SensitiveFlow.AspNetCore.EFCore + +Purpose: + +- **Recommended first package.** Official high-level composition for ASP.NET Core + EF Core apps. Provides a single `AddSensitiveFlowWeb()` entry point that wires the recommended stack: audit, token store, outbox, JSON and logging redaction, EF Core interception, ASP.NET Core context, validation, diagnostics, health checks, anonymization, and retention. + +Primary APIs: + +- `AddSensitiveFlowWeb(options => { ... })` +- `UseSensitiveFlow()` (middleware — wraps `UseSensitiveFlowAudit()`) +- `SensitiveFlowWebOptions` (fluent builder) + - `UseProfile(SensitiveFlowProfile profile)` + - `UseEfCoreStores(configureAuditStore, configureTokenStore)` (provider-agnostic shorthand) + - `UseEfCoreAuditStore(Action)` + - `UseEfCoreTokenStore(Action)` + - `EnableOutbox()`, `EnableDiagnostics()`, `EnableAuditStoreRetry()` + - `EnableCachingTokenStore()` + - `EnableDataSubjectExport()`, `EnableDataSubjectErasure()` + - `EnableLoggingRedaction()`, `EnableJsonRedaction()` + - `EnableEfCoreAudit()`, `EnableAspNetCoreContext()` + - `EnableValidation()`, `EnableHealthChecks()` + - `EnableRetention()`, `EnableRetentionExecutor()` + +Install when: + +- You're building an ASP.NET Core + EF Core app and want a single-line setup. +- You're onboarding to SensitiveFlow and want sensible defaults without reading every package's setup docs. + +Recommended setup: + +```csharp +builder.Services.AddSensitiveFlowWeb(options => +{ + options.UseProfile(SensitiveFlowProfile.Balanced); + options.UseEfCoreStores( + audit => audit.UseSqlServer(builder.Configuration.GetConnectionString("Audit")!), + tokens => tokens.UseSqlServer(builder.Configuration.GetConnectionString("Tokens")!)); + options.EnableEfCoreAudit(); + options.EnableAspNetCoreContext(); + options.EnableJsonRedaction(); + options.EnableLoggingRedaction(); + options.EnableValidation(); + options.EnableHealthChecks(); +}); + +app.UseSensitiveFlow(); +app.MapHealthChecks("/health"); +``` + +Operational notes: + +- This package depends on all other SensitiveFlow packages. Database provider packages remain app-owned, so the same API works with SQL Server, PostgreSQL, SQLite, MySQL, or any EF Core provider. +- Every `Enable*()` method maps to the corresponding granular extension method (`AddEfCoreAuditStore`, `AddSensitiveFlowEFCore`, etc.). +- SensitiveFlow does not create database tables automatically. If the audit/token/outbox tables do not exist, EF Core persistence will fail on first write. That is intentional: schema creation is an app/deployment responsibility. +- Use migrations, checked-in SQL scripts, or your deployment tooling to create the app tables and SensitiveFlow infrastructure tables before startup. +- When you outgrow the composition layer, switch to per-package setup for full control. +- Health checks automatically include `AddAuditOutboxCheck()` when outbox is enabled. + ## SensitiveFlow.Core Purpose: diff --git a/samples/Console.Sample/Program.cs b/samples/Console.Sample/Program.cs index a772924..f03c7d2 100644 --- a/samples/Console.Sample/Program.cs +++ b/samples/Console.Sample/Program.cs @@ -1,11 +1,12 @@ // --------------------------------------------- // SensitiveFlow - Console Sample // -// Shows a real persistence setup: -// - SQLite via EF Core (durable AuditStore + TokenStore) -// - Serilog for structured log redaction -// - OpenTelemetry for activity traces -// - [PersonalData] / [SensitiveData] attributes + retention evaluation +// Demonstrates core concepts without ASP.NET Core DI: +// - Manual EF Core + SQLite for durable AuditStore and TokenStore +// - Attribute-based classification ([PersonalData] / [SensitiveData]) +// - Masking, pseudonymization, retention evaluation, and discovery +// +// For the recommended DI-based setup, see QuickStart.Sample. // --------------------------------------------- using System.Diagnostics; @@ -29,7 +30,7 @@ using SensitiveFlow.EFCore.Interceptors; using SensitiveFlow.Retention.Services; -// ── Serilog ──────────────────────────────────────────────────────────────── +// -- Serilog ---------------------------------------------------------------- Log.Logger = new LoggerConfiguration() .MinimumLevel.Debug() .WriteTo.Console(outputTemplate: @@ -39,7 +40,7 @@ restrictedToMinimumLevel: LogEventLevel.Information) .CreateLogger(); -// ── OpenTelemetry ─────────────────────────────────────────────────────────── +// -- OpenTelemetry ----------------------------------------------------------- var activitySource = new ActivitySource("SensitiveFlow.Console.Sample"); using var tracerProvider = Sdk.CreateTracerProviderBuilder() @@ -48,7 +49,7 @@ .AddConsoleExporter() .Build(); -// ── EF Core: SQLite with durable AuditStore and TokenStore ───────────────── +// -- EF Core: SQLite with durable AuditStore and TokenStore ----------------- var options = new DbContextOptionsBuilder() .UseSqlite("Data Source=sensitiveflow-console.db") .Options; @@ -65,9 +66,9 @@ var auditContext = new StaticAuditContext("console-runner", ipToken: null); var interceptor = new SensitiveDataAuditInterceptor(auditStore, auditContext); -// ────────────────────────────────────────────────────────────────────────── +// -------------------------------------------------------------------------- Section("1. Annotating models with [PersonalData] / [SensitiveData]"); -// ────────────────────────────────────────────────────────────────────────── +// -------------------------------------------------------------------------- var customer = new Customer { @@ -83,9 +84,9 @@ customer.Name.MaskName(), customer.Email.MaskEmail()); -// ────────────────────────────────────────────────────────────────────────── +// -------------------------------------------------------------------------- Section("2. Persisting to SQLite and generating automatic audit trail"); -// ────────────────────────────────────────────────────────────────────────── +// -------------------------------------------------------------------------- using (var activity = activitySource.StartActivity("SaveCustomer")) { @@ -104,9 +105,9 @@ Log.Information("Customer {DataSubjectId} saved", customer.DataSubjectId); } -// ────────────────────────────────────────────────────────────────────────── +// -------------------------------------------------------------------------- Section("3. Querying the durable audit trail from SQLite"); -// ────────────────────────────────────────────────────────────────────────── +// -------------------------------------------------------------------------- var auditRecords = await auditStore.QueryByDataSubjectAsync(customer.DataSubjectId); Console.WriteLine($" Audit records for {customer.DataSubjectId}:"); @@ -115,9 +116,9 @@ Console.WriteLine($" [{r.Timestamp:u}] {r.Operation,-10} {r.Entity}.{r.Field,-12} actor={r.ActorId ?? "-"}"); } -// ────────────────────────────────────────────────────────────────────────── -Section("4. IP pseudonymization — token survives to SQLite"); -// ────────────────────────────────────────────────────────────────────────── +// -------------------------------------------------------------------------- +Section("4. IP pseudonymization -- token survives to SQLite"); +// -------------------------------------------------------------------------- var rawIp = "192.168.100.42"; var ipToken = await pseudonymizer.PseudonymizeAsync(rawIp); @@ -139,18 +140,18 @@ await auditStore.AppendAsync(new AuditRecord var resolvedIp = await pseudonymizer.ReverseAsync(ipToken); Console.WriteLine($" Token {ipToken[..8]}... resolved to: {resolvedIp}"); -// ────────────────────────────────────────────────────────────────────────── +// -------------------------------------------------------------------------- Section("5. Masking and anonymization"); -// ────────────────────────────────────────────────────────────────────────── +// -------------------------------------------------------------------------- Console.WriteLine($" Name masked : {customer.Name.MaskName()}"); Console.WriteLine($" Email masked : {customer.Email.MaskEmail()}"); Console.WriteLine($" Phone masked : {customer.Phone.MaskPhone()}"); Console.WriteLine($" TaxId anon. : {new BrazilianTaxIdAnonymizer().Anonymize(customer.TaxId)}"); -// ────────────────────────────────────────────────────────────────────────── +// -------------------------------------------------------------------------- Section("6. Retention evaluation"); -// ────────────────────────────────────────────────────────────────────────── +// -------------------------------------------------------------------------- var retentionEvaluator = new RetentionEvaluator(Enumerable.Empty()); @@ -165,9 +166,9 @@ await auditStore.AppendAsync(new AuditRecord Console.WriteLine($" Retention expired: {ex.Message}"); } -// ────────────────────────────────────────────────────────────────────────── +// -------------------------------------------------------------------------- Section("7. OpenTelemetry trace exported"); -// ────────────────────────────────────────────────────────────────────────── +// -------------------------------------------------------------------------- using (var span = activitySource.StartActivity("AuditQueryDemo")) { @@ -209,15 +210,15 @@ await auditStore.AppendAsync(new AuditRecord Log.Information("Console sample finished."); await Log.CloseAndFlushAsync(); -// ── Helpers ──────────────────────────────────────────────────────────────── +// -- Helpers ---------------------------------------------------------------- static void Section(string title) { Console.WriteLine(); - Console.WriteLine($"── {title} ──"); + Console.WriteLine($"-- {title} --"); } -// ── Domain model ────────────────────────────────────────────────────────── +// -- Domain model ---------------------------------------------------------- public sealed class Customer { @@ -240,7 +241,7 @@ public sealed class Customer public DateTimeOffset CreatedAt { get; set; } } -// ── EF Core DbContext with durable AuditStore and TokenStore ────────────── +// -- EF Core DbContext with durable AuditStore and TokenStore -------------- public sealed class SampleDbContext : DbContext { @@ -275,7 +276,7 @@ public sealed class TokenMappingEntity public string Token { get; set; } = string.Empty; } -// ── Durable IAuditStore backed by EF Core / SQLite ──────────────────────── +// -- Durable IAuditStore backed by EF Core / SQLite ------------------------ public sealed class EfCoreAuditStore : IAuditStore { @@ -336,7 +337,7 @@ public async Task> QueryByDataSubjectAsync( }; } -// ── Durable ITokenStore backed by EF Core / SQLite ──────────────────────── +// -- Durable ITokenStore backed by EF Core / SQLite ------------------------ public sealed class EfCoreTokenStore : ITokenStore { @@ -373,7 +374,7 @@ public async Task ResolveTokenAsync(string token, CancellationToken canc } } -// ── Static IAuditContext for console (no HTTP context) ──────────────────── +// -- Static IAuditContext for console (no HTTP context) -------------------- public sealed class StaticAuditContext : IAuditContext { diff --git a/samples/MinimalApi.Sample/MinimalApi.Sample.csproj b/samples/MinimalApi.Sample/MinimalApi.Sample.csproj index 1d3595d..3675fdd 100644 --- a/samples/MinimalApi.Sample/MinimalApi.Sample.csproj +++ b/samples/MinimalApi.Sample/MinimalApi.Sample.csproj @@ -1,7 +1,7 @@ - net10.0 + net10.0 enable enable false @@ -11,28 +11,10 @@ - - - - - - - - - - - - - - - - - - - + diff --git a/samples/MinimalApi.Sample/Program.cs b/samples/MinimalApi.Sample/Program.cs index 5608293..16bbbfa 100644 --- a/samples/MinimalApi.Sample/Program.cs +++ b/samples/MinimalApi.Sample/Program.cs @@ -1,378 +1,103 @@ using Microsoft.EntityFrameworkCore; using MinimalApi.Sample.Infrastructure; -using OpenTelemetry.Metrics; -using OpenTelemetry.Resources; -using OpenTelemetry.Trace; -using Serilog; -using Serilog.Events; -using SensitiveFlow.Anonymization.Erasure; -using SensitiveFlow.Anonymization.Extensions; -using SensitiveFlow.Anonymization.Export; -using SensitiveFlow.Audit.EFCore; -using SensitiveFlow.Audit.EFCore.Extensions; -using SensitiveFlow.Audit.Extensions; -using SensitiveFlow.AspNetCore.Extensions; -using SensitiveFlow.Core.Diagnostics; -using SensitiveFlow.Core.Discovery; -using SensitiveFlow.Core.Enums; -using SensitiveFlow.Core.Interfaces; -using SensitiveFlow.Core.Models; -using SensitiveFlow.Core.Policies; +using SensitiveFlow.AspNetCore.EFCore.Extensions; using SensitiveFlow.Core.Profiles; -using SensitiveFlow.Diagnostics.Extensions; -using SensitiveFlow.EFCore.Extensions; -using SensitiveFlow.EFCore.Interceptors; -using SensitiveFlow.HealthChecks.Extensions; -using SensitiveFlow.Json.Configuration; -using SensitiveFlow.Json.Enums; -using SensitiveFlow.Json.Extensions; -using SensitiveFlow.Logging.Extensions; -using SensitiveFlow.Retention.Extensions; -using SensitiveFlow.Retention.Services; -using SensitiveFlow.TokenStore.EFCore; -using SensitiveFlow.TokenStore.EFCore.Extensions; -Log.Logger = new LoggerConfiguration() - .MinimumLevel.Override("Microsoft.AspNetCore", LogEventLevel.Warning) - .Enrich.FromLogContext() - .WriteTo.Console(outputTemplate: - "[{Timestamp:HH:mm:ss} {Level:u3}] {Message:lj}{NewLine}{Exception}") - .WriteTo.File("logs/minimalapi-sample-.log", - rollingInterval: RollingInterval.Day, - restrictedToMinimumLevel: LogEventLevel.Information) - .CreateBootstrapLogger(); +var builder = WebApplication.CreateBuilder(args); -try -{ - var builder = WebApplication.CreateBuilder(args); - - builder.Host.UseSerilog((ctx, services, config) => config - .MinimumLevel.Override("Microsoft.AspNetCore", LogEventLevel.Warning) - .Enrich.FromLogContext() - .WriteTo.Console(outputTemplate: - "[{Timestamp:HH:mm:ss} {Level:u3}] {Message:lj}{NewLine}{Exception}") - .WriteTo.File("logs/minimalapi-sample-.log", rollingInterval: RollingInterval.Day)); - - var appConnection = builder.Configuration.GetConnectionString("Default") - ?? "Data Source=sensitiveflow-minimalapi.db"; - var auditConnection = builder.Configuration.GetConnectionString("Audit") - ?? "Data Source=sensitiveflow-minimalapi-audit.db"; - var tokenConnection = builder.Configuration.GetConnectionString("Tokens") - ?? "Data Source=sensitiveflow-minimalapi-tokens.db"; - - builder.Services.AddDbContext((sp, options) => - options.UseSqlite(appConnection) - .AddInterceptors(sp.GetRequiredService())); - - builder.Services.AddEfCoreAuditStore(options => options.UseSqlite(auditConnection)); - builder.Services.AddAuditStoreRetry(); - builder.Services.AddSensitiveFlowDiagnostics(); - builder.Services.AddEfCoreTokenStore(options => options.UseSqlite(tokenConnection)); - builder.Services.AddCachingTokenStore(); - builder.Services.AddDataSubjectExport(); - builder.Services.AddDataSubjectErasure(); - - var sensitiveFlowOptions = SensitiveFlowPolicyConfiguration.Create(options => - { - options.UseProfile(SensitiveFlowProfile.Balanced); - options.Policies.ForCategory(DataCategory.Contact) - .MaskInLogs() - .RedactInJson() - .AuditOnChange(); - options.Policies.ForSensitiveCategory(SensitiveDataCategory.Other) - .OmitInJson() - .RequireAudit(); - }); - - builder.Services.AddSingleton(sensitiveFlowOptions); - builder.Services.AddSensitiveFlowLogging(options => - { - options.Policies = sensitiveFlowOptions.Policies; - }); - builder.Services.AddSensitiveFlowEFCore(); - builder.Services.AddSensitiveFlowAspNetCore(); - builder.Services.AddSensitiveFlowValidation(options => - { - options.RequireAuditStore = true; - options.RequireTokenStore = true; - options.RequireJsonRedaction = true; - options.RequireRetention = true; - }); - builder.Services.AddSensitiveFlowJsonRedaction(options => - { - options.DefaultMode = JsonRedactionMode.Mask; - options.Policies = sensitiveFlowOptions.Policies; - }); - builder.Services.AddRetention(); - builder.Services.AddRetentionExecutor(); - builder.Services.AddSensitiveFlowHealthChecks() - .AddAuditStoreCheck() - .AddTokenStoreCheck(); - builder.Services.ConfigureHttpJsonOptions(options => - options.SerializerOptions.WithSensitiveDataRedaction( - new JsonRedactionOptions - { - DefaultMode = JsonRedactionMode.Mask, - Policies = sensitiveFlowOptions.Policies, - })); - - builder.Services.AddOpenTelemetry() - .WithTracing(tracing => tracing - .SetResourceBuilder(ResourceBuilder.CreateDefault() - .AddService("SensitiveFlow.MinimalApi.Sample")) - .AddSource(SensitiveFlowDiagnostics.ActivitySourceName) - .AddAspNetCoreInstrumentation() - .AddHttpClientInstrumentation() - .AddConsoleExporter()) - .WithMetrics(metrics => metrics - .AddMeter(SensitiveFlowDiagnostics.MeterName) - .AddConsoleExporter()); - - builder.Services.AddOpenApi(); - - var app = builder.Build(); - - using (var scope = app.Services.CreateScope()) - { - await scope.ServiceProvider.GetRequiredService() - .Database.EnsureCreatedAsync(); - - await using var auditDb = await scope.ServiceProvider - .GetRequiredService>() - .CreateDbContextAsync(); - await auditDb.Database.EnsureCreatedAsync(); - - await using var tokenDb = await scope.ServiceProvider - .GetRequiredService>() - .CreateDbContextAsync(); - await tokenDb.Database.EnsureCreatedAsync(); - } - - if (app.Environment.IsDevelopment()) - { - app.MapOpenApi(); - } - - app.UseHttpsRedirection(); - app.UseSensitiveFlowAudit(); - app.MapHealthChecks("/health/sensitiveflow"); - - app.MapPost("/customers", async ( - CreateCustomerRequest request, - SampleDbContext db, - ILogger logger, - CancellationToken ct) => - { - var customer = new Customer - { - DataSubjectId = Guid.NewGuid().ToString(), - Name = request.Name, - Email = request.Email, - TaxId = request.TaxId, - Phone = request.Phone, - CreatedAt = DateTimeOffset.UtcNow, - }; - - db.Customers.Add(customer); - await db.SaveChangesAsync(ct); - - logger.LogInformation("Customer created - name: {Name}, email: {[Sensitive]Email}", - customer.Name.MaskName(), - customer.Email.MaskEmail()); - - return Results.Created($"/customers/{customer.DataSubjectId}", ToResponse(customer)); - }) - .WithName("CreateCustomer"); - - app.MapGet("/customers/{id}", async ( - string id, - SampleDbContext db, - IAuditStore auditStore, - IAuditContext auditContext, - ILogger logger, - CancellationToken ct) => - { - var customer = await db.Customers - .FirstOrDefaultAsync(c => c.DataSubjectId == id, ct); +var appConnection = builder.Configuration.GetConnectionString("Default") + ?? "Data Source=sensitiveflow-minimalapi.db"; +var auditConnection = builder.Configuration.GetConnectionString("Audit") + ?? "Data Source=sensitiveflow-minimalapi-audit.db"; +var tokenConnection = builder.Configuration.GetConnectionString("Tokens") + ?? "Data Source=sensitiveflow-minimalapi-tokens.db"; - if (customer is null) - { - return Results.NotFound(); - } +builder.Services.AddDbContext((sp, options) => + options.UseSqlite(appConnection) + .AddInterceptors(sp.GetRequiredService())); - await auditStore.AppendAsync(new AuditRecord - { - DataSubjectId = customer.DataSubjectId, - Entity = nameof(Customer), - Field = "*", - Operation = AuditOperation.Access, - ActorId = auditContext.ActorId, - IpAddressToken = auditContext.IpAddressToken, - }, ct); - - logger.LogInformation("Customer {Id} accessed - email: {[Sensitive]Email}", - id.SanitizeForLog(), customer.Email.MaskEmail()); - - return Results.Ok(ToResponse(customer)); - }) - .WithName("GetCustomer"); - - app.MapGet("/customers/{id}/audit", async ( - string id, - IAuditStore auditStore, - CancellationToken ct) => - { - var records = await auditStore.QueryByDataSubjectAsync(id, cancellationToken: ct); - return Results.Ok(records); - }) - .WithName("GetCustomerAudit"); - - app.MapGet("/customers/{id}/json", async ( - string id, - SampleDbContext db, - CancellationToken ct) => - { - var customer = await db.Customers - .FirstOrDefaultAsync(c => c.DataSubjectId == id, ct); - - return customer is null ? Results.NotFound() : Results.Ok(customer); - }) - .WithName("GetCustomerWithJsonRedaction"); - - app.MapGet("/customers/{id}/export", async ( - string id, - SampleDbContext db, - IDataSubjectExporter exporter, - CancellationToken ct) => - { - var customer = await db.Customers - .AsNoTracking() - .FirstOrDefaultAsync(c => c.DataSubjectId == id, ct); - - return customer is null ? Results.NotFound() : Results.Ok(exporter.Export(customer)); - }) - .WithName("ExportCustomerData"); - - app.MapPost("/customers/{id}/erase", async ( - string id, - SampleDbContext db, - IDataSubjectErasureService erasure, - IAuditStore auditStore, - IAuditContext auditContext, - CancellationToken ct) => - { - var customer = await db.Customers - .FirstOrDefaultAsync(c => c.DataSubjectId == id, ct); - - if (customer is null) - { - return Results.NotFound(); - } +builder.Services.AddSensitiveFlowWeb(options => +{ + options.UseProfile(SensitiveFlowProfile.Balanced); - var changed = erasure.Erase(customer); - await db.SaveChangesAsync(ct); + options.UseEfCoreStores( + audit => audit.UseSqlite(auditConnection), + tokens => tokens.UseSqlite(tokenConnection)); - await auditStore.AppendAsync(new AuditRecord - { - DataSubjectId = customer.DataSubjectId, - Entity = nameof(Customer), - Field = "*", - Operation = AuditOperation.Anonymize, - ActorId = auditContext.ActorId, - IpAddressToken = auditContext.IpAddressToken, - Details = $"Erased {changed} annotated fields.", - }, ct); + options.EnableEfCoreAudit(); + options.EnableAspNetCoreContext(); + options.EnableJsonRedaction(); + options.EnableLoggingRedaction(); + options.EnableValidation(); + options.EnableHealthChecks(); +}); - return Results.Ok(new { changed }); - }) - .WithName("EraseCustomerData"); +builder.Services.AddOpenApi(); - app.MapPost("/retention/run", async ( - SampleDbContext db, - RetentionExecutor retention, - CancellationToken ct) => - { - var customers = await db.Customers.ToListAsync(ct); - var report = await retention.ExecuteAsync( - customers, - entity => ((Customer)entity).CreatedAt, - ct); +var app = builder.Build(); - await db.SaveChangesAsync(ct); +if (app.Environment.IsDevelopment()) +{ + app.MapOpenApi(); +} - return Results.Ok(new - { - report.AnonymizedFieldCount, - report.DeletePendingEntityCount, - Entries = report.Entries.Select(e => new - { - e.FieldName, - e.ExpiredAt, - Action = e.Action.ToString(), - }), - }); - }) - .WithName("RunRetention"); +app.UseHttpsRedirection(); +app.UseSensitiveFlow(); +app.MapHealthChecks("/health/sensitiveflow"); - app.MapPost("/retention/dry-run", async ( - SampleDbContext db, - RetentionExecutor retention, - CancellationToken ct) => - { - var customers = await db.Customers.AsNoTracking().ToListAsync(ct); - var report = await retention.DryRunAsync( - customers, - entity => ((Customer)entity).CreatedAt, - ct); +app.MapPost("/customers", async ( + CreateCustomerRequest request, + SampleDbContext db, + CancellationToken ct) => +{ + var customer = new Customer + { + DataSubjectId = Guid.NewGuid().ToString(), + Name = request.Name, + Email = request.Email, + TaxId = request.TaxId, + Phone = request.Phone, + CreatedAt = DateTimeOffset.UtcNow, + }; + + db.Customers.Add(customer); + await db.SaveChangesAsync(ct); + + return Results.Created($"/customers/{customer.DataSubjectId}", ToResponse(customer)); +}) +.WithName("CreateCustomer"); + +app.MapGet("/customers/{id}", async ( + string id, + SampleDbContext db, + CancellationToken ct) => +{ + var customer = await db.Customers + .FirstOrDefaultAsync(c => c.DataSubjectId == id, ct); - return Results.Ok(new - { - report.AnonymizedFieldCount, - report.DeletePendingEntityCount, - Entries = report.Entries.Select(e => new - { - e.FieldName, - e.ExpiredAt, - Action = e.Action.ToString(), - }), - }); - }) - .WithName("DryRunRetention"); + return customer is null ? Results.NotFound() : Results.Ok(customer); +}) +.WithName("GetCustomer"); - app.MapGet("/sensitiveflow/discovery", () => - { - var report = SensitiveDataDiscovery.Scan(typeof(Customer).Assembly); - return Results.Text(report.ToMarkdown(), "text/markdown"); - }) - .WithName("SensitiveFlowDiscovery"); +app.MapGet("/customers/{id}/json", async ( + string id, + SampleDbContext db, + CancellationToken ct) => +{ + var customer = await db.Customers + .FirstOrDefaultAsync(c => c.DataSubjectId == id, ct); - app.MapGet("/sensitiveflow/diagnostics", (IServiceProvider services) => - { - var report = services.ValidateSensitiveFlow(); - return Results.Ok(new - { - report.IsValid, - report.Diagnostics, - }); - }) - .WithName("SensitiveFlowDiagnostics"); + return customer is null ? Results.NotFound() : Results.Ok(customer); +}) +.WithName("GetCustomerWithJsonRedaction"); - app.Run(); -} -catch (Exception ex) -{ - Log.Fatal(ex, "Host terminated unexpectedly"); -} -finally -{ - await Log.CloseAndFlushAsync(); -} +app.Run(); static CustomerResponse ToResponse(Customer c) => new( c.DataSubjectId, - c.Name.MaskName(), - c.Email.MaskEmail(), - c.Phone.MaskPhone()); + c.Name, + c.Email, + c.Phone); public sealed record CreateCustomerRequest( string Name, diff --git a/samples/QuickStart.Sample/AppDbContext.cs b/samples/QuickStart.Sample/AppDbContext.cs new file mode 100644 index 0000000..2dbc717 --- /dev/null +++ b/samples/QuickStart.Sample/AppDbContext.cs @@ -0,0 +1,29 @@ +using Microsoft.EntityFrameworkCore; +using SensitiveFlow.Core.Attributes; +using SensitiveFlow.Core.Enums; + +namespace QuickStart.Sample; + +public sealed class Customer +{ + public int Id { get; set; } + public string DataSubjectId { get; set; } = string.Empty; + + [PersonalData(Category = DataCategory.Identification)] + public string Name { get; set; } = string.Empty; + + [PersonalData(Category = DataCategory.Contact)] + public string Email { get; set; } = string.Empty; + + [SensitiveData(Category = SensitiveDataCategory.Other)] + public string TaxId { get; set; } = string.Empty; + + public DateTimeOffset CreatedAt { get; set; } +} + +public sealed class AppDbContext : DbContext +{ + public AppDbContext(DbContextOptions options) : base(options) { } + + public DbSet Customers => Set(); +} diff --git a/samples/QuickStart.Sample/Program.cs b/samples/QuickStart.Sample/Program.cs new file mode 100644 index 0000000..caf1697 --- /dev/null +++ b/samples/QuickStart.Sample/Program.cs @@ -0,0 +1,69 @@ +using Microsoft.EntityFrameworkCore; +using QuickStart.Sample; +using SensitiveFlow.AspNetCore.EFCore.Extensions; +using SensitiveFlow.Core.Profiles; + +var builder = WebApplication.CreateBuilder(args); + +var connectionString = builder.Configuration.GetConnectionString("Default") + ?? "Data Source=quickstart.db"; + +builder.Services.AddDbContext((sp, options) => + options.UseSqlite(connectionString) + .AddInterceptors(sp.GetRequiredService())); + +builder.Services.AddSensitiveFlowWeb(options => +{ + options.UseProfile(SensitiveFlowProfile.Balanced); + + options.UseEfCoreStores( + audit => audit.UseSqlite("Data Source=quickstart-audit.db"), + tokens => tokens.UseSqlite("Data Source=quickstart-tokens.db")); + + options.EnableJsonRedaction(); + options.EnableLoggingRedaction(); + options.EnableEfCoreAudit(); + options.EnableAspNetCoreContext(); + options.EnableValidation(); + options.EnableHealthChecks(); +}); + +builder.Services.AddOpenApi(); + +var app = builder.Build(); + +if (app.Environment.IsDevelopment()) +{ + app.MapOpenApi(); +} + +app.UseHttpsRedirection(); +app.UseSensitiveFlow(); +app.MapHealthChecks("/health"); + +app.MapPost("/customers", async (CreateCustomerRequest request, AppDbContext db, CancellationToken ct) => +{ + var customer = new Customer + { + DataSubjectId = Guid.NewGuid().ToString(), + Name = request.Name, + Email = request.Email, + TaxId = request.TaxId, + CreatedAt = DateTimeOffset.UtcNow, + }; + + db.Customers.Add(customer); + await db.SaveChangesAsync(ct); + + return Results.Created($"/customers/{customer.DataSubjectId}", customer); +}); + +app.MapGet("/customers/{id}", async (string id, AppDbContext db, CancellationToken ct) => +{ + var customer = await db.Customers.FirstOrDefaultAsync(c => c.DataSubjectId == id, ct); + return customer is null ? Results.NotFound() : Results.Ok(customer); +}); + +app.Run(); + +public sealed record CreateCustomerRequest(string Name, string Email, string TaxId); diff --git a/samples/QuickStart.Sample/QuickStart.Sample.csproj b/samples/QuickStart.Sample/QuickStart.Sample.csproj new file mode 100644 index 0000000..c5abc11 --- /dev/null +++ b/samples/QuickStart.Sample/QuickStart.Sample.csproj @@ -0,0 +1,21 @@ + + + + net10.0 + enable + enable + false + false + + + + + + + + + + + + + diff --git a/samples/WebApi.Sample/Program.cs b/samples/WebApi.Sample/Program.cs index 97f83d5..9765fd5 100644 --- a/samples/WebApi.Sample/Program.cs +++ b/samples/WebApi.Sample/Program.cs @@ -4,28 +4,16 @@ using OpenTelemetry.Trace; using Serilog; using Serilog.Events; -using SensitiveFlow.Anonymization.Extensions; +using SensitiveFlow.AspNetCore.EFCore.Extensions; using SensitiveFlow.Audit.EFCore; -using SensitiveFlow.Audit.EFCore.Extensions; -using SensitiveFlow.Audit.EFCore.Outbox.Extensions; -using SensitiveFlow.Audit.Extensions; -using SensitiveFlow.AspNetCore.Extensions; +using SensitiveFlow.Audit.Outbox; using SensitiveFlow.Core.Diagnostics; using SensitiveFlow.Core.Enums; using SensitiveFlow.Core.Interfaces; -using SensitiveFlow.Core.Policies; using SensitiveFlow.Core.Profiles; -using SensitiveFlow.Diagnostics.Extensions; -using SensitiveFlow.EFCore.Extensions; using SensitiveFlow.EFCore.Interceptors; -using SensitiveFlow.HealthChecks.Extensions; using SensitiveFlow.Json.Configuration; using SensitiveFlow.Json.Enums; -using SensitiveFlow.Json.Extensions; -using SensitiveFlow.Logging.Extensions; -using SensitiveFlow.Retention.Extensions; -using SensitiveFlow.TokenStore.EFCore; -using SensitiveFlow.TokenStore.EFCore.Extensions; using WebApi.Sample.Infrastructure; Log.Logger = new LoggerConfiguration() @@ -60,59 +48,53 @@ options.UseSqlite(appConnection) .AddInterceptors(sp.GetRequiredService())); - builder.Services.AddEfCoreAuditStore(options => options.UseSqlite(auditConnection)); - builder.Services.AddEfCoreAuditOutbox(options => - { - // Configure durable outbox with sensible defaults - options.PollInterval = TimeSpan.FromSeconds(5); // Poll every 5 seconds - options.BatchSize = 100; // Process up to 100 entries per batch - options.MaxAttempts = 5; // Retry up to 5 times before dead-lettering - }); - builder.Services.AddScoped(); - builder.Services.AddScoped(sp => sp.GetRequiredService()); - builder.Services.AddAuditStoreRetry(); - builder.Services.AddSensitiveFlowDiagnostics(); - builder.Services.AddEfCoreTokenStore(options => options.UseSqlite(tokenConnection)); - builder.Services.AddCachingTokenStore(); - builder.Services.AddDataSubjectExport(); - builder.Services.AddDataSubjectErasure(); - - var sensitiveFlowOptions = SensitiveFlowPolicyConfiguration.Create(options => + builder.Services.AddSensitiveFlowWeb(options => { options.UseProfile(SensitiveFlowProfile.Strict); - options.Policies.ForCategory(DataCategory.Contact) - .MaskInLogs() - .RedactInJson() - .AuditOnChange(); - options.Policies.ForSensitiveCategory(SensitiveDataCategory.Other) - .OmitInJson() - .RequireAudit(); - }); - builder.Services.AddSingleton(sensitiveFlowOptions); - builder.Services.AddSensitiveFlowLogging(options => - { - options.Policies = sensitiveFlowOptions.Policies; - }); - builder.Services.AddSensitiveFlowEFCore(); - builder.Services.AddSensitiveFlowAspNetCore(); - builder.Services.AddSensitiveFlowValidation(options => - { - options.RequireAuditStore = true; - options.RequireTokenStore = true; - options.RequireJsonRedaction = true; - options.RequireRetention = true; + options.UseEfCoreStores( + audit => audit.UseSqlite(auditConnection), + tokens => tokens.UseSqlite(tokenConnection)); + + options.EnableEfCoreAudit(); + options.EnableAspNetCoreContext(); + options.EnableJsonRedaction(json => + { + json.DefaultMode = JsonRedactionMode.Mask; + }); + options.EnableLoggingRedaction(); + options.EnableValidation(); + options.EnableDiagnostics(); + options.EnableAuditStoreRetry(); + options.EnableCachingTokenStore(); + options.EnableDataSubjectExport(); + options.EnableDataSubjectErasure(); + options.EnableRetention().EnableRetentionExecutor(); + options.EnableOutbox(outbox => + { + outbox.PollInterval = TimeSpan.FromSeconds(5); + outbox.BatchSize = 100; + outbox.MaxAttempts = 5; + }); + options.EnableHealthChecks(); + + // Custom policy overrides on top of Strict profile + options.ConfigurePolicies(policies => + { + policies.Policies.ForCategory(DataCategory.Contact) + .MaskInLogs() + .RedactInJson() + .AuditOnChange(); + policies.Policies.ForSensitiveCategory(SensitiveDataCategory.Other) + .OmitInJson() + .RequireAudit(); + }); }); - builder.Services.AddSensitiveFlowJsonRedaction(options => - { - options.DefaultMode = JsonRedactionMode.Mask; - options.Policies = sensitiveFlowOptions.Policies; - }); - builder.Services.AddRetention(); - builder.Services.AddRetentionExecutor(); - builder.Services.AddSensitiveFlowHealthChecks() - .AddAuditStoreCheck() - .AddTokenStoreCheck(); + + // Register the outbox publisher (app-level concern) + builder.Services.AddScoped(); + builder.Services.AddScoped(sp => + sp.GetRequiredService()); builder.Services.AddOpenTelemetry() .WithTracing(tracing => tracing @@ -126,41 +108,18 @@ .AddMeter(SensitiveFlowDiagnostics.MeterName) .AddConsoleExporter()); - builder.Services.AddControllers() - .AddJsonOptions(options => - options.JsonSerializerOptions.WithSensitiveDataRedaction( - new JsonRedactionOptions - { - DefaultMode = JsonRedactionMode.Mask, - Policies = sensitiveFlowOptions.Policies, - })); + builder.Services.AddControllers(); builder.Services.AddOpenApi(); var app = builder.Build(); - using (var scope = app.Services.CreateScope()) - { - await scope.ServiceProvider.GetRequiredService() - .Database.EnsureCreatedAsync(); - - await using var auditDb = await scope.ServiceProvider - .GetRequiredService>() - .CreateDbContextAsync(); - await auditDb.Database.EnsureCreatedAsync(); - - await using var tokenDb = await scope.ServiceProvider - .GetRequiredService>() - .CreateDbContextAsync(); - await tokenDb.Database.EnsureCreatedAsync(); - } - if (app.Environment.IsDevelopment()) { app.MapOpenApi(); } app.UseHttpsRedirection(); - app.UseSensitiveFlowAudit(); + app.UseSensitiveFlow(); app.UseAuthorization(); app.MapHealthChecks("/health/sensitiveflow"); app.MapControllers(); diff --git a/samples/WebApi.Sample/WebApi.Sample.csproj b/samples/WebApi.Sample/WebApi.Sample.csproj index 51ec071..dc7b92d 100644 --- a/samples/WebApi.Sample/WebApi.Sample.csproj +++ b/samples/WebApi.Sample/WebApi.Sample.csproj @@ -21,19 +21,7 @@ - - - - - - - - - - - - - + diff --git a/sensitiveflow.slnx b/sensitiveflow.slnx index a493298..b5c6e6d 100644 --- a/sensitiveflow.slnx +++ b/sensitiveflow.slnx @@ -2,6 +2,7 @@ + @@ -10,6 +11,7 @@ + @@ -30,6 +32,7 @@ + diff --git a/src/SensitiveFlow.AspNetCore.EFCore/Extensions/SensitiveFlowWebOptions.cs b/src/SensitiveFlow.AspNetCore.EFCore/Extensions/SensitiveFlowWebOptions.cs new file mode 100644 index 0000000..a8b600c --- /dev/null +++ b/src/SensitiveFlow.AspNetCore.EFCore/Extensions/SensitiveFlowWebOptions.cs @@ -0,0 +1,346 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using SensitiveFlow.Audit.Decorators; +using SensitiveFlow.Audit.Outbox; +using SensitiveFlow.Anonymization.Decorators; +using SensitiveFlow.Core.Profiles; +using SensitiveFlow.Diagnostics.Validation; +using SensitiveFlow.Json.Configuration; +using SensitiveFlow.Json.Enums; +using SensitiveFlow.Logging.Configuration; +using SensitiveFlow.Retention.Services; + +namespace SensitiveFlow.AspNetCore.EFCore.Extensions; + +/// +/// Fluent builder for configuring the high-level SensitiveFlow web composition. +/// +public sealed class SensitiveFlowWebOptions +{ + /// + /// The profile to apply. Defaults to . + /// + public SensitiveFlowProfile Profile { get; set; } = SensitiveFlowDefaults.Profile; + + /// + /// Gets whether the EF Core audit store is enabled. + /// + public bool AuditStoreEnabled { get; private set; } + + /// + /// Gets whether the EF Core token store is enabled. + /// + public bool TokenStoreEnabled { get; private set; } + + /// + /// Gets whether the durable audit outbox is enabled. + /// + public bool OutboxEnabled { get; private set; } + + /// + /// Gets whether JSON redaction is enabled. + /// + public bool JsonRedactionEnabled { get; private set; } + + /// + /// Gets whether logging redaction is enabled. + /// + public bool LoggingRedactionEnabled { get; private set; } + + /// + /// Gets whether EF Core audit interception is enabled. + /// + public bool EfCoreAuditEnabled { get; private set; } + + /// + /// Gets whether ASP.NET Core context enrichment is enabled. + /// + public bool AspNetCoreContextEnabled { get; private set; } + + /// + /// Gets whether startup validation is enabled. + /// + public bool ValidationEnabled { get; private set; } + + /// + /// Gets whether health checks are enabled. + /// + public bool HealthChecksEnabled { get; private set; } + + /// + /// Gets whether diagnostics (OpenTelemetry) wrapping is enabled. + /// + public bool DiagnosticsEnabled { get; private set; } + + /// + /// Gets whether audit store retry is enabled. + /// + public bool AuditStoreRetryEnabled { get; private set; } + + /// + /// Gets whether caching token store is enabled. + /// + public bool CachingTokenStoreEnabled { get; private set; } + + /// + /// Gets whether data-subject export is enabled. + /// + public bool DataSubjectExportEnabled { get; private set; } + + /// + /// Gets whether data-subject erasure is enabled. + /// + public bool DataSubjectErasureEnabled { get; private set; } + + /// + /// Gets whether retention services are enabled. + /// + public bool RetentionEnabled { get; private set; } + + /// + /// Gets whether the retention executor is enabled. + /// + public bool RetentionExecutorEnabled { get; private set; } + + /// + /// Optional callback for custom policy overrides applied after the profile. + /// + internal Action? PoliciesConfigure { get; private set; } + + /// + /// The action used to configure the EF Core audit store. + /// + internal Action? AuditStoreOptionsAction { get; private set; } + + /// + /// The action used to configure the EF Core token store. + /// + internal Action? TokenStoreOptionsAction { get; private set; } + + /// + /// Optional configuration for the audit outbox dispatcher. + /// + internal Action? OutboxConfigure { get; private set; } + + /// + /// Optional configuration for JSON redaction. + /// + internal Action? JsonRedactionConfigure { get; private set; } + + /// + /// Optional configuration for logging redaction. + /// + internal Action? LoggingConfigure { get; private set; } + + /// + /// Optional configuration for startup validation. + /// + internal Action? ValidationConfigure { get; private set; } + + /// + /// Optional configuration for audit store retry. + /// + internal Action? RetryConfigure { get; private set; } + + /// + /// Optional configuration for caching token store. + /// + internal Action? CachingTokenStoreConfigure { get; private set; } + + /// + /// Optional configuration for retention executor. + /// + internal Action? RetentionExecutorConfigure { get; private set; } + + /// + /// Applies a built-in profile. + /// + public SensitiveFlowWebOptions UseProfile(SensitiveFlowProfile profile) + { + Profile = profile; + return this; + } + + /// + /// Applies custom policy overrides after the profile is applied. + /// Use this for fine-grained per-category rules (e.g. downgrading + /// OmitInJson to RedactInJson for a specific category, or adding + /// audit flags). + /// + public SensitiveFlowWebOptions ConfigurePolicies(Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + PoliciesConfigure = configure; + return this; + } + + /// + /// Enables the EF Core audit store with the given provider configuration. + /// + public SensitiveFlowWebOptions UseEfCoreAuditStore(Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + AuditStoreEnabled = true; + AuditStoreOptionsAction = configure; + return this; + } + + /// + /// Enables the EF Core audit store and token store in one provider-agnostic call. + /// The callbacks can use any EF Core provider package referenced by the application + /// (SQL Server, PostgreSQL, SQLite, MySQL, or another provider). + /// + public SensitiveFlowWebOptions UseEfCoreStores( + Action configureAuditStore, + Action configureTokenStore) + { + UseEfCoreAuditStore(configureAuditStore); + UseEfCoreTokenStore(configureTokenStore); + return this; + } + + /// + /// Enables the EF Core token store with the given provider configuration. + /// + public SensitiveFlowWebOptions UseEfCoreTokenStore(Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + TokenStoreEnabled = true; + TokenStoreOptionsAction = configure; + return this; + } + + /// + /// Enables the durable audit outbox with optional dispatcher configuration. + /// + public SensitiveFlowWebOptions EnableOutbox(Action? configure = null) + { + OutboxEnabled = true; + OutboxConfigure = configure; + return this; + } + + /// + /// Enables JSON redaction with optional configuration. + /// + public SensitiveFlowWebOptions EnableJsonRedaction(Action? configure = null) + { + JsonRedactionEnabled = true; + JsonRedactionConfigure = configure; + return this; + } + + /// + /// Enables logging redaction with optional configuration. + /// + public SensitiveFlowWebOptions EnableLoggingRedaction(Action? configure = null) + { + LoggingRedactionEnabled = true; + LoggingConfigure = configure; + return this; + } + + /// + /// Enables EF Core audit interception. + /// + public SensitiveFlowWebOptions EnableEfCoreAudit() + { + EfCoreAuditEnabled = true; + return this; + } + + /// + /// Enables ASP.NET Core context enrichment (actor, IP pseudonymization). + /// + public SensitiveFlowWebOptions EnableAspNetCoreContext() + { + AspNetCoreContextEnabled = true; + return this; + } + + /// + /// Enables startup validation with optional configuration. + /// + public SensitiveFlowWebOptions EnableValidation(Action? configure = null) + { + ValidationEnabled = true; + ValidationConfigure = configure; + return this; + } + + /// + /// Enables health checks for registered SensitiveFlow infrastructure. + /// + public SensitiveFlowWebOptions EnableHealthChecks() + { + HealthChecksEnabled = true; + return this; + } + + /// + /// Enables OpenTelemetry diagnostics wrapping on the audit store. + /// + public SensitiveFlowWebOptions EnableDiagnostics() + { + DiagnosticsEnabled = true; + return this; + } + + /// + /// Enables retry decorator on the audit store with optional configuration. + /// + public SensitiveFlowWebOptions EnableAuditStoreRetry(Action? configure = null) + { + AuditStoreRetryEnabled = true; + RetryConfigure = configure; + return this; + } + + /// + /// Enables caching token store with optional configuration. + /// + public SensitiveFlowWebOptions EnableCachingTokenStore(Action? configure = null) + { + CachingTokenStoreEnabled = true; + CachingTokenStoreConfigure = configure; + return this; + } + + /// + /// Enables data-subject export service. + /// + public SensitiveFlowWebOptions EnableDataSubjectExport() + { + DataSubjectExportEnabled = true; + return this; + } + + /// + /// Enables data-subject erasure service. + /// + public SensitiveFlowWebOptions EnableDataSubjectErasure() + { + DataSubjectErasureEnabled = true; + return this; + } + + /// + /// Enables retention evaluation services. + /// + public SensitiveFlowWebOptions EnableRetention() + { + RetentionEnabled = true; + return this; + } + + /// + /// Enables the retention executor with optional configuration. + /// + public SensitiveFlowWebOptions EnableRetentionExecutor(Action? configure = null) + { + RetentionExecutorEnabled = true; + RetentionExecutorConfigure = configure; + return this; + } + +} diff --git a/src/SensitiveFlow.AspNetCore.EFCore/Extensions/SensitiveFlowWebServiceCollectionExtensions.cs b/src/SensitiveFlow.AspNetCore.EFCore/Extensions/SensitiveFlowWebServiceCollectionExtensions.cs new file mode 100644 index 0000000..7027ea6 --- /dev/null +++ b/src/SensitiveFlow.AspNetCore.EFCore/Extensions/SensitiveFlowWebServiceCollectionExtensions.cs @@ -0,0 +1,320 @@ +using Microsoft.AspNetCore.Builder; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using SensitiveFlow.Audit.Decorators; +using SensitiveFlow.Audit.EFCore; +using SensitiveFlow.Audit.EFCore.Extensions; +using SensitiveFlow.Audit.EFCore.Outbox.Extensions; +using SensitiveFlow.Audit.Extensions; +using SensitiveFlow.Audit.Outbox; +using SensitiveFlow.Anonymization.Decorators; +using SensitiveFlow.Anonymization.Extensions; +using SensitiveFlow.AspNetCore.Extensions; +using SensitiveFlow.Core.Profiles; +using SensitiveFlow.Diagnostics.Extensions; +using SensitiveFlow.Diagnostics.Validation; +using SensitiveFlow.EFCore.Extensions; +using SensitiveFlow.HealthChecks.Extensions; +using SensitiveFlow.Json.Configuration; +using SensitiveFlow.Json.Enums; +using SensitiveFlow.Json.Extensions; +using SensitiveFlow.Logging.Configuration; +using SensitiveFlow.Logging.Extensions; +using SensitiveFlow.Logging.Loggers; +using SensitiveFlow.Logging.Redaction; +using SensitiveFlow.Retention.Extensions; +using SensitiveFlow.Retention.Services; +using SensitiveFlow.TokenStore.EFCore; +using SensitiveFlow.TokenStore.EFCore.Extensions; + +namespace SensitiveFlow.AspNetCore.EFCore.Extensions; + +/// +/// High-level DI extensions that compose the recommended ASP.NET Core + EF Core +/// SensitiveFlow stack with a single call. +/// +/// +/// The granular extension methods (e.g. AddEfCoreAuditStore, +/// AddSensitiveFlowEFCore) continue to exist for advanced scenarios. +/// This composition layer is the official "happy path" for new adopters. +/// +public static class SensitiveFlowWebServiceCollectionExtensions +{ + /// + /// Registers the full recommended SensitiveFlow web stack. + /// + /// The service collection. + /// Configuration callback for the fluent builder. + /// + /// + /// builder.Services.AddSensitiveFlowWeb(options => + /// { + /// options.UseProfile(SensitiveFlowProfile.Balanced); + /// options.UseEfCoreStores( + /// audit => audit.UseSqlite(builder.Configuration.GetConnectionString("Audit")), + /// tokens => tokens.UseSqlite(builder.Configuration.GetConnectionString("Tokens"))); + /// options.EnableJsonRedaction(); + /// options.EnableLoggingRedaction(); + /// options.EnableEfCoreAudit(); + /// options.EnableAspNetCoreContext(); + /// options.EnableValidation(); + /// options.EnableHealthChecks(); + /// }); + /// + /// + public static IServiceCollection AddSensitiveFlowWeb( + this IServiceCollection services, + Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + var options = new SensitiveFlowWebOptions(); + configure(options); + + // 1. Core options (profile + policies) + var sensitiveFlowOptions = new SensitiveFlowOptions(); + sensitiveFlowOptions.UseProfile(options.Profile); + options.PoliciesConfigure?.Invoke(sensitiveFlowOptions); + services.AddSingleton(sensitiveFlowOptions); + + // 2. Audit store + if (options.AuditStoreEnabled) + { + if (options.AuditStoreOptionsAction is null) + { + throw new InvalidOperationException("Audit store is enabled but no DbContext configuration was provided. Call UseEfCoreAuditStore(...)."); + } + + services.AddEfCoreAuditStore(options.AuditStoreOptionsAction); + } + + // 3. Audit store retry + if (options.AuditStoreRetryEnabled) + { + services.AddAuditStoreRetry(options.RetryConfigure); + } + + // 4. Diagnostics (must be after audit store registration) + if (options.DiagnosticsEnabled) + { + services.AddSensitiveFlowDiagnostics(); + } + + // 5. Outbox (must be after audit store) + if (options.OutboxEnabled) + { + services.AddEfCoreAuditOutbox(options.OutboxConfigure); + } + + // 6. Token store + if (options.TokenStoreEnabled) + { + if (options.TokenStoreOptionsAction is null) + { + throw new InvalidOperationException("Token store is enabled but no DbContext configuration was provided. Call UseEfCoreTokenStore(...)."); + } + + services.AddEfCoreTokenStore(options.TokenStoreOptionsAction); + } + + // 7. Caching token store + if (options.CachingTokenStoreEnabled) + { + services.AddCachingTokenStore(options.CachingTokenStoreConfigure); + } + + // 8. Data-subject export / erasure + if (options.DataSubjectExportEnabled) + { + services.AddDataSubjectExport(); + } + + if (options.DataSubjectErasureEnabled) + { + services.AddDataSubjectErasure(); + } + + // 9. Logging redaction + if (options.LoggingRedactionEnabled) + { + AddLoggingRedaction(services, options.LoggingConfigure, sensitiveFlowOptions); + } + + // 10. EF Core audit interceptor + if (options.EfCoreAuditEnabled) + { + services.AddSensitiveFlowEFCore(); + } + + // 11. ASP.NET Core context enrichment + if (options.AspNetCoreContextEnabled) + { + services.AddSensitiveFlowAspNetCore(); + } + + // 12. JSON redaction + if (options.JsonRedactionEnabled) + { + JsonRedactionOptions jsonRedactionOptions; + if (options.JsonRedactionConfigure is not null) + { + services.AddSensitiveFlowJsonRedaction(options.JsonRedactionConfigure); + jsonRedactionOptions = new JsonRedactionOptions(); + options.JsonRedactionConfigure(jsonRedactionOptions); + } + else + { + jsonRedactionOptions = new JsonRedactionOptions + { + DefaultMode = JsonRedactionMode.Mask, + Policies = sensitiveFlowOptions.Policies, + }; + services.AddSensitiveFlowJsonRedaction(json => + { + json.DefaultMode = jsonRedactionOptions.DefaultMode; + json.Policies = jsonRedactionOptions.Policies; + }); + } + + ConfigureAspNetCoreJsonRedaction(services, jsonRedactionOptions); + } + + // 13. Retention + if (options.RetentionEnabled) + { + services.AddRetention(); + } + + if (options.RetentionExecutorEnabled) + { + services.AddRetentionExecutor(options.RetentionExecutorConfigure); + } + + // 14. Validation + if (options.ValidationEnabled) + { + if (options.ValidationConfigure is not null) + { + services.AddSensitiveFlowValidation(options.ValidationConfigure); + } + else + { + services.AddSensitiveFlowValidation(validation => + { + validation.RequireAuditStore = options.AuditStoreEnabled; + validation.RequireTokenStore = options.TokenStoreEnabled; + validation.RequireJsonRedaction = options.JsonRedactionEnabled; + validation.RequireRetention = options.RetentionEnabled; + }); + } + } + + // 15. Health checks + if (options.HealthChecksEnabled) + { + var healthBuilder = services.AddSensitiveFlowHealthChecks(); + + if (options.AuditStoreEnabled) + { + healthBuilder.AddAuditStoreCheck(); + } + + if (options.TokenStoreEnabled) + { + healthBuilder.AddTokenStoreCheck(); + } + + if (options.OutboxEnabled) + { + healthBuilder.AddAuditOutboxCheck(); + } + } + + return services; + } + + /// + /// Adds the SensitiveFlow audit middleware to the ASP.NET Core pipeline. + /// Place this before authentication/authorization so the pseudonymized IP token + /// is available throughout the request. + /// + public static IApplicationBuilder UseSensitiveFlow(this IApplicationBuilder app) + { + ArgumentNullException.ThrowIfNull(app); + app.UseSensitiveFlowAudit(); + return app; + } + + private static void ConfigureAspNetCoreJsonRedaction( + IServiceCollection services, + JsonRedactionOptions redactionOptions) + { + services.ConfigureHttpJsonOptions(options => + options.SerializerOptions.WithSensitiveDataRedaction(redactionOptions)); + + services.Configure(options => + options.JsonSerializerOptions.WithSensitiveDataRedaction(redactionOptions)); + } + + private static void AddLoggingRedaction( + IServiceCollection services, + Action? configure, + SensitiveFlowOptions sensitiveFlowOptions) + { + if (configure is not null) + { + services.AddSensitiveFlowLogging(configure); + } + else + { + services.AddSensitiveFlowLogging(logging => + { + logging.Policies = sensitiveFlowOptions.Policies; + }); + } + + WrapRegisteredLoggerProviders(services); + } + + private static void WrapRegisteredLoggerProviders(IServiceCollection services) + { + var providerDescriptors = services + .Where(d => d.ServiceType == typeof(ILoggerProvider) + && d.ImplementationType != typeof(RedactingLoggerProvider)) + .ToList(); + + foreach (var descriptor in providerDescriptors) + { + services.Remove(descriptor); + services.Add(new ServiceDescriptor( + typeof(ILoggerProvider), + sp => new RedactingLoggerProvider( + CreateLoggerProvider(sp, descriptor), + sp.GetRequiredService(), + sp.GetRequiredService()), + descriptor.Lifetime)); + } + } + + private static ILoggerProvider CreateLoggerProvider(IServiceProvider sp, ServiceDescriptor descriptor) + { + if (descriptor.ImplementationInstance is ILoggerProvider instance) + { + return instance; + } + + if (descriptor.ImplementationFactory is not null) + { + return (ILoggerProvider)descriptor.ImplementationFactory(sp)!; + } + + if (descriptor.ImplementationType is not null) + { + return (ILoggerProvider)ActivatorUtilities.CreateInstance(sp, descriptor.ImplementationType); + } + + throw new InvalidOperationException("Unsupported ILoggerProvider registration."); + } +} diff --git a/src/SensitiveFlow.AspNetCore.EFCore/SensitiveFlow.AspNetCore.EFCore.csproj b/src/SensitiveFlow.AspNetCore.EFCore/SensitiveFlow.AspNetCore.EFCore.csproj new file mode 100644 index 0000000..7a28d39 --- /dev/null +++ b/src/SensitiveFlow.AspNetCore.EFCore/SensitiveFlow.AspNetCore.EFCore.csproj @@ -0,0 +1,32 @@ + + + + Official high-level composition for ASP.NET Core + EF Core. Provides a single AddSensitiveFlowWeb() entry point that wires audit, token store, JSON redaction, logging redaction, EF Core interception, ASP.NET Core context, validation, and health checks. + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SensitiveFlow.Audit.EFCore.Outbox/Extensions/OutboxEFCoreServiceCollectionExtensions.cs b/src/SensitiveFlow.Audit.EFCore.Outbox/Extensions/OutboxEFCoreServiceCollectionExtensions.cs index 8f333bc..9e8b6bb 100644 --- a/src/SensitiveFlow.Audit.EFCore.Outbox/Extensions/OutboxEFCoreServiceCollectionExtensions.cs +++ b/src/SensitiveFlow.Audit.EFCore.Outbox/Extensions/OutboxEFCoreServiceCollectionExtensions.cs @@ -1,5 +1,6 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; +using SensitiveFlow.Audit.Decorators; using SensitiveFlow.Audit.EFCore; using SensitiveFlow.Audit.EFCore.Outbox.Stores; using SensitiveFlow.Audit.Outbox; @@ -33,6 +34,8 @@ public static IServiceCollection AddEfCoreAuditOutbox( services.AddSingleton(sp => (IAuditOutbox)sp.GetRequiredService()); + DecorateAuditStoreWhenRegistered(services); + // Register the dispatcher var dispatcherOptions = new AuditOutboxDispatcherOptions(); configureDispatcher?.Invoke(dispatcherOptions); @@ -41,4 +44,40 @@ public static IServiceCollection AddEfCoreAuditOutbox( return services; } + + private static void DecorateAuditStoreWhenRegistered(IServiceCollection services) + { + var existing = services.FirstOrDefault(d => d.ServiceType == typeof(IAuditStore)); + if (existing is null) + { + return; + } + + services.Remove(existing); + + if (existing.ImplementationType is not null) + { + services.Add(new ServiceDescriptor(existing.ImplementationType, existing.ImplementationType, existing.Lifetime)); + services.Add(new ServiceDescriptor(typeof(IAuditStore), + sp => new OutboxAuditStore( + (IAuditStore)sp.GetRequiredService(existing.ImplementationType), + sp.GetRequiredService()), + existing.Lifetime)); + } + else if (existing.ImplementationFactory is not null) + { + var factory = existing.ImplementationFactory; + services.Add(new ServiceDescriptor(typeof(IAuditStore), + sp => new OutboxAuditStore( + (IAuditStore)factory(sp)!, + sp.GetRequiredService()), + existing.Lifetime)); + } + else if (existing.ImplementationInstance is IAuditStore instance) + { + services.AddSingleton(sp => new OutboxAuditStore( + instance, + sp.GetRequiredService())); + } + } } diff --git a/tests/SensitiveFlow.AspNetCore.EFCore.Tests/SensitiveFlow.AspNetCore.EFCore.Tests.csproj b/tests/SensitiveFlow.AspNetCore.EFCore.Tests/SensitiveFlow.AspNetCore.EFCore.Tests.csproj new file mode 100644 index 0000000..df0c17c --- /dev/null +++ b/tests/SensitiveFlow.AspNetCore.EFCore.Tests/SensitiveFlow.AspNetCore.EFCore.Tests.csproj @@ -0,0 +1,18 @@ + + + false + false + + + + + + + + + + + + + + diff --git a/tests/SensitiveFlow.AspNetCore.EFCore.Tests/SensitiveFlowWebServiceCollectionExtensionsTests.cs b/tests/SensitiveFlow.AspNetCore.EFCore.Tests/SensitiveFlowWebServiceCollectionExtensionsTests.cs new file mode 100644 index 0000000..5bcee95 --- /dev/null +++ b/tests/SensitiveFlow.AspNetCore.EFCore.Tests/SensitiveFlowWebServiceCollectionExtensionsTests.cs @@ -0,0 +1,153 @@ +using System.Text.Json; +using FluentAssertions; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using SensitiveFlow.AspNetCore.EFCore.Extensions; +using SensitiveFlow.Audit.Decorators; +using SensitiveFlow.Core.Attributes; +using SensitiveFlow.Core.Enums; +using SensitiveFlow.Core.Interfaces; +using SensitiveFlow.EFCore.Interceptors; +using SensitiveFlow.Logging.Loggers; + +namespace SensitiveFlow.AspNetCore.EFCore.Tests; + +public sealed class SensitiveFlowWebServiceCollectionExtensionsTests +{ + [Fact] + public void AddSensitiveFlowWeb_DoesNotRegisterDatabaseInitializer() + { + var services = new ServiceCollection(); + + services.AddSensitiveFlowWeb(options => + { + options.UseEfCoreStores( + audit => audit.UseSqlite("Data Source=:memory:"), + tokens => tokens.UseSqlite("Data Source=:memory:")); + }); + + services.Should().NotContain(d => d.ServiceType == typeof(IHostedService)); + } + + [Fact] + public void AddSensitiveFlowWeb_RegistersProviderAgnosticStoresAndInterceptor() + { + var services = new ServiceCollection(); + + services.AddSensitiveFlowWeb(options => + { + options.UseEfCoreStores( + audit => audit.UseSqlite("Data Source=:memory:"), + tokens => tokens.UseSqlite("Data Source=:memory:")); + options.EnableEfCoreAudit(); + }); + + using var provider = services.BuildServiceProvider(); + + provider.GetRequiredService().Should().NotBeNull(); + provider.GetRequiredService().Should().NotBeNull(); + provider.GetRequiredService().Should().NotBeNull(); + } + + [Fact] + public void AddSensitiveFlowWeb_EnableJsonRedaction_ConfiguresHttpJsonOptions() + { + var services = new ServiceCollection(); + services.AddSensitiveFlowWeb(options => options.EnableJsonRedaction()); + using var provider = services.BuildServiceProvider(); + var jsonOptions = provider.GetRequiredService>() + .Value + .SerializerOptions; + + var json = JsonSerializer.Serialize(new Customer { Email = "alice@example.test" }, jsonOptions); + + json.Should().NotContain("alice@example.test"); + json.Should().Contain("[REDACTED]"); + } + + [Fact] + public void AddSensitiveFlowWeb_EnableJsonRedaction_ConfiguresMvcJsonOptions() + { + var services = new ServiceCollection(); + services.AddSensitiveFlowWeb(options => options.EnableJsonRedaction()); + using var provider = services.BuildServiceProvider(); + var jsonOptions = provider.GetRequiredService>() + .Value + .JsonSerializerOptions; + + var json = JsonSerializer.Serialize(new Customer { Email = "alice@example.test" }, jsonOptions); + + json.Should().NotContain("alice@example.test"); + json.Should().Contain("[REDACTED]"); + } + + [Fact] + public void AddSensitiveFlowWeb_EnableLoggingRedaction_WrapsExistingLoggerProviders() + { + var services = new ServiceCollection(); + services.AddSingleton(); + + services.AddSensitiveFlowWeb(options => options.EnableLoggingRedaction()); + using var provider = services.BuildServiceProvider(); + + provider.GetServices() + .Should() + .ContainSingle(provider => provider is RedactingLoggerProvider); + } + + [Fact] + public void AddSensitiveFlowWeb_EnableOutbox_WrapsAuditStore() + { + var services = new ServiceCollection(); + + services.AddSensitiveFlowWeb(options => + { + options.UseEfCoreStores( + audit => audit.UseSqlite("Data Source=:memory:"), + tokens => tokens.UseSqlite("Data Source=:memory:")); + options.EnableOutbox(); + }); + + using var provider = services.BuildServiceProvider(); + + provider.GetRequiredService().Should().BeOfType(); + } + + private sealed class Customer + { + [PersonalData(Category = DataCategory.Contact)] + public string Email { get; set; } = string.Empty; + } + + private sealed class CapturingLoggerProvider : ILoggerProvider + { + public ILogger CreateLogger(string categoryName) => NullLogger.Instance; + + public void Dispose() + { + } + } + + private sealed class NullLogger : ILogger + { + public static readonly NullLogger Instance = new(); + + public IDisposable? BeginScope(TState state) + where TState : notnull + => null; + + public bool IsEnabled(LogLevel logLevel) => false; + + public void Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + } + } +} diff --git a/tests/SensitiveFlow.Audit.EFCore.Outbox.Tests/OutboxEFCoreServiceCollectionExtensionsTests.cs b/tests/SensitiveFlow.Audit.EFCore.Outbox.Tests/OutboxEFCoreServiceCollectionExtensionsTests.cs index c40995c..8efe3aa 100644 --- a/tests/SensitiveFlow.Audit.EFCore.Outbox.Tests/OutboxEFCoreServiceCollectionExtensionsTests.cs +++ b/tests/SensitiveFlow.Audit.EFCore.Outbox.Tests/OutboxEFCoreServiceCollectionExtensionsTests.cs @@ -2,7 +2,9 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using SensitiveFlow.Audit.Decorators; using SensitiveFlow.Audit.EFCore; +using SensitiveFlow.Audit.EFCore.Extensions; using SensitiveFlow.Audit.EFCore.Outbox.Extensions; using SensitiveFlow.Audit.Outbox; using SensitiveFlow.Core.Interfaces; @@ -63,6 +65,18 @@ public void AddEfCoreAuditOutbox_RegistersHostedService() hostedServices.Should().ContainSingle(d => d.ImplementationType == typeof(AuditOutboxDispatcher)); } + [Fact] + public void AddEfCoreAuditOutbox_WhenAuditStoreRegistered_WrapsAuditStore() + { + var services = new ServiceCollection(); + services.AddEfCoreAuditStore(options => options.UseSqlite("Data Source=:memory:")); + + services.AddEfCoreAuditOutbox(); + var provider = services.BuildServiceProvider(); + + provider.GetRequiredService().Should().BeOfType(); + } + [Fact] public void AddEfCoreAuditOutbox_NullServices_ThrowsArgumentNullException() { From e5a4858829d635c8acdc2172b9517f8fe50e0954 Mon Sep 17 00:00:00 2001 From: Mateus Silva Date: Mon, 11 May 2026 21:10:56 -0300 Subject: [PATCH 12/23] feat: Add SQLite schema and initialization for MinimalApi and WebApi samples - Created README.md files for both MinimalApi.Sample and WebApi.Sample to document SQLite schema and manual database initialization. - Added SQL scripts for creating necessary tables and indexes in both samples: - MinimalApi.Sample: - Customers table with relevant fields and indexes. - Audit records and outbox entries for audit functionality. - Token mappings for sensitive data handling. - WebApi.Sample: - Similar structure as MinimalApi with Customers, AuditRecords, and TokenMappings tables. - Updated QuickStart.Sample to include a new HTML interface for customer management and ensure database initialization on startup. - Refactored QuickStart.Sample's Program.cs to remove OpenApi references and streamline the application setup. - Enhanced CustomersController in WebApi.Sample to support listing customers and improved error handling. - Modified SensitiveFlowAuditMiddleware to accept IPseudonymizer as a parameter for better dependency management. - Improved AuditOutboxDispatcher to handle infrastructure failures gracefully and added options for retry delays and suspension behavior. - Updated tests to reflect changes in middleware and dispatcher behavior, ensuring robust coverage for new functionalities. --- README.md | 13 ++ docs/getting-started.md | 14 ++ .../MinimalApi.Sample.csproj | 3 +- samples/MinimalApi.Sample/Program.cs | 131 ++++++++++++++- samples/MinimalApi.Sample/schema/README.md | 21 +++ .../MinimalApi.Sample/schema/sqlite-app.sql | 12 ++ .../MinimalApi.Sample/schema/sqlite-audit.sql | 44 ++++++ .../schema/sqlite-tokens.sql | 11 ++ samples/QuickStart.Sample/Program.cs | 92 ++++++++++- .../QuickStart.Sample.csproj | 3 +- .../Controllers/CustomersController.cs | 42 +++-- samples/WebApi.Sample/Program.cs | 149 +++++++++++++++++- samples/WebApi.Sample/WebApi.Sample.csproj | 1 - samples/WebApi.Sample/schema/README.md | 21 +++ samples/WebApi.Sample/schema/sqlite-app.sql | 12 ++ samples/WebApi.Sample/schema/sqlite-audit.sql | 44 ++++++ .../WebApi.Sample/schema/sqlite-tokens.sql | 11 ++ .../SensitiveFlowAuditMiddleware.cs | 7 +- .../Outbox/AuditOutboxDispatcher.cs | 39 ++++- .../Outbox/AuditOutboxDispatcherOptions.cs | 14 ++ .../SensitiveFlowAuditMiddlewareTests.cs | 20 +-- .../AuditOutboxDispatcherTests.cs | 142 ++++++++++++++--- 22 files changed, 770 insertions(+), 76 deletions(-) create mode 100644 samples/MinimalApi.Sample/schema/README.md create mode 100644 samples/MinimalApi.Sample/schema/sqlite-app.sql create mode 100644 samples/MinimalApi.Sample/schema/sqlite-audit.sql create mode 100644 samples/MinimalApi.Sample/schema/sqlite-tokens.sql create mode 100644 samples/WebApi.Sample/schema/README.md create mode 100644 samples/WebApi.Sample/schema/sqlite-app.sql create mode 100644 samples/WebApi.Sample/schema/sqlite-audit.sql create mode 100644 samples/WebApi.Sample/schema/sqlite-tokens.sql diff --git a/README.md b/README.md index dd3064c..481563c 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,10 @@ stores and outbox, Token store, Anonymization, JSON and logging redaction, Reten Diagnostics, and Health Checks. Database provider packages stay app-owned, so the same setup works with SQL Server, PostgreSQL, SQLite, MySQL, or any EF Core provider. +The repository samples target .NET 10. The library packages remain multi-targeted +where supported, but the documented runnable examples are intentionally kept on +the current .NET sample baseline. + ### 2. Annotate your model ```csharp @@ -145,6 +149,14 @@ SensitiveFlow audit/token/outbox tables before the first write. SensitiveFlow do not create tables automatically, even in samples, because schema creation belongs to migrations or deployment tooling. +The hosted audit outbox dispatcher does not create schema either. If its durable +table is missing, it logs the infrastructure failure and suspends polling by +default (`SuspendOnInfrastructureFailure = true`) instead of stopping the host. + +The `WebApi.Sample` is intentionally different for DX: it creates its local +SQLite demo schema on startup so the UI and routes are immediately testable. Use +migrations or deployment-owned scripts in real applications. + ### Optional: policies, reports, and diagnostics ```csharp @@ -173,6 +185,7 @@ var startupReport = app.Services.ValidateSensitiveFlow(); | Logging | `[Sensitive]` markers redacted; annotated members redacted unless `MaskInLogs()` | | Retention anonymization | `[ANONYMIZED]` | | Audit outbox (when enabled) | `PollInterval=1s`, `BatchSize=100`, `MaxAttempts=5`, `BackoffStrategy=Exponential` | +| Outbox infrastructure failures | Dispatcher suspends polling by default with `SuspendOnInfrastructureFailure=true` | | Data-subject export | Raw annotated values by default; use `[Redaction(Export = ...)]` to override | | Health checks | `sensitiveflow-audit-store`, `sensitiveflow-token-store`, `sensitiveflow-audit-outbox` | diff --git a/docs/getting-started.md b/docs/getting-started.md index f5ae294..c123aa9 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -15,6 +15,10 @@ This brings in the full recommended ASP.NET Core + EF Core stack. The database provider package remains app-owned, so the same SensitiveFlow setup works with any EF Core provider. +The runnable samples in this repository target .NET 10. Use the .NET 10 SDK when +running `samples/QuickStart.Sample`, `samples/MinimalApi.Sample`, or +`samples/WebApi.Sample`. + ### Step 1 - Annotate your model ```csharp @@ -128,3 +132,13 @@ Use EF Core migrations, checked-in SQL scripts, or your deployment tooling to create schema in every environment. This is true for local development too, so local behavior matches production and missing schema fails early instead of being hidden by `EnsureCreated`. + +The audit outbox dispatcher is defensive around missing schema: if polling fails +because the durable outbox table is absent or the database is unavailable, it logs +the infrastructure failure and suspends polling by default instead of stopping the +application host. Apply the schema and restart the app. + +The repository samples are easier on purpose: `samples/WebApi.Sample` creates its +local SQLite schema on startup so you can try the UI and routes immediately. Do +not copy that startup behavior into production apps; use migrations or deployment +scripts for real databases. diff --git a/samples/MinimalApi.Sample/MinimalApi.Sample.csproj b/samples/MinimalApi.Sample/MinimalApi.Sample.csproj index 3675fdd..f87ca7c 100644 --- a/samples/MinimalApi.Sample/MinimalApi.Sample.csproj +++ b/samples/MinimalApi.Sample/MinimalApi.Sample.csproj @@ -1,7 +1,7 @@ - net10.0 + net10.0 enable enable false @@ -9,7 +9,6 @@ - diff --git a/samples/MinimalApi.Sample/Program.cs b/samples/MinimalApi.Sample/Program.cs index 16bbbfa..16d5af4 100644 --- a/samples/MinimalApi.Sample/Program.cs +++ b/samples/MinimalApi.Sample/Program.cs @@ -1,7 +1,9 @@ using Microsoft.EntityFrameworkCore; using MinimalApi.Sample.Infrastructure; using SensitiveFlow.AspNetCore.EFCore.Extensions; +using SensitiveFlow.Audit.EFCore; using SensitiveFlow.Core.Profiles; +using SensitiveFlow.TokenStore.EFCore; var builder = WebApplication.CreateBuilder(args); @@ -32,19 +34,116 @@ options.EnableHealthChecks(); }); -builder.Services.AddOpenApi(); - var app = builder.Build(); -if (app.Environment.IsDevelopment()) -{ - app.MapOpenApi(); -} +await InitializeSampleDatabasesAsync(app.Services); app.UseHttpsRedirection(); app.UseSensitiveFlow(); app.MapHealthChecks("/health/sensitiveflow"); +app.MapGet("/", () => Results.Content(""" + + + + + + SensitiveFlow Minimal API Sample + + + +
+

SensitiveFlow Minimal API Sample

+
+ Sample database. This sample creates its local SQLite tables on startup so the routes work immediately. + Production apps should use EF Core migrations or deployment-owned SQL scripts instead. +
+
+
+

Create customer

+ + + + + +
+
+

Read customer

+ + + + +
+
+
+

Response

+ 
Ready.
+
+
+ + + +""", "text/html")); + +app.MapGet("/customers", async ( + SampleDbContext db, + CancellationToken ct) => +{ + var customers = await db.Customers + .Take(100) + .ToListAsync(ct); + + return Results.Ok(customers + .OrderByDescending(c => c.CreatedAt) + .Take(20) + .Select(ToResponse)); +}) +.WithName("ListCustomers"); + app.MapPost("/customers", async ( CreateCustomerRequest request, SampleDbContext db, @@ -93,6 +192,26 @@ app.Run(); +static async Task InitializeSampleDatabasesAsync(IServiceProvider services) +{ + using var scope = services.CreateScope(); + + await scope.ServiceProvider + .GetRequiredService() + .Database + .EnsureCreatedAsync(); + + await using var auditDb = await scope.ServiceProvider + .GetRequiredService>() + .CreateDbContextAsync(); + await auditDb.Database.EnsureCreatedAsync(); + + await using var tokenDb = await scope.ServiceProvider + .GetRequiredService>() + .CreateDbContextAsync(); + await tokenDb.Database.EnsureCreatedAsync(); +} + static CustomerResponse ToResponse(Customer c) => new( c.DataSubjectId, c.Name, diff --git a/samples/MinimalApi.Sample/schema/README.md b/samples/MinimalApi.Sample/schema/README.md new file mode 100644 index 0000000..2e7c797 --- /dev/null +++ b/samples/MinimalApi.Sample/schema/README.md @@ -0,0 +1,21 @@ +# MinimalApi.Sample SQLite schema + +The MinimalApi.Sample creates its local SQLite tables automatically on startup so +the routes work immediately. + +These scripts are kept as an explicit reference for the schema created by the +sample. You can also run them manually if you want to inspect or reset the +databases yourself: + +```powershell +sqlite3 sensitiveflow-minimalapi.db ".read schema/sqlite-app.sql" +sqlite3 sensitiveflow-minimalapi-audit.db ".read schema/sqlite-audit.sql" +sqlite3 sensitiveflow-minimalapi-tokens.db ".read schema/sqlite-tokens.sql" +``` + +The app database stores the sample `Customers` table. The audit database stores +`SensitiveFlow_AuditRecords` and `AuditOutboxEntries`. The token database stores +`SensitiveFlow_TokenMappings`. + +SensitiveFlow library packages do not create production schema automatically. In +real applications, prefer EF Core migrations or deployment-owned SQL scripts. diff --git a/samples/MinimalApi.Sample/schema/sqlite-app.sql b/samples/MinimalApi.Sample/schema/sqlite-app.sql new file mode 100644 index 0000000..fb35ea0 --- /dev/null +++ b/samples/MinimalApi.Sample/schema/sqlite-app.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS "Customers" ( + "Id" INTEGER NOT NULL CONSTRAINT "PK_Customers" PRIMARY KEY AUTOINCREMENT, + "DataSubjectId" TEXT NOT NULL, + "Name" TEXT NOT NULL, + "Email" TEXT NOT NULL, + "TaxId" TEXT NOT NULL, + "Phone" TEXT NOT NULL, + "CreatedAt" TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS "IX_Customers_DataSubjectId" + ON "Customers" ("DataSubjectId"); diff --git a/samples/MinimalApi.Sample/schema/sqlite-audit.sql b/samples/MinimalApi.Sample/schema/sqlite-audit.sql new file mode 100644 index 0000000..e1de549 --- /dev/null +++ b/samples/MinimalApi.Sample/schema/sqlite-audit.sql @@ -0,0 +1,44 @@ +CREATE TABLE IF NOT EXISTS "SensitiveFlow_AuditRecords" ( + "Id" INTEGER NOT NULL CONSTRAINT "PK_SensitiveFlow_AuditRecords" PRIMARY KEY AUTOINCREMENT, + "RecordId" TEXT NOT NULL, + "DataSubjectId" TEXT NOT NULL, + "Entity" TEXT NOT NULL, + "Field" TEXT NOT NULL, + "Operation" INTEGER NOT NULL, + "Timestamp" TEXT NOT NULL, + "ActorId" TEXT NULL, + "IpAddressToken" TEXT NULL, + "Details" TEXT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS "IX_SensitiveFlow_AuditRecords_RecordId" + ON "SensitiveFlow_AuditRecords" ("RecordId"); + +CREATE INDEX IF NOT EXISTS "IX_SensitiveFlow_AuditRecords_Timestamp" + ON "SensitiveFlow_AuditRecords" ("Timestamp"); + +CREATE INDEX IF NOT EXISTS "IX_SensitiveFlow_AuditRecords_DataSubjectId_Timestamp" + ON "SensitiveFlow_AuditRecords" ("DataSubjectId", "Timestamp"); + +CREATE TABLE IF NOT EXISTS "AuditOutboxEntries" ( + "Id" TEXT NOT NULL CONSTRAINT "PK_AuditOutboxEntries" PRIMARY KEY, + "AuditRecordId" TEXT NOT NULL, + "Payload" TEXT NOT NULL, + "Attempts" INTEGER NOT NULL DEFAULT 0, + "EnqueuedAt" TEXT NOT NULL, + "LastAttemptAt" TEXT NULL, + "LastError" TEXT NULL, + "IsProcessed" INTEGER NOT NULL DEFAULT 0, + "ProcessedAt" TEXT NULL, + "IsDeadLettered" INTEGER NOT NULL DEFAULT 0, + "DeadLetterReason" TEXT NULL +); + +CREATE INDEX IF NOT EXISTS "IX_AuditOutboxEntries_IsProcessed_IsDeadLettered" + ON "AuditOutboxEntries" ("IsProcessed", "IsDeadLettered"); + +CREATE INDEX IF NOT EXISTS "IX_AuditOutboxEntries_IsDeadLettered" + ON "AuditOutboxEntries" ("IsDeadLettered"); + +CREATE INDEX IF NOT EXISTS "IX_AuditOutboxEntries_EnqueuedAt" + ON "AuditOutboxEntries" ("EnqueuedAt"); diff --git a/samples/MinimalApi.Sample/schema/sqlite-tokens.sql b/samples/MinimalApi.Sample/schema/sqlite-tokens.sql new file mode 100644 index 0000000..936a225 --- /dev/null +++ b/samples/MinimalApi.Sample/schema/sqlite-tokens.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS "SensitiveFlow_TokenMappings" ( + "Id" INTEGER NOT NULL CONSTRAINT "PK_SensitiveFlow_TokenMappings" PRIMARY KEY AUTOINCREMENT, + "Value" TEXT NOT NULL, + "Token" TEXT NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS "IX_SensitiveFlow_TokenMappings_Value" + ON "SensitiveFlow_TokenMappings" ("Value"); + +CREATE INDEX IF NOT EXISTS "IX_SensitiveFlow_TokenMappings_Token" + ON "SensitiveFlow_TokenMappings" ("Token"); diff --git a/samples/QuickStart.Sample/Program.cs b/samples/QuickStart.Sample/Program.cs index caf1697..e98d5f9 100644 --- a/samples/QuickStart.Sample/Program.cs +++ b/samples/QuickStart.Sample/Program.cs @@ -28,19 +28,97 @@ options.EnableHealthChecks(); }); -builder.Services.AddOpenApi(); - var app = builder.Build(); -if (app.Environment.IsDevelopment()) -{ - app.MapOpenApi(); -} - app.UseHttpsRedirection(); app.UseSensitiveFlow(); app.MapHealthChecks("/health"); +app.MapGet("/", () => Results.Content(""" + + + + + + SensitiveFlow QuickStart + + + +
+

SensitiveFlow QuickStart

+
+ Schema required. This sample does not create tables. Create/migrate the app, audit, and token databases before testing writes. +
+
+

Create customer

+ + + + +
+
+

Get customer

+ + + +
+
+

Response

+ 
Ready.
+
+
+ + + +""", "text/html")); + +app.MapGet("/customers", async (AppDbContext db, CancellationToken ct) => +{ + var customers = await db.Customers + .Take(100) + .ToListAsync(ct); + + return Results.Ok(customers + .OrderByDescending(c => c.CreatedAt) + .Take(20)); +}); + app.MapPost("/customers", async (CreateCustomerRequest request, AppDbContext db, CancellationToken ct) => { var customer = new Customer diff --git a/samples/QuickStart.Sample/QuickStart.Sample.csproj b/samples/QuickStart.Sample/QuickStart.Sample.csproj index c5abc11..efd7e1e 100644 --- a/samples/QuickStart.Sample/QuickStart.Sample.csproj +++ b/samples/QuickStart.Sample/QuickStart.Sample.csproj @@ -1,7 +1,7 @@ - net10.0 + net10.0 enable enable false @@ -9,7 +9,6 @@ - diff --git a/samples/WebApi.Sample/Controllers/CustomersController.cs b/samples/WebApi.Sample/Controllers/CustomersController.cs index ab184e1..41f1cbc 100644 --- a/samples/WebApi.Sample/Controllers/CustomersController.cs +++ b/samples/WebApi.Sample/Controllers/CustomersController.cs @@ -41,6 +41,19 @@ public CustomersController( _logger = logger; } + [HttpGet] + public async Task List(CancellationToken ct) + { + var customers = await _db.Customers + .Take(100) + .ToListAsync(ct); + + return Ok(customers + .OrderByDescending(c => c.CreatedAt) + .Take(20) + .Select(ToResponse)); + } + [HttpPost] public async Task Create([FromBody] CreateCustomerRequest request, CancellationToken ct) { @@ -67,8 +80,7 @@ public async Task Create([FromBody] CreateCustomerRequest request [HttpGet("{id}")] public async Task Get(string id, CancellationToken ct) { - var customer = await _db.Customers - .FirstOrDefaultAsync(c => c.DataSubjectId == id, ct); + var customer = await FindCustomerAsync(id, ct); if (customer is null) { @@ -94,15 +106,20 @@ await _auditStore.AppendAsync(new AuditRecord [HttpGet("{id}/audit")] public async Task GetAudit(string id, CancellationToken ct) { - var records = await _auditStore.QueryByDataSubjectAsync(id, cancellationToken: ct); + var customer = await FindCustomerAsync(id, ct); + if (customer is null) + { + return NotFound(); + } + + var records = await _auditStore.QueryByDataSubjectAsync(customer.DataSubjectId, cancellationToken: ct); return Ok(records); } [HttpGet("{id}/json")] public async Task GetWithJsonRedaction(string id, CancellationToken ct) { - var customer = await _db.Customers - .FirstOrDefaultAsync(c => c.DataSubjectId == id, ct); + var customer = await FindCustomerAsync(id, ct); return customer is null ? NotFound() : Ok(customer); } @@ -110,9 +127,7 @@ public async Task GetWithJsonRedaction(string id, CancellationTok [HttpGet("{id}/export")] public async Task Export(string id, CancellationToken ct) { - var customer = await _db.Customers - .AsNoTracking() - .FirstOrDefaultAsync(c => c.DataSubjectId == id, ct); + var customer = await FindCustomerAsync(id, ct, asNoTracking: true); if (customer is null) { @@ -135,8 +150,7 @@ await _auditStore.AppendAsync(new AuditRecord [HttpPost("{id}/erase")] public async Task Erase(string id, CancellationToken ct) { - var customer = await _db.Customers - .FirstOrDefaultAsync(c => c.DataSubjectId == id, ct); + var customer = await FindCustomerAsync(id, ct); if (customer is null) { @@ -211,6 +225,14 @@ public async Task DryRunRetention(CancellationToken ct) c.Name.MaskName(), c.Email.MaskEmail(), c.Phone.MaskPhone()); + + private Task FindCustomerAsync(string id, CancellationToken ct, bool asNoTracking = false) + { + var query = asNoTracking ? _db.Customers.AsNoTracking() : _db.Customers; + return int.TryParse(id, out var numericId) + ? query.FirstOrDefaultAsync(c => c.Id == numericId, ct) + : query.FirstOrDefaultAsync(c => c.DataSubjectId == id, ct); + } } public sealed record CreateCustomerRequest( diff --git a/samples/WebApi.Sample/Program.cs b/samples/WebApi.Sample/Program.cs index 9765fd5..21b4033 100644 --- a/samples/WebApi.Sample/Program.cs +++ b/samples/WebApi.Sample/Program.cs @@ -14,6 +14,7 @@ using SensitiveFlow.EFCore.Interceptors; using SensitiveFlow.Json.Configuration; using SensitiveFlow.Json.Enums; +using SensitiveFlow.TokenStore.EFCore; using WebApi.Sample.Infrastructure; Log.Logger = new LoggerConfiguration() @@ -38,11 +39,11 @@ .WriteTo.File("logs/webapi-sample-.log", rollingInterval: RollingInterval.Day)); var appConnection = builder.Configuration.GetConnectionString("Default") - ?? "Data Source=sensitiveflow-webapi.db"; + ?? SqliteConnectionInContentRoot(builder.Environment.ContentRootPath, "sensitiveflow-webapi.db"); var auditConnection = builder.Configuration.GetConnectionString("Audit") - ?? "Data Source=sensitiveflow-webapi-audit.db"; + ?? SqliteConnectionInContentRoot(builder.Environment.ContentRootPath, "sensitiveflow-webapi-audit.db"); var tokenConnection = builder.Configuration.GetConnectionString("Tokens") - ?? "Data Source=sensitiveflow-webapi-tokens.db"; + ?? SqliteConnectionInContentRoot(builder.Environment.ContentRootPath, "sensitiveflow-webapi-tokens.db"); builder.Services.AddDbContext((sp, options) => options.UseSqlite(appConnection) @@ -109,19 +110,128 @@ .AddConsoleExporter()); builder.Services.AddControllers(); - builder.Services.AddOpenApi(); var app = builder.Build(); - if (app.Environment.IsDevelopment()) - { - app.MapOpenApi(); - } + await InitializeSampleDatabasesAsync(app.Services); app.UseHttpsRedirection(); app.UseSensitiveFlow(); app.UseAuthorization(); app.MapHealthChecks("/health/sensitiveflow"); + app.MapGet("/", () => Results.Content(""" + + + + + + SensitiveFlow Web API Sample + + + +
+

SensitiveFlow Web API Sample

+
+ Sample database. This sample creates its local SQLite tables on startup so the routes work immediately. + Production apps should use EF Core migrations or deployment-owned SQL scripts instead. +
+
+
+

Create customer

+ + + + + +
+
+

Customer workflows

+ + + + + + + +
+
+

Retention

+ + + +
+
+
+

Response

+ 
Ready.
+
+
+ + + +""", "text/html")); app.MapControllers(); app.Run(); @@ -134,3 +244,26 @@ { await Log.CloseAndFlushAsync(); } + +static async Task InitializeSampleDatabasesAsync(IServiceProvider services) +{ + using var scope = services.CreateScope(); + + await scope.ServiceProvider + .GetRequiredService() + .Database + .EnsureCreatedAsync(); + + await using var auditDb = await scope.ServiceProvider + .GetRequiredService>() + .CreateDbContextAsync(); + await auditDb.Database.EnsureCreatedAsync(); + + await using var tokenDb = await scope.ServiceProvider + .GetRequiredService>() + .CreateDbContextAsync(); + await tokenDb.Database.EnsureCreatedAsync(); +} + +static string SqliteConnectionInContentRoot(string contentRoot, string fileName) + => $"Data Source={Path.Combine(contentRoot, fileName)}"; diff --git a/samples/WebApi.Sample/WebApi.Sample.csproj b/samples/WebApi.Sample/WebApi.Sample.csproj index dc7b92d..3a40c98 100644 --- a/samples/WebApi.Sample/WebApi.Sample.csproj +++ b/samples/WebApi.Sample/WebApi.Sample.csproj @@ -9,7 +9,6 @@ - diff --git a/samples/WebApi.Sample/schema/README.md b/samples/WebApi.Sample/schema/README.md new file mode 100644 index 0000000..7053329 --- /dev/null +++ b/samples/WebApi.Sample/schema/README.md @@ -0,0 +1,21 @@ +# WebApi.Sample SQLite schema + +The WebApi.Sample creates its local SQLite tables automatically on startup so +the routes work immediately. + +These scripts are kept as an explicit reference for the schema created by the +sample. You can also run them manually if you want to inspect or reset the +databases yourself: + +```powershell +sqlite3 sensitiveflow-webapi.db ".read schema/sqlite-app.sql" +sqlite3 sensitiveflow-webapi-audit.db ".read schema/sqlite-audit.sql" +sqlite3 sensitiveflow-webapi-tokens.db ".read schema/sqlite-tokens.sql" +``` + +The app database stores the sample `Customers` table. The audit database stores +`SensitiveFlow_AuditRecords` and `AuditOutboxEntries`. The token database stores +`SensitiveFlow_TokenMappings`. + +SensitiveFlow library packages do not create production schema automatically. In +real applications, prefer EF Core migrations or deployment-owned SQL scripts. diff --git a/samples/WebApi.Sample/schema/sqlite-app.sql b/samples/WebApi.Sample/schema/sqlite-app.sql new file mode 100644 index 0000000..fb35ea0 --- /dev/null +++ b/samples/WebApi.Sample/schema/sqlite-app.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS "Customers" ( + "Id" INTEGER NOT NULL CONSTRAINT "PK_Customers" PRIMARY KEY AUTOINCREMENT, + "DataSubjectId" TEXT NOT NULL, + "Name" TEXT NOT NULL, + "Email" TEXT NOT NULL, + "TaxId" TEXT NOT NULL, + "Phone" TEXT NOT NULL, + "CreatedAt" TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS "IX_Customers_DataSubjectId" + ON "Customers" ("DataSubjectId"); diff --git a/samples/WebApi.Sample/schema/sqlite-audit.sql b/samples/WebApi.Sample/schema/sqlite-audit.sql new file mode 100644 index 0000000..e1de549 --- /dev/null +++ b/samples/WebApi.Sample/schema/sqlite-audit.sql @@ -0,0 +1,44 @@ +CREATE TABLE IF NOT EXISTS "SensitiveFlow_AuditRecords" ( + "Id" INTEGER NOT NULL CONSTRAINT "PK_SensitiveFlow_AuditRecords" PRIMARY KEY AUTOINCREMENT, + "RecordId" TEXT NOT NULL, + "DataSubjectId" TEXT NOT NULL, + "Entity" TEXT NOT NULL, + "Field" TEXT NOT NULL, + "Operation" INTEGER NOT NULL, + "Timestamp" TEXT NOT NULL, + "ActorId" TEXT NULL, + "IpAddressToken" TEXT NULL, + "Details" TEXT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS "IX_SensitiveFlow_AuditRecords_RecordId" + ON "SensitiveFlow_AuditRecords" ("RecordId"); + +CREATE INDEX IF NOT EXISTS "IX_SensitiveFlow_AuditRecords_Timestamp" + ON "SensitiveFlow_AuditRecords" ("Timestamp"); + +CREATE INDEX IF NOT EXISTS "IX_SensitiveFlow_AuditRecords_DataSubjectId_Timestamp" + ON "SensitiveFlow_AuditRecords" ("DataSubjectId", "Timestamp"); + +CREATE TABLE IF NOT EXISTS "AuditOutboxEntries" ( + "Id" TEXT NOT NULL CONSTRAINT "PK_AuditOutboxEntries" PRIMARY KEY, + "AuditRecordId" TEXT NOT NULL, + "Payload" TEXT NOT NULL, + "Attempts" INTEGER NOT NULL DEFAULT 0, + "EnqueuedAt" TEXT NOT NULL, + "LastAttemptAt" TEXT NULL, + "LastError" TEXT NULL, + "IsProcessed" INTEGER NOT NULL DEFAULT 0, + "ProcessedAt" TEXT NULL, + "IsDeadLettered" INTEGER NOT NULL DEFAULT 0, + "DeadLetterReason" TEXT NULL +); + +CREATE INDEX IF NOT EXISTS "IX_AuditOutboxEntries_IsProcessed_IsDeadLettered" + ON "AuditOutboxEntries" ("IsProcessed", "IsDeadLettered"); + +CREATE INDEX IF NOT EXISTS "IX_AuditOutboxEntries_IsDeadLettered" + ON "AuditOutboxEntries" ("IsDeadLettered"); + +CREATE INDEX IF NOT EXISTS "IX_AuditOutboxEntries_EnqueuedAt" + ON "AuditOutboxEntries" ("EnqueuedAt"); diff --git a/samples/WebApi.Sample/schema/sqlite-tokens.sql b/samples/WebApi.Sample/schema/sqlite-tokens.sql new file mode 100644 index 0000000..936a225 --- /dev/null +++ b/samples/WebApi.Sample/schema/sqlite-tokens.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS "SensitiveFlow_TokenMappings" ( + "Id" INTEGER NOT NULL CONSTRAINT "PK_SensitiveFlow_TokenMappings" PRIMARY KEY AUTOINCREMENT, + "Value" TEXT NOT NULL, + "Token" TEXT NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS "IX_SensitiveFlow_TokenMappings_Value" + ON "SensitiveFlow_TokenMappings" ("Value"); + +CREATE INDEX IF NOT EXISTS "IX_SensitiveFlow_TokenMappings_Token" + ON "SensitiveFlow_TokenMappings" ("Token"); diff --git a/src/SensitiveFlow.AspNetCore/Middleware/SensitiveFlowAuditMiddleware.cs b/src/SensitiveFlow.AspNetCore/Middleware/SensitiveFlowAuditMiddleware.cs index e807ff0..3920a77 100644 --- a/src/SensitiveFlow.AspNetCore/Middleware/SensitiveFlowAuditMiddleware.cs +++ b/src/SensitiveFlow.AspNetCore/Middleware/SensitiveFlowAuditMiddleware.cs @@ -15,22 +15,19 @@ public sealed class SensitiveFlowAuditMiddleware public const string IpTokenKey = "SensitiveFlow.IpToken"; private readonly RequestDelegate _next; - private readonly IPseudonymizer _pseudonymizer; private readonly SensitiveFlowAspNetCorePipelineDiagnostics? _diagnostics; /// Initializes a new instance of . public SensitiveFlowAuditMiddleware( RequestDelegate next, - IPseudonymizer pseudonymizer, SensitiveFlowAspNetCorePipelineDiagnostics? diagnostics = null) { _next = next; - _pseudonymizer = pseudonymizer; _diagnostics = diagnostics; } /// Pseudonymizes the remote IP and stores the token before passing to the next middleware. - public async Task InvokeAsync(HttpContext context) + public async Task InvokeAsync(HttpContext context, IPseudonymizer pseudonymizer) { if (context.User.Identity?.IsAuthenticated == true) { @@ -40,7 +37,7 @@ public async Task InvokeAsync(HttpContext context) var ip = context.Connection.RemoteIpAddress?.ToString(); if (!string.IsNullOrEmpty(ip)) { - context.Items[IpTokenKey] = await _pseudonymizer.PseudonymizeAsync( + context.Items[IpTokenKey] = await pseudonymizer.PseudonymizeAsync( ip, context.RequestAborted); } diff --git a/src/SensitiveFlow.Audit/Outbox/AuditOutboxDispatcher.cs b/src/SensitiveFlow.Audit/Outbox/AuditOutboxDispatcher.cs index 881dc16..a76f1d4 100644 --- a/src/SensitiveFlow.Audit/Outbox/AuditOutboxDispatcher.cs +++ b/src/SensitiveFlow.Audit/Outbox/AuditOutboxDispatcher.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using SensitiveFlow.Core.Interfaces; @@ -10,19 +11,19 @@ namespace SensitiveFlow.Audit.Outbox; public sealed class AuditOutboxDispatcher : BackgroundService { private readonly IDurableAuditOutbox? _outbox; - private readonly IEnumerable _publishers; + private readonly IServiceScopeFactory _scopeFactory; private readonly AuditOutboxDispatcherOptions _options; private readonly ILogger? _logger; /// Initializes a new instance. public AuditOutboxDispatcher( IDurableAuditOutbox? outbox, - IEnumerable publishers, + IServiceScopeFactory scopeFactory, AuditOutboxDispatcherOptions options, ILogger? logger = null) { _outbox = outbox; - _publishers = publishers ?? []; + _scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory)); _options = options ?? throw new ArgumentNullException(nameof(options)); _logger = logger; } @@ -30,14 +31,39 @@ public AuditOutboxDispatcher( /// protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - if (_outbox is null || !_publishers.Any()) + if (_outbox is null) { return; } while (!stoppingToken.IsCancellationRequested) { - await DispatchOnceAsync(stoppingToken); + try + { + await DispatchOnceAsync(stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + SensitiveFlowAuditDiagnostics.RecordFailed(); + _logger?.LogError( + ex, + "Audit outbox dispatcher failed while polling. Ensure the durable outbox schema exists and the database is reachable."); + + if (_options.SuspendOnInfrastructureFailure) + { + _logger?.LogWarning( + "Audit outbox dispatcher polling was suspended after an infrastructure failure. Restart the application after fixing the schema or database connection."); + break; + } + + await Task.Delay(_options.InfrastructureFailureRetryDelay, stoppingToken); + continue; + } + await Task.Delay(_options.PollInterval, stoppingToken); } } @@ -50,7 +76,8 @@ public async Task DispatchOnceAsync(CancellationToken cancellationToken = defaul return; } - var publishers = _publishers.ToArray(); + using var scope = _scopeFactory.CreateScope(); + var publishers = scope.ServiceProvider.GetServices().ToArray(); if (publishers.Length == 0) { return; diff --git a/src/SensitiveFlow.Audit/Outbox/AuditOutboxDispatcherOptions.cs b/src/SensitiveFlow.Audit/Outbox/AuditOutboxDispatcherOptions.cs index 3d1cf19..5669ffc 100644 --- a/src/SensitiveFlow.Audit/Outbox/AuditOutboxDispatcherOptions.cs +++ b/src/SensitiveFlow.Audit/Outbox/AuditOutboxDispatcherOptions.cs @@ -17,4 +17,18 @@ public sealed class AuditOutboxDispatcherOptions /// Whether entries should be treated as dead-lettered after max attempts. public bool DeadLetterAfterMax { get; set; } = true; + + /// + /// Whether the hosted dispatcher should stop polling after an infrastructure failure. + /// Defaults to so a missing table, unavailable database, or invalid + /// connection string does not crash the application host or flood logs. + /// + public bool SuspendOnInfrastructureFailure { get; set; } = true; + + /// + /// Delay before retrying after an infrastructure failure when + /// is . + /// Defaults to 30 seconds. + /// + public TimeSpan InfrastructureFailureRetryDelay { get; set; } = TimeSpan.FromSeconds(30); } diff --git a/tests/SensitiveFlow.AspNetCore.Tests/SensitiveFlowAuditMiddlewareTests.cs b/tests/SensitiveFlow.AspNetCore.Tests/SensitiveFlowAuditMiddlewareTests.cs index 603f4cf..88c8af4 100644 --- a/tests/SensitiveFlow.AspNetCore.Tests/SensitiveFlowAuditMiddlewareTests.cs +++ b/tests/SensitiveFlow.AspNetCore.Tests/SensitiveFlowAuditMiddlewareTests.cs @@ -27,8 +27,8 @@ public async Task Middleware_WithRemoteIp_StoresTokenInItems() var httpContext = MakeContext("192.168.1.42"); - var middleware = new SensitiveFlowAuditMiddleware(_ => Task.CompletedTask, pseudonymizer); - await middleware.InvokeAsync(httpContext); + var middleware = new SensitiveFlowAuditMiddleware(_ => Task.CompletedTask); + await middleware.InvokeAsync(httpContext, pseudonymizer); httpContext.Items[SensitiveFlowAuditMiddleware.IpTokenKey].Should().Be("token-abc"); } @@ -42,8 +42,8 @@ public async Task Middleware_CallsPseudonymizer_WithRemoteIp() var httpContext = MakeContext("10.0.0.1"); - var middleware = new SensitiveFlowAuditMiddleware(_ => Task.CompletedTask, pseudonymizer); - await middleware.InvokeAsync(httpContext); + var middleware = new SensitiveFlowAuditMiddleware(_ => Task.CompletedTask); + await middleware.InvokeAsync(httpContext, pseudonymizer); await pseudonymizer.Received(1).PseudonymizeAsync("10.0.0.1", Arg.Any()); } @@ -55,8 +55,8 @@ public async Task Middleware_NullRemoteIp_DoesNotStoreToken() var httpContext = MakeContext(null); - var middleware = new SensitiveFlowAuditMiddleware(_ => Task.CompletedTask, pseudonymizer); - await middleware.InvokeAsync(httpContext); + var middleware = new SensitiveFlowAuditMiddleware(_ => Task.CompletedTask); + await middleware.InvokeAsync(httpContext, pseudonymizer); httpContext.Items.Should().NotContainKey(SensitiveFlowAuditMiddleware.IpTokenKey); await pseudonymizer.DidNotReceive().PseudonymizeAsync(Arg.Any(), Arg.Any()); @@ -76,9 +76,9 @@ public async Task Middleware_AlwaysCallsNext() { nextCalled = true; return Task.CompletedTask; - }, pseudonymizer); + }); - await middleware.InvokeAsync(httpContext); + await middleware.InvokeAsync(httpContext, pseudonymizer); nextCalled.Should().BeTrue(); } @@ -94,9 +94,9 @@ public async Task Middleware_NullIp_StillCallsNext() { nextCalled = true; return Task.CompletedTask; - }, pseudonymizer); + }); - await middleware.InvokeAsync(httpContext); + await middleware.InvokeAsync(httpContext, pseudonymizer); nextCalled.Should().BeTrue(); } diff --git a/tests/SensitiveFlow.Audit.Tests/AuditOutboxDispatcherTests.cs b/tests/SensitiveFlow.Audit.Tests/AuditOutboxDispatcherTests.cs index 17be0a6..51f5534 100644 --- a/tests/SensitiveFlow.Audit.Tests/AuditOutboxDispatcherTests.cs +++ b/tests/SensitiveFlow.Audit.Tests/AuditOutboxDispatcherTests.cs @@ -1,4 +1,5 @@ using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using NSubstitute; using SensitiveFlow.Audit.Outbox; @@ -13,24 +14,24 @@ public sealed class AuditOutboxDispatcherTests [Fact] public async Task DispatchOnceAsync_WithNoOutbox_ReturnsImmediately() { - var dispatcher = new AuditOutboxDispatcher( + using var fixture = CreateDispatcher( null, [Substitute.For()], new AuditOutboxDispatcherOptions()); - await dispatcher.DispatchOnceAsync(); + await fixture.Dispatcher.DispatchOnceAsync(); } [Fact] public async Task DispatchOnceAsync_WithNoPublishers_ReturnsImmediately() { var outbox = Substitute.For(); - var dispatcher = new AuditOutboxDispatcher( + using var fixture = CreateDispatcher( outbox, [], new AuditOutboxDispatcherOptions()); - await dispatcher.DispatchOnceAsync(); + await fixture.Dispatcher.DispatchOnceAsync(); await outbox.DidNotReceive().DequeueBatchAsync(Arg.Any(), Arg.Any()); } @@ -45,12 +46,12 @@ public async Task DispatchOnceAsync_DispatchesEntryToPublisher() outbox.DequeueBatchAsync(100, Arg.Any()) .Returns([entry]); - var dispatcher = new AuditOutboxDispatcher( + using var fixture = CreateDispatcher( outbox, [publisher], new AuditOutboxDispatcherOptions { BatchSize = 100 }); - await dispatcher.DispatchOnceAsync(); + await fixture.Dispatcher.DispatchOnceAsync(); await publisher.Received(1).PublishAsync(entry, Arg.Any()); await outbox.Received(1).MarkProcessedAsync( @@ -70,12 +71,12 @@ public async Task DispatchOnceAsync_MarksFailed_WhenPublisherThrows() publisher.PublishAsync(entry, Arg.Any()) .Returns(Task.FromException(new InvalidOperationException("boom"))); - var dispatcher = new AuditOutboxDispatcher( + using var fixture = CreateDispatcher( outbox, [publisher], new AuditOutboxDispatcherOptions { BatchSize = 100 }); - await dispatcher.DispatchOnceAsync(); + await fixture.Dispatcher.DispatchOnceAsync(); await outbox.Received(1).MarkFailedAsync( entry.Id, @@ -93,7 +94,7 @@ public async Task DispatchOnceAsync_DeadLetters_WhenMaxAttemptsReached() outbox.DequeueBatchAsync(100, Arg.Any()) .Returns([entry]); - var dispatcher = new AuditOutboxDispatcher( + using var fixture = CreateDispatcher( outbox, [publisher], new AuditOutboxDispatcherOptions @@ -103,7 +104,7 @@ public async Task DispatchOnceAsync_DeadLetters_WhenMaxAttemptsReached() DeadLetterAfterMax = true, }); - await dispatcher.DispatchOnceAsync(); + await fixture.Dispatcher.DispatchOnceAsync(); await outbox.Received(1).MarkDeadLetteredAsync( entry.Id, @@ -122,7 +123,7 @@ public async Task DispatchOnceAsync_DoesNotDeadLetter_WhenDeadLetterAfterMaxIsFa outbox.DequeueBatchAsync(100, Arg.Any()) .Returns([entry]); - var dispatcher = new AuditOutboxDispatcher( + using var fixture = CreateDispatcher( outbox, [publisher], new AuditOutboxDispatcherOptions @@ -132,7 +133,7 @@ public async Task DispatchOnceAsync_DoesNotDeadLetter_WhenDeadLetterAfterMaxIsFa DeadLetterAfterMax = false, }); - await dispatcher.DispatchOnceAsync(); + await fixture.Dispatcher.DispatchOnceAsync(); await outbox.DidNotReceive().MarkDeadLetteredAsync( Arg.Any(), @@ -152,17 +153,40 @@ public async Task DispatchOnceAsync_CallsMultiplePublishers() outbox.DequeueBatchAsync(100, Arg.Any()) .Returns([entry]); - var dispatcher = new AuditOutboxDispatcher( + using var fixture = CreateDispatcher( outbox, [pub1, pub2], new AuditOutboxDispatcherOptions { BatchSize = 100 }); - await dispatcher.DispatchOnceAsync(); + await fixture.Dispatcher.DispatchOnceAsync(); await pub1.Received(1).PublishAsync(entry, Arg.Any()); await pub2.Received(1).PublishAsync(entry, Arg.Any()); } + [Fact] + public async Task DispatchOnceAsync_ResolvesScopedPublishersInsideDispatchScope() + { + var outbox = Substitute.For(); + var publisher = Substitute.For(); + var entry = SampleEntry(); + + outbox.DequeueBatchAsync(100, Arg.Any()) + .Returns([entry]); + + using var services = new ServiceCollection() + .AddScoped(_ => publisher) + .BuildServiceProvider(validateScopes: true); + var dispatcher = new AuditOutboxDispatcher( + outbox, + services.GetRequiredService(), + new AuditOutboxDispatcherOptions { BatchSize = 100 }); + + await dispatcher.DispatchOnceAsync(); + + await publisher.Received(1).PublishAsync(entry, Arg.Any()); + } + [Fact] public async Task DispatchOnceAsync_StopsOnCancellation() { @@ -172,16 +196,70 @@ public async Task DispatchOnceAsync_StopsOnCancellation() var cts = new CancellationTokenSource(); cts.Cancel(); - var dispatcher = new AuditOutboxDispatcher( + using var fixture = CreateDispatcher( outbox, [publisher], new AuditOutboxDispatcherOptions { PollInterval = TimeSpan.FromMilliseconds(10) }); - await dispatcher.StartAsync(cts.Token); + await fixture.Dispatcher.StartAsync(cts.Token); // Should complete quickly without throwing using var timeoutCts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); - await dispatcher.StopAsync(timeoutCts.Token); + await fixture.Dispatcher.StopAsync(timeoutCts.Token); + } + + [Fact] + public async Task ExecuteAsync_SuspendsWithoutThrowing_WhenOutboxInfrastructureFails() + { + var outbox = Substitute.For(); + var publisher = Substitute.For(); + outbox.DequeueBatchAsync(100, Arg.Any()) + .Returns(Task.FromException>(new InvalidOperationException("missing table"))); + + using var fixture = CreateDispatcher( + outbox, + [publisher], + new AuditOutboxDispatcherOptions + { + BatchSize = 100, + PollInterval = TimeSpan.FromMilliseconds(10), + SuspendOnInfrastructureFailure = true, + }); + + await fixture.Dispatcher.StartAsync(CancellationToken.None); + await Task.Delay(100); + await fixture.Dispatcher.StopAsync(CancellationToken.None); + + await outbox.Received(1).DequeueBatchAsync(100, Arg.Any()); + await publisher.DidNotReceive().PublishAsync(Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task ExecuteAsync_RetriesInfrastructureFailures_WhenSuspensionDisabled() + { + var outbox = Substitute.For(); + var publisher = Substitute.For(); + outbox.DequeueBatchAsync(100, Arg.Any()) + .Returns(Task.FromException>(new InvalidOperationException("database unavailable"))); + + using var fixture = CreateDispatcher( + outbox, + [publisher], + new AuditOutboxDispatcherOptions + { + BatchSize = 100, + PollInterval = TimeSpan.FromMilliseconds(10), + InfrastructureFailureRetryDelay = TimeSpan.FromMilliseconds(10), + SuspendOnInfrastructureFailure = false, + }); + + await fixture.Dispatcher.StartAsync(CancellationToken.None); + await Task.Delay(120); + await fixture.Dispatcher.StopAsync(CancellationToken.None); + + outbox.ReceivedCalls() + .Count(call => call.GetMethodInfo().Name == nameof(IDurableAuditOutbox.DequeueBatchAsync)) + .Should().BeGreaterThanOrEqualTo(2); } [Theory] @@ -199,7 +277,7 @@ public async Task DispatchOnceAsync_AppliesBackoffDelay(BackoffStrategy strategy outbox.DequeueBatchAsync(100, Arg.Any()) .Returns([entry]); - var dispatcher = new AuditOutboxDispatcher( + using var fixture = CreateDispatcher( outbox, [publisher], new AuditOutboxDispatcherOptions @@ -209,7 +287,7 @@ public async Task DispatchOnceAsync_AppliesBackoffDelay(BackoffStrategy strategy }); var sw = System.Diagnostics.Stopwatch.StartNew(); - await dispatcher.DispatchOnceAsync(); + await fixture.Dispatcher.DispatchOnceAsync(); sw.Stop(); if (expectedDelayMs == 0) @@ -222,6 +300,32 @@ public async Task DispatchOnceAsync_AppliesBackoffDelay(BackoffStrategy strategy } } + private static DispatcherFixture CreateDispatcher( + IDurableAuditOutbox? outbox, + IEnumerable publishers, + AuditOutboxDispatcherOptions options) + { + var services = new ServiceCollection(); + foreach (var publisher in publishers) + { + services.AddSingleton(publisher); + } + + var provider = services.BuildServiceProvider(validateScopes: true); + var dispatcher = new AuditOutboxDispatcher( + outbox, + provider.GetRequiredService(), + options); + return new DispatcherFixture(dispatcher, provider); + } + + private sealed record DispatcherFixture( + AuditOutboxDispatcher Dispatcher, + ServiceProvider ServiceProvider) : IDisposable + { + public void Dispose() => ServiceProvider.Dispose(); + } + private static AuditOutboxEntry SampleEntry(int attempts = 0) => new() { Id = Guid.NewGuid(), From 6c374d6463bc9df867e14796e96e536921088961 Mon Sep 17 00:00:00 2001 From: Mateus Silva Date: Mon, 11 May 2026 23:14:42 -0300 Subject: [PATCH 13/23] feat: Enhance error handling and validation in SensitiveFlow - Added SchemaErrorTranslator to translate provider-specific exceptions into a user-friendly SensitiveFlowSchemaNotInitializedException. - Updated EfCoreAuditStore and EfCoreTokenStore to utilize SchemaErrorTranslator for better exception handling during SaveChangesAsync and data retrieval. - Improved AuditSnapshotEntityTypeConfiguration and TokenMappingEntityTypeConfiguration to support optional schema specification. - Enhanced SensitiveFlowValidationServiceCollectionExtensions to throw exceptions on validation errors when FailOnError is enabled. - Updated validation logic in SensitiveFlowConfigurationValidator to report errors instead of warnings for missing required services. - Added comprehensive SQL migration scripts for PostgreSQL, SQL Server, and SQLite to create necessary tables for audit records, outbox entries, token mappings, and snapshots. - Created README documentation for migration scripts to guide users on applying schemas across different database providers. - Updated unit tests to reflect changes in validation behavior and ensure proper error reporting. --- docs/README.md | 1 + docs/database-providers.md | 114 ++++++++++++++++ docs/getting-started.md | 29 ++-- samples/QuickStart.Sample/Program.cs | 27 +++- .../AuditOutboxEntryConfiguration.cs | 42 +++++- .../Entities/AuditOutboxEntryEntity.cs | 18 ++- .../Stores/EfCoreAuditOutbox.cs | 19 +-- .../AuditRecordEntityTypeConfiguration.cs | 33 ++++- .../Stores/EfCoreAuditStore.cs | 19 ++- .../AuditSnapshotEntityTypeConfiguration.cs | 33 ++++- .../Exceptions/SchemaErrorTranslator.cs | 129 ++++++++++++++++++ ...sitiveFlowSchemaNotInitializedException.cs | 50 +++++++ ...owValidationServiceCollectionExtensions.cs | 40 +++++- .../SensitiveFlowConfigurationValidator.cs | 16 ++- .../SensitiveFlowValidationOptions.cs | 8 +- .../TokenMappingEntityTypeConfiguration.cs | 33 ++++- .../Stores/EfCoreTokenStore.cs | 33 +++-- .../SensitiveFlowValidationTests.cs | 59 ++++++-- tools/migrations/README.md | 68 +++++++++ tools/migrations/postgres/01-audit.sql | 24 ++++ tools/migrations/postgres/02-outbox.sql | 28 ++++ tools/migrations/postgres/03-tokens.sql | 13 ++ tools/migrations/postgres/04-snapshots.sql | 23 ++++ tools/migrations/sqlite/01-audit.sql | 24 ++++ tools/migrations/sqlite/02-outbox.sql | 26 ++++ tools/migrations/sqlite/03-tokens.sql | 13 ++ tools/migrations/sqlite/04-snapshots.sql | 23 ++++ tools/migrations/sqlserver/01-audit.sql | 34 +++++ tools/migrations/sqlserver/02-outbox.sql | 38 ++++++ tools/migrations/sqlserver/03-tokens.sql | 21 +++ tools/migrations/sqlserver/04-snapshots.sql | 33 +++++ 31 files changed, 992 insertions(+), 79 deletions(-) create mode 100644 docs/database-providers.md create mode 100644 src/SensitiveFlow.Core/Exceptions/SchemaErrorTranslator.cs create mode 100644 src/SensitiveFlow.Core/Exceptions/SensitiveFlowSchemaNotInitializedException.cs create mode 100644 tools/migrations/README.md create mode 100644 tools/migrations/postgres/01-audit.sql create mode 100644 tools/migrations/postgres/02-outbox.sql create mode 100644 tools/migrations/postgres/03-tokens.sql create mode 100644 tools/migrations/postgres/04-snapshots.sql create mode 100644 tools/migrations/sqlite/01-audit.sql create mode 100644 tools/migrations/sqlite/02-outbox.sql create mode 100644 tools/migrations/sqlite/03-tokens.sql create mode 100644 tools/migrations/sqlite/04-snapshots.sql create mode 100644 tools/migrations/sqlserver/01-audit.sql create mode 100644 tools/migrations/sqlserver/02-outbox.sql create mode 100644 tools/migrations/sqlserver/03-tokens.sql create mode 100644 tools/migrations/sqlserver/04-snapshots.sql diff --git a/docs/README.md b/docs/README.md index e571164..b57c282 100644 --- a/docs/README.md +++ b/docs/README.md @@ -12,6 +12,7 @@ Use this page as the documentation index. The files are grouped by task so contr - [Audit](audit.md): `IAuditStore`, batching, retry, buffering, snapshots, and audit retention concepts. - [EF Core](efcore.md): `SaveChanges` interceptor and entity requirements. +- [Database providers](database-providers.md): support matrix, schema configuration, and provider-specific notes (SQLite/SQL Server/Postgres). - [ASP.NET Core](aspnetcore.md): request audit context and IP pseudonymization middleware. - [JSON redaction](json.md): `System.Text.Json` output protection. - [Logging](logging.md): `ILogger` redaction provider and limits. diff --git a/docs/database-providers.md b/docs/database-providers.md new file mode 100644 index 0000000..7a21391 --- /dev/null +++ b/docs/database-providers.md @@ -0,0 +1,114 @@ +# Database providers + +SensitiveFlow's EF Core packages are **provider-agnostic**: the library never calls +`UseSqlServer`, `UseSqlite`, or `UseNpgsql` on your behalf. You pick the provider in +your own `optionsBuilder` and SensitiveFlow uses whatever you wire in. + +This page documents what works out of the box, and the small caveats per provider. + +## Support matrix + +| Feature | SQLite | SQL Server | PostgreSQL | MySQL / MariaDB | Oracle | +|----------------------------------------|--------|------------|------------|-----------------|--------| +| `SensitiveFlow.Audit.EFCore` | ✅ | ✅ | ✅ | ✅ (1) | ✅ (1) | +| `SensitiveFlow.TokenStore.EFCore` | ✅ | ✅ | ✅ | ✅ (1) | ✅ (1) | +| `SensitiveFlow.Audit.EFCore.Outbox` | ✅ | ✅ | ✅ | ✅ (1) | ✅ (1) | +| `SensitiveFlow.Audit.Snapshots.EFCore` | ✅ | ✅ | ✅ | ✅ (1) | ✅ (1) | +| Schemas (`ToTable(name, "schema")`) | ❌ (2) | ✅ | ✅ | ❌ | ✅ | +| `EnsureCreatedAsync` for samples | ✅ | ✅ | ✅ | ✅ | ✅ | +| `ExecuteDeleteAsync` (retention) | ✅ | ✅ | ✅ | ✅ | ✅ | + +1. Pass an EF Core provider package (e.g. `Pomelo.EntityFrameworkCore.MySql`, + `Oracle.EntityFrameworkCore`) and call `UseMySql(...)` / `UseOracle(...)` in + your `optionsBuilder`. Nothing in SensitiveFlow is SQL Server-specific. +2. SQLite does not implement schemas. The default `ToTable(...)` calls in + SensitiveFlow's configurations omit the schema for this reason; if you opt + into a schema for SQL Server / Postgres, **do not pass one for SQLite**. + +## Ordering and timestamps + +The outbox stores both `EnqueuedAt` (`DateTimeOffset`) and `EnqueuedAtTicks` +(`long`). `EnqueuedAtTicks` exists because some providers (notably SQLite) cannot +ORDER BY `DateTimeOffset` server-side. Sorting the outbox queue uses +`EnqueuedAtTicks`, which is a plain integer and works identically on every +backend. + +If you maintain your own EF Core migrations, make sure both columns and the +matching index (`IX_AuditOutboxEntries_EnqueuedAtTicks`) are part of them. + +## Configuring custom table names and schemas + +Every entity configuration accepts both a table name and an optional schema: + +```csharp +modelBuilder.ApplyConfiguration(new AuditRecordEntityTypeConfiguration( + tableName: "MyAuditTrail", + schema: "compliance")); // null on SQLite + +modelBuilder.ApplyConfiguration(new TokenMappingEntityTypeConfiguration( + tableName: "MyTokenStore")); // schema defaults to null + +modelBuilder.ApplyConfiguration(new AuditOutboxEntryConfiguration( + tableName: "AuditOutbox", + schema: "sensitiveflow")); +``` + +If you stick with the defaults, you can omit the call entirely — SensitiveFlow's +`AuditDbContext` and `TokenDbContext` register the right configurations for you. + +## SQL Server + +- Install `Microsoft.EntityFrameworkCore.SqlServer` in the project that owns the + `DbContext`. +- The provided `tools/migrations/sqlserver/*.sql` scripts deploy into the + default `dbo` schema. Create a dedicated schema first (e.g. + `CREATE SCHEMA sensitiveflow;`) and update the configurations if you prefer + isolation. +- `NVARCHAR(MAX)` is used for unbounded JSON payloads. Index pages have a 1700-byte + limit; the SensitiveFlow indexes only target bounded columns. + +## PostgreSQL + +- Install `Npgsql.EntityFrameworkCore.PostgreSQL`. +- Default deployment uses the `public` schema. Use the `schema:` parameter or + `npgsql.SetPostgresExtension("...")` for custom schemas. +- The provided scripts use `TIMESTAMPTZ` for outbox timestamps so they round-trip + cleanly through `DateTimeOffset`. + +## SQLite + +- Install `Microsoft.EntityFrameworkCore.Sqlite`. +- **Never pass a non-null schema** — SQLite ignores schemas and EF Core logs a + warning. The SensitiveFlow defaults already omit the schema. +- Use SQLite only for samples, tests, or single-process apps. The library's + outbox polling is safe on SQLite, but SQLite's writer lock will bottleneck + high-throughput workloads. + +## MySQL / MariaDB / Oracle + +These providers are supported transitively (nothing in the library targets a +specific dialect), but they are **not part of the test matrix**. If you adopt +SensitiveFlow on one of them, please: + +1. Run `samples/WebApi.Sample` or `samples/MinimalApi.Sample` against your + provider with `EnsureCreatedAsync()` to verify model compilation. +2. Open an issue if any DDL fails — we will accept a `tools/migrations//` + contribution. + +## Schema-not-initialized errors + +When the runtime cannot find a SensitiveFlow table, the EF Core stores throw +`SensitiveFlowSchemaNotInitializedException` (code `SF-SCHEMA-001`). The message +includes the missing table name (when detectable) and points back to this page. + +To reproduce the friendly error: delete the database file and run the sample +without `EnsureCreatedAsync()`. + +## Picking the right entry point + +| You want to ... | Use ... | +|----------------------------------------------|----------------------------------------------------------| +| Production deployment | EF Core migrations **or** `tools/migrations//` | +| Try the library locally in 30 seconds | `EnsureCreatedAsync()` in `samples/*` | +| Isolate audit/token state per environment | Separate `DbContext` + connection string per store | +| Run the outbox on Postgres + dedicated schema| Pass `schema: "sensitiveflow"` to the configuration | diff --git a/docs/getting-started.md b/docs/getting-started.md index c123aa9..cf8117a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -128,17 +128,22 @@ to work and are documented in the package reference for teams that need them. SensitiveFlow does not create tables automatically. The app tables and the SensitiveFlow audit/token/outbox tables must exist before the first write. -Use EF Core migrations, checked-in SQL scripts, or your deployment tooling to -create schema in every environment. This is true for local development too, so -local behavior matches production and missing schema fails early instead of being -hidden by `EnsureCreated`. - -The audit outbox dispatcher is defensive around missing schema: if polling fails -because the durable outbox table is absent or the database is unavailable, it logs -the infrastructure failure and suspends polling by default instead of stopping the +Three options, in order of preference: + +1. **EF Core migrations** owned by your app — best for teams already using EF Core. +2. **Idempotent SQL scripts** shipped in [`tools/migrations/`](../tools/migrations/) — one folder per supported provider (`sqlite`, `sqlserver`, `postgres`). See [Database providers](database-providers.md) for the full matrix and provider-specific notes. +3. **`EnsureCreatedAsync()`** — fine for samples and tests, **not for production**: it skips migrations entirely. + +If the schema is missing at runtime, SensitiveFlow now throws +`SensitiveFlowSchemaNotInitializedException` (code `SF-SCHEMA-001`) instead of +leaking a raw provider error. The message points you back to the migration scripts. + +The audit outbox dispatcher is also defensive: if polling fails because the +durable outbox table is absent or the database is unavailable, it logs the +infrastructure failure and suspends polling by default instead of stopping the application host. Apply the schema and restart the app. -The repository samples are easier on purpose: `samples/WebApi.Sample` creates its -local SQLite schema on startup so you can try the UI and routes immediately. Do -not copy that startup behavior into production apps; use migrations or deployment -scripts for real databases. +The repository samples (`QuickStart`, `MinimalApi`, `WebApi`) call +`EnsureCreatedAsync()` on startup so you can try them immediately. **Do not copy +that pattern into production apps** — use migrations or the scripts in +`tools/migrations/`. diff --git a/samples/QuickStart.Sample/Program.cs b/samples/QuickStart.Sample/Program.cs index e98d5f9..b01c1b9 100644 --- a/samples/QuickStart.Sample/Program.cs +++ b/samples/QuickStart.Sample/Program.cs @@ -1,7 +1,9 @@ using Microsoft.EntityFrameworkCore; using QuickStart.Sample; using SensitiveFlow.AspNetCore.EFCore.Extensions; +using SensitiveFlow.Audit.EFCore; using SensitiveFlow.Core.Profiles; +using SensitiveFlow.TokenStore.EFCore; var builder = WebApplication.CreateBuilder(args); @@ -30,6 +32,8 @@ var app = builder.Build(); +await InitializeSampleDatabasesAsync(app.Services); + app.UseHttpsRedirection(); app.UseSensitiveFlow(); app.MapHealthChecks("/health"); @@ -56,7 +60,8 @@

SensitiveFlow QuickStart

- Schema required. This sample does not create tables. Create/migrate the app, audit, and token databases before testing writes. + Sample database. This sample creates its local SQLite tables on startup so the routes work immediately. + Production apps should use EF Core migrations or deployment-owned SQL scripts instead.

Create customer

@@ -144,4 +149,24 @@ await show(await fetch('/customers', { app.Run(); +static async Task InitializeSampleDatabasesAsync(IServiceProvider services) +{ + using var scope = services.CreateScope(); + + await scope.ServiceProvider + .GetRequiredService() + .Database + .EnsureCreatedAsync(); + + await using var auditDb = await scope.ServiceProvider + .GetRequiredService>() + .CreateDbContextAsync(); + await auditDb.Database.EnsureCreatedAsync(); + + await using var tokenDb = await scope.ServiceProvider + .GetRequiredService>() + .CreateDbContextAsync(); + await tokenDb.Database.EnsureCreatedAsync(); +} + public sealed record CreateCustomerRequest(string Name, string Email, string TaxId); diff --git a/src/SensitiveFlow.Audit.EFCore.Outbox/Configuration/AuditOutboxEntryConfiguration.cs b/src/SensitiveFlow.Audit.EFCore.Outbox/Configuration/AuditOutboxEntryConfiguration.cs index 3805cdf..cb39cdc 100644 --- a/src/SensitiveFlow.Audit.EFCore.Outbox/Configuration/AuditOutboxEntryConfiguration.cs +++ b/src/SensitiveFlow.Audit.EFCore.Outbox/Configuration/AuditOutboxEntryConfiguration.cs @@ -7,12 +7,46 @@ namespace SensitiveFlow.Audit.EFCore.Outbox.Configuration; /// /// EF Core model configuration for . /// +/// +/// Default table name is AuditOutboxEntries with no schema, ensuring compatibility +/// with providers that do not support schemas (e.g. SQLite). Override the table name or +/// schema via the constructor if you need a different layout. +/// public sealed class AuditOutboxEntryConfiguration : IEntityTypeConfiguration { + /// Default table name when none is specified. + public const string DefaultTableName = "AuditOutboxEntries"; + + private readonly string _tableName; + private readonly string? _schema; + + /// Creates a configuration with the default table name and no schema. + public AuditOutboxEntryConfiguration() + : this(DefaultTableName, null) + { + } + + /// Creates a configuration with a custom table name and optional schema. + /// Table name. Defaults to . + /// Optional schema. Leave null for providers without schemas (e.g. SQLite). + public AuditOutboxEntryConfiguration(string tableName, string? schema = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(tableName); + _tableName = tableName; + _schema = string.IsNullOrWhiteSpace(schema) ? null : schema; + } + /// public void Configure(EntityTypeBuilder builder) { - builder.ToTable("AuditOutboxEntries", "sensitiveflow"); + if (_schema is null) + { + builder.ToTable(_tableName); + } + else + { + builder.ToTable(_tableName, _schema); + } builder.HasKey(e => e.Id); @@ -32,6 +66,10 @@ public void Configure(EntityTypeBuilder builder) builder.Property(e => e.EnqueuedAt) .IsRequired(); + // Provider-agnostic ordering column: ticks (long) sorts identically on every backend. + builder.Property(e => e.EnqueuedAtTicks) + .IsRequired(); + builder.Property(e => e.LastAttemptAt); builder.Property(e => e.LastError) @@ -51,6 +89,6 @@ public void Configure(EntityTypeBuilder builder) // Indexes for querying pending/dead-lettered entries builder.HasIndex(e => new { e.IsProcessed, e.IsDeadLettered }); builder.HasIndex(e => e.IsDeadLettered); - builder.HasIndex(e => e.EnqueuedAt); + builder.HasIndex(e => e.EnqueuedAtTicks); } } diff --git a/src/SensitiveFlow.Audit.EFCore.Outbox/Entities/AuditOutboxEntryEntity.cs b/src/SensitiveFlow.Audit.EFCore.Outbox/Entities/AuditOutboxEntryEntity.cs index f96e52a..dbe1bf5 100644 --- a/src/SensitiveFlow.Audit.EFCore.Outbox/Entities/AuditOutboxEntryEntity.cs +++ b/src/SensitiveFlow.Audit.EFCore.Outbox/Entities/AuditOutboxEntryEntity.cs @@ -6,6 +6,8 @@ namespace SensitiveFlow.Audit.EFCore.Outbox.Entities; ///
public sealed class AuditOutboxEntryEntity { + private DateTimeOffset _enqueuedAt = DateTimeOffset.UtcNow; + /// Unique stable identifier for this outbox entry. public Guid Id { get; set; } = Guid.NewGuid(); @@ -19,7 +21,21 @@ public sealed class AuditOutboxEntryEntity public int Attempts { get; set; } /// When the entry was enqueued (UTC). - public DateTimeOffset EnqueuedAt { get; set; } = DateTimeOffset.UtcNow; + public DateTimeOffset EnqueuedAt + { + get => _enqueuedAt; + set + { + _enqueuedAt = value; + EnqueuedAtTicks = value.UtcTicks; + } + } + + /// + /// Mirrors as ticks (UTC) to enable provider-agnostic ORDER BY + /// (some providers, e.g. SQLite, cannot sort server-side). + /// + public long EnqueuedAtTicks { get; set; } = DateTimeOffset.UtcNow.UtcTicks; /// When the last delivery attempt was made (UTC), or null if never attempted. public DateTimeOffset? LastAttemptAt { get; set; } diff --git a/src/SensitiveFlow.Audit.EFCore.Outbox/Stores/EfCoreAuditOutbox.cs b/src/SensitiveFlow.Audit.EFCore.Outbox/Stores/EfCoreAuditOutbox.cs index 8bc16cf..397648c 100644 --- a/src/SensitiveFlow.Audit.EFCore.Outbox/Stores/EfCoreAuditOutbox.cs +++ b/src/SensitiveFlow.Audit.EFCore.Outbox/Stores/EfCoreAuditOutbox.cs @@ -49,15 +49,12 @@ public async Task> DequeueBatchAsync(int max, Ca await using var ctx = await _factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); var pending = await ctx.Set() .Where(e => !e.IsProcessed && !e.IsDeadLettered) + .OrderBy(e => e.EnqueuedAtTicks) + .Take(max) .ToListAsync(cancellationToken) .ConfigureAwait(false); - // Order by EnqueuedAt on client side (SQLite doesn't support DateTimeOffset in ORDER BY) - return pending - .OrderBy(e => e.EnqueuedAt) - .Take(max) - .Select(ToOutboxEntry) - .ToList(); + return pending.Select(ToOutboxEntry).ToList(); } /// @@ -135,15 +132,13 @@ public async Task> GetDeadLetteredAsync(int skip await using var ctx = await _factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); var deadLettered = await ctx.Set() .Where(e => e.IsDeadLettered) + .OrderByDescending(e => e.EnqueuedAtTicks) + .Skip(skip) + .Take(take) .ToListAsync(cancellationToken) .ConfigureAwait(false); - return deadLettered - .OrderByDescending(e => e.EnqueuedAt) - .Skip(skip) - .Take(take) - .Select(ToOutboxEntry) - .ToList(); + return deadLettered.Select(ToOutboxEntry).ToList(); } private static AuditOutboxEntry ToOutboxEntry(AuditOutboxEntryEntity entity) diff --git a/src/SensitiveFlow.Audit.EFCore/Configuration/AuditRecordEntityTypeConfiguration.cs b/src/SensitiveFlow.Audit.EFCore/Configuration/AuditRecordEntityTypeConfiguration.cs index ee19ed7..08939e9 100644 --- a/src/SensitiveFlow.Audit.EFCore/Configuration/AuditRecordEntityTypeConfiguration.cs +++ b/src/SensitiveFlow.Audit.EFCore/Configuration/AuditRecordEntityTypeConfiguration.cs @@ -12,22 +12,41 @@ namespace SensitiveFlow.Audit.EFCore.Configuration; /// public sealed class AuditRecordEntityTypeConfiguration : IEntityTypeConfiguration { + /// Default table name when none is specified. + public const string DefaultTableName = "SensitiveFlow_AuditRecords"; + private readonly string _tableName; + private readonly string? _schema; - /// Initializes the configuration with the given table name. - public AuditRecordEntityTypeConfiguration(string tableName = "SensitiveFlow_AuditRecords") + /// Initializes the configuration with the given table name and no schema. + public AuditRecordEntityTypeConfiguration(string tableName = DefaultTableName) + : this(tableName, null) { - if (string.IsNullOrWhiteSpace(tableName)) - { - throw new ArgumentException("Table name must not be empty.", nameof(tableName)); - } + } + + /// + /// Initializes the configuration with a custom table name and optional schema. + /// + /// Table name. Defaults to . + /// Optional schema. Leave null for providers without schemas (e.g. SQLite). + public AuditRecordEntityTypeConfiguration(string tableName, string? schema) + { + ArgumentException.ThrowIfNullOrWhiteSpace(tableName); _tableName = tableName; + _schema = string.IsNullOrWhiteSpace(schema) ? null : schema; } /// public void Configure(EntityTypeBuilder builder) { - builder.ToTable(_tableName); + if (_schema is null) + { + builder.ToTable(_tableName); + } + else + { + builder.ToTable(_tableName, _schema); + } builder.HasKey(e => e.Id); builder.Property(e => e.Id).ValueGeneratedOnAdd(); diff --git a/src/SensitiveFlow.Audit.EFCore/Stores/EfCoreAuditStore.cs b/src/SensitiveFlow.Audit.EFCore/Stores/EfCoreAuditStore.cs index 17bce46..5991e97 100644 --- a/src/SensitiveFlow.Audit.EFCore/Stores/EfCoreAuditStore.cs +++ b/src/SensitiveFlow.Audit.EFCore/Stores/EfCoreAuditStore.cs @@ -1,5 +1,6 @@ using Microsoft.EntityFrameworkCore; using SensitiveFlow.Audit.EFCore.Entities; +using SensitiveFlow.Core.Exceptions; using SensitiveFlow.Core.Interfaces; using SensitiveFlow.Core.Models; @@ -45,7 +46,14 @@ public async Task AppendAsync(AuditRecord record, CancellationToken cancellation await using var ctx = await _factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); var set = _setSelector(ctx); set.Add(AuditRecordEntity.FromRecord(record)); - await ctx.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + try + { + await ctx.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + throw SchemaErrorTranslator.Translate(ex, typeof(TContext).Name); + } } /// @@ -60,7 +68,14 @@ public async Task AppendRangeAsync(IReadOnlyCollection records, Can await using var ctx = await _factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); var set = _setSelector(ctx); set.AddRange(records.Select(AuditRecordEntity.FromRecord)); - await ctx.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + try + { + await ctx.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + throw SchemaErrorTranslator.Translate(ex, typeof(TContext).Name); + } } /// diff --git a/src/SensitiveFlow.Audit.Snapshots.EFCore/Configuration/AuditSnapshotEntityTypeConfiguration.cs b/src/SensitiveFlow.Audit.Snapshots.EFCore/Configuration/AuditSnapshotEntityTypeConfiguration.cs index b6c9560..4a65a48 100644 --- a/src/SensitiveFlow.Audit.Snapshots.EFCore/Configuration/AuditSnapshotEntityTypeConfiguration.cs +++ b/src/SensitiveFlow.Audit.Snapshots.EFCore/Configuration/AuditSnapshotEntityTypeConfiguration.cs @@ -10,22 +10,41 @@ namespace SensitiveFlow.Audit.Snapshots.EFCore.Configuration; /// public sealed class AuditSnapshotEntityTypeConfiguration : IEntityTypeConfiguration { + /// Default table name when none is specified. + public const string DefaultTableName = "SensitiveFlow_AuditSnapshots"; + private readonly string _tableName; + private readonly string? _schema; - /// Initializes the configuration with the given table name. - public AuditSnapshotEntityTypeConfiguration(string tableName = "SensitiveFlow_AuditSnapshots") + /// Initializes the configuration with the given table name and no schema. + public AuditSnapshotEntityTypeConfiguration(string tableName = DefaultTableName) + : this(tableName, null) { - if (string.IsNullOrWhiteSpace(tableName)) - { - throw new ArgumentException("Table name must not be empty.", nameof(tableName)); - } + } + + /// + /// Initializes the configuration with a custom table name and optional schema. + /// + /// Table name. Defaults to . + /// Optional schema. Leave null for providers without schemas (e.g. SQLite). + public AuditSnapshotEntityTypeConfiguration(string tableName, string? schema) + { + ArgumentException.ThrowIfNullOrWhiteSpace(tableName); _tableName = tableName; + _schema = string.IsNullOrWhiteSpace(schema) ? null : schema; } /// public void Configure(EntityTypeBuilder builder) { - builder.ToTable(_tableName); + if (_schema is null) + { + builder.ToTable(_tableName); + } + else + { + builder.ToTable(_tableName, _schema); + } builder.HasKey(e => e.Id); builder.Property(e => e.Id).ValueGeneratedOnAdd(); diff --git a/src/SensitiveFlow.Core/Exceptions/SchemaErrorTranslator.cs b/src/SensitiveFlow.Core/Exceptions/SchemaErrorTranslator.cs new file mode 100644 index 0000000..798b47b --- /dev/null +++ b/src/SensitiveFlow.Core/Exceptions/SchemaErrorTranslator.cs @@ -0,0 +1,129 @@ +namespace SensitiveFlow.Core.Exceptions; + +/// +/// Translates provider-specific "table does not exist" exceptions into a friendly +/// with remediation guidance. +/// +/// +/// Cross-provider detection is best-effort and matches well-known message fragments: +/// +/// SQLite: no such table +/// SQL Server: Invalid object name +/// PostgreSQL: relation + does not exist +/// MySQL/MariaDB: doesn't exist +/// Oracle: ORA-00942 +/// +/// If no match is found, the original exception is returned unchanged. +/// +public static class SchemaErrorTranslator +{ + /// + /// Returns a wrapping + /// when the underlying message indicates the table + /// does not exist. Otherwise returns unchanged so the + /// caller can throw; the original. + /// + public static Exception Translate(Exception exception, string contextName) + { + ArgumentNullException.ThrowIfNull(exception); + + if (TryGetMissingTable(exception, out var tableName)) + { + return new SensitiveFlowSchemaNotInitializedException(tableName, contextName, exception); + } + + return exception; + } + + private static bool TryGetMissingTable(Exception exception, out string? tableName) + { + for (var current = exception; current is not null; current = current.InnerException) + { + var msg = current.Message; + if (string.IsNullOrEmpty(msg)) + { + continue; + } + + // SQLite: "SQLite Error 1: 'no such table: SensitiveFlow_TokenMappings'." + var idx = msg.IndexOf("no such table:", StringComparison.OrdinalIgnoreCase); + if (idx >= 0) + { + tableName = ExtractAfter(msg, idx + "no such table:".Length); + return true; + } + + // SQL Server: "Invalid object name 'SensitiveFlow_TokenMappings'." + idx = msg.IndexOf("Invalid object name", StringComparison.OrdinalIgnoreCase); + if (idx >= 0) + { + tableName = ExtractQuoted(msg, idx); + return true; + } + + // PostgreSQL: "relation \"sensitiveflow_tokenmappings\" does not exist" + if (msg.Contains("does not exist", StringComparison.OrdinalIgnoreCase) && + msg.Contains("relation", StringComparison.OrdinalIgnoreCase)) + { + tableName = ExtractQuoted(msg, 0); + return true; + } + + // MySQL/MariaDB: "Table 'app.sensitiveflow_tokenmappings' doesn't exist" + if (msg.Contains("doesn't exist", StringComparison.OrdinalIgnoreCase) && + msg.Contains("Table", StringComparison.OrdinalIgnoreCase)) + { + tableName = ExtractQuoted(msg, 0); + return true; + } + + // Oracle: "ORA-00942: table or view does not exist" + if (msg.Contains("ORA-00942", StringComparison.OrdinalIgnoreCase)) + { + tableName = null; + return true; + } + } + + tableName = null; + return false; + } + + private static string? ExtractAfter(string message, int start) + { + var slice = message[start..].TrimStart(); + var end = slice.IndexOfAny(new[] { '\'', '"', '.', '\n', '\r' }); + return end > 0 ? slice[..end].Trim() : slice.Trim(); + } + + private static string? ExtractQuoted(string message, int start) + { + var slice = message[start..]; + var openSingle = slice.IndexOf('\''); + var openDouble = slice.IndexOf('"'); + int open; + char quote; + if (openSingle == -1 && openDouble == -1) + { + return null; + } + if (openSingle == -1) + { + open = openDouble; + quote = '"'; + } + else if (openDouble == -1 || openSingle < openDouble) + { + open = openSingle; + quote = '\''; + } + else + { + open = openDouble; + quote = '"'; + } + + var close = slice.IndexOf(quote, open + 1); + return close > open ? slice[(open + 1)..close] : null; + } +} diff --git a/src/SensitiveFlow.Core/Exceptions/SensitiveFlowSchemaNotInitializedException.cs b/src/SensitiveFlow.Core/Exceptions/SensitiveFlowSchemaNotInitializedException.cs new file mode 100644 index 0000000..eaa489f --- /dev/null +++ b/src/SensitiveFlow.Core/Exceptions/SensitiveFlowSchemaNotInitializedException.cs @@ -0,0 +1,50 @@ +namespace SensitiveFlow.Core.Exceptions; + +/// +/// Thrown when a SensitiveFlow store fails to access its persistence schema because +/// the underlying table is missing. +/// +/// +/// This wraps the provider-specific exception (e.g. SqliteException, +/// SqlException, NpgsqlException) with an actionable message pointing the +/// caller at EnsureCreatedAsync() (for samples) or dotnet ef database update +/// (for production deployments). +/// +public sealed class SensitiveFlowSchemaNotInitializedException : SensitiveFlowException +{ + /// Machine-readable error code. + public const string ErrorCode = "SF-SCHEMA-001"; + + /// Initializes a new instance. + /// Name of the missing table (best-effort, may be null). + /// Type name of the EF Core DbContext that owns the table. + /// The original provider-specific exception. + public SensitiveFlowSchemaNotInitializedException( + string? tableName, + string contextName, + Exception innerException) + : base(BuildMessage(tableName, contextName), ErrorCode, innerException) + { + TableName = tableName; + ContextName = contextName; + } + + /// Name of the missing table, when detectable. + public string? TableName { get; } + + /// The EF Core DbContext type that owns the missing table. + public string ContextName { get; } + + private static string BuildMessage(string? tableName, string contextName) + { + var tableHint = string.IsNullOrWhiteSpace(tableName) + ? "the required SensitiveFlow table" + : $"table '{tableName}'"; + + return $"SensitiveFlow could not access {tableHint} on '{contextName}': the table does not exist. " + + "Create the schema before starting the app. For samples, call " + + "'await db.Database.EnsureCreatedAsync()' on each SensitiveFlow context. " + + "For production, run EF Core migrations ('dotnet ef database update') or apply the " + + "SQL scripts shipped in 'tools/migrations//'."; + } +} diff --git a/src/SensitiveFlow.Diagnostics/Extensions/SensitiveFlowValidationServiceCollectionExtensions.cs b/src/SensitiveFlow.Diagnostics/Extensions/SensitiveFlowValidationServiceCollectionExtensions.cs index f7b865d..65a90f2 100644 --- a/src/SensitiveFlow.Diagnostics/Extensions/SensitiveFlowValidationServiceCollectionExtensions.cs +++ b/src/SensitiveFlow.Diagnostics/Extensions/SensitiveFlowValidationServiceCollectionExtensions.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.DependencyInjection; +using SensitiveFlow.Core.Exceptions; using SensitiveFlow.Core.Profiles; using SensitiveFlow.Diagnostics.Validation; @@ -38,19 +39,48 @@ public static IServiceCollection AddSensitiveFlowValidation( return services; } - /// Runs SensitiveFlow startup validation from a built service provider. + /// + /// Runs SensitiveFlow startup validation from a built service provider. + /// + /// + /// When the resolved is true + /// (the default), this throws on any + /// Error diagnostic. Warnings never throw — they are returned via the report so + /// the caller can log them. + /// public static SensitiveFlowConfigurationReport ValidateSensitiveFlow(this IServiceProvider services) { ArgumentNullException.ThrowIfNull(services); - var validator = services.GetService() - ?? new SensitiveFlowConfigurationValidator(new SensitiveFlowValidationOptions + var options = services.GetService() + ?? new SensitiveFlowValidationOptions { RequireAuditStore = true, RequireTokenStore = false, RequireJsonRedaction = false, RequireRetention = false, - }); + }; + + var validator = services.GetService() + ?? new SensitiveFlowConfigurationValidator(options); + + var report = validator.Validate(services); + + if (options.FailOnError && !report.IsValid) + { + var errors = report.Diagnostics + .Where(d => d.Severity == SensitiveFlowDiagnosticSeverity.Error) + .Select(d => $" [{d.Code}] {d.Message}"); + + var message = + "SensitiveFlow startup validation failed with one or more errors:" + + Environment.NewLine + + string.Join(Environment.NewLine, errors) + + Environment.NewLine + + "Resolve them, or set SensitiveFlowValidationOptions.FailOnError = false to downgrade to warnings."; + + throw new SensitiveFlowConfigurationException(message, "SF-CONFIG-FAIL"); + } - return validator.Validate(services); + return report; } } diff --git a/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs index 8075cc0..eb17359 100644 --- a/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs +++ b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs @@ -31,12 +31,12 @@ public SensitiveFlowConfigurationReport Validate(IServiceProvider services) if (_options.RequireAuditStore && provider.GetService() is null) { - diagnostics.Add(Warning("SF-CONFIG-001", "No IAuditStore registration was found.")); + diagnostics.Add(Error("SF-CONFIG-001", "No IAuditStore registration was found. Register one via AddEfCoreAuditStore() or AddInMemoryAuditStore() (dev only).")); } if (_options.RequireTokenStore && provider.GetService() is null) { - diagnostics.Add(Warning("SF-CONFIG-002", "No durable ITokenStore registration was found.")); + diagnostics.Add(Error("SF-CONFIG-002", "No durable ITokenStore registration was found. Register one via AddEfCoreTokenStore() or AddRedisTokenStore().")); } if (provider.GetService() is not null && provider.GetService() is null) @@ -165,7 +165,7 @@ private static void AddOutboxDiagnostics( var durableOutbox = provider.GetService(); if (durableOutbox is not null && !provider.GetServices().Any()) { - diagnostics.Add(Warning("SF-CONFIG-014", "IDurableAuditOutbox is registered, but no IAuditOutboxPublisher registration was found.")); + diagnostics.Add(Error("SF-CONFIG-014", "IDurableAuditOutbox is registered, but no IAuditOutboxPublisher registration was found. The dispatcher will fail every poll. Register an IAuditOutboxPublisher or disable the outbox.")); } } @@ -223,4 +223,14 @@ private static SensitiveFlowConfigurationDiagnostic Warning(string code, string Severity = SensitiveFlowDiagnosticSeverity.Warning, }; } + + private static SensitiveFlowConfigurationDiagnostic Error(string code, string message) + { + return new SensitiveFlowConfigurationDiagnostic + { + Code = code, + Message = message, + Severity = SensitiveFlowDiagnosticSeverity.Error, + }; + } } diff --git a/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowValidationOptions.cs b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowValidationOptions.cs index b2d62c1..59a7a0d 100644 --- a/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowValidationOptions.cs +++ b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowValidationOptions.cs @@ -16,5 +16,11 @@ public sealed class SensitiveFlowValidationOptions /// Gets or sets whether retention services are required. public bool RequireRetention { get; set; } -} + /// + /// When true, -driven validation + /// throws on diagnostics with severity Error instead of silently returning a report. + /// Defaults to true so misconfigurations surface at startup, not at the first write. + /// + public bool FailOnError { get; set; } = true; +} diff --git a/src/SensitiveFlow.TokenStore.EFCore/Configuration/TokenMappingEntityTypeConfiguration.cs b/src/SensitiveFlow.TokenStore.EFCore/Configuration/TokenMappingEntityTypeConfiguration.cs index d536d7e..aa047f0 100644 --- a/src/SensitiveFlow.TokenStore.EFCore/Configuration/TokenMappingEntityTypeConfiguration.cs +++ b/src/SensitiveFlow.TokenStore.EFCore/Configuration/TokenMappingEntityTypeConfiguration.cs @@ -11,22 +11,41 @@ namespace SensitiveFlow.TokenStore.EFCore.Configuration; /// public sealed class TokenMappingEntityTypeConfiguration : IEntityTypeConfiguration { + /// Default table name when none is specified. + public const string DefaultTableName = "SensitiveFlow_TokenMappings"; + private readonly string _tableName; + private readonly string? _schema; - /// Initializes the configuration with the given table name. - public TokenMappingEntityTypeConfiguration(string tableName = "SensitiveFlow_TokenMappings") + /// Initializes the configuration with the given table name and no schema. + public TokenMappingEntityTypeConfiguration(string tableName = DefaultTableName) + : this(tableName, null) { - if (string.IsNullOrWhiteSpace(tableName)) - { - throw new ArgumentException("Table name must not be empty.", nameof(tableName)); - } + } + + /// + /// Initializes the configuration with a custom table name and optional schema. + /// + /// Table name. Defaults to . + /// Optional schema. Leave null for providers without schemas (e.g. SQLite). + public TokenMappingEntityTypeConfiguration(string tableName, string? schema) + { + ArgumentException.ThrowIfNullOrWhiteSpace(tableName); _tableName = tableName; + _schema = string.IsNullOrWhiteSpace(schema) ? null : schema; } /// public void Configure(EntityTypeBuilder builder) { - builder.ToTable(_tableName); + if (_schema is null) + { + builder.ToTable(_tableName); + } + else + { + builder.ToTable(_tableName, _schema); + } builder.HasKey(e => e.Id); builder.Property(e => e.Id).ValueGeneratedOnAdd(); diff --git a/src/SensitiveFlow.TokenStore.EFCore/Stores/EfCoreTokenStore.cs b/src/SensitiveFlow.TokenStore.EFCore/Stores/EfCoreTokenStore.cs index 0ef80a5..b9ed81d 100644 --- a/src/SensitiveFlow.TokenStore.EFCore/Stores/EfCoreTokenStore.cs +++ b/src/SensitiveFlow.TokenStore.EFCore/Stores/EfCoreTokenStore.cs @@ -1,4 +1,5 @@ using Microsoft.EntityFrameworkCore; +using SensitiveFlow.Core.Exceptions; using SensitiveFlow.Core.Interfaces; using SensitiveFlow.TokenStore.EFCore.Entities; @@ -49,10 +50,18 @@ public async Task GetOrCreateTokenAsync(string value, CancellationToken await using var ctx = await _factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); var set = _setSelector(ctx); - var existing = await set - .AsNoTracking() - .FirstOrDefaultAsync(t => t.Value == value, cancellationToken) - .ConfigureAwait(false); + TokenMappingEntity? existing; + try + { + existing = await set + .AsNoTracking() + .FirstOrDefaultAsync(t => t.Value == value, cancellationToken) + .ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + throw SchemaErrorTranslator.Translate(ex, typeof(TContext).Name); + } if (existing is not null) { @@ -93,10 +102,18 @@ public async Task ResolveTokenAsync(string token, CancellationToken canc await using var ctx = await _factory.CreateDbContextAsync(cancellationToken).ConfigureAwait(false); var set = _setSelector(ctx); - var mapping = await set - .AsNoTracking() - .FirstOrDefaultAsync(t => t.Token == token, cancellationToken) - .ConfigureAwait(false); + TokenMappingEntity? mapping; + try + { + mapping = await set + .AsNoTracking() + .FirstOrDefaultAsync(t => t.Token == token, cancellationToken) + .ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + throw SchemaErrorTranslator.Translate(ex, typeof(TContext).Name); + } return mapping?.Value ?? throw new KeyNotFoundException($"Token '{token}' not found in the store."); diff --git a/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs b/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs index ba08ecd..6f3de0d 100644 --- a/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs +++ b/tests/SensitiveFlow.Diagnostics.Tests/SensitiveFlowValidationTests.cs @@ -20,14 +20,19 @@ namespace SensitiveFlow.Diagnostics.Tests; public sealed class SensitiveFlowValidationTests { [Fact] - public void ValidateSensitiveFlow_WhenAuditStoreRequiredAndMissing_ReportsWarning() + public void ValidateSensitiveFlow_WhenAuditStoreRequiredAndMissing_ReportsError() { var services = new ServiceCollection(); - services.AddSensitiveFlowValidation(o => o.RequireAuditStore = true); + services.AddSensitiveFlowValidation(o => + { + o.RequireAuditStore = true; + o.FailOnError = false; + }); var report = services.BuildServiceProvider().ValidateSensitiveFlow(); - report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-001"); + report.Diagnostics.Should().Contain(d => + d.Code == "SF-CONFIG-001" && d.Severity == SensitiveFlowDiagnosticSeverity.Error); } [Fact] @@ -43,14 +48,19 @@ public void ValidateSensitiveFlow_WhenRequiredStoresExist_ReturnsNoStoreWarnings } [Fact] - public void ValidateSensitiveFlow_WhenTokenStoreRequiredAndMissing_ReportsWarning() + public void ValidateSensitiveFlow_WhenTokenStoreRequiredAndMissing_ReportsError() { var services = new ServiceCollection(); - services.AddSensitiveFlowValidation(o => o.RequireTokenStore = true); + services.AddSensitiveFlowValidation(o => + { + o.RequireTokenStore = true; + o.FailOnError = false; + }); var report = services.BuildServiceProvider().ValidateSensitiveFlow(); - report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-002"); + report.Diagnostics.Should().Contain(d => + d.Code == "SF-CONFIG-002" && d.Severity == SensitiveFlowDiagnosticSeverity.Error); } [Fact] @@ -99,9 +109,14 @@ public void ValidateSensitiveFlow_WhenPseudonymizerWithoutTokenStore_ReportsWarn [Fact] public void ValidateSensitiveFlow_WithoutRegisteredValidator_UsesDefaultValidator() { - var report = new ServiceCollection().BuildServiceProvider().ValidateSensitiveFlow(); + // The default validator requires an audit store and (now) fails on error, + // which is the documented behavior. We assert that contract here. + var services = new ServiceCollection().BuildServiceProvider(); - report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-001"); + Action act = () => services.ValidateSensitiveFlow(); + + act.Should().Throw() + .Which.Message.Should().Contain("SF-CONFIG-001"); } [Fact] @@ -229,15 +244,37 @@ public void ValidateSensitiveFlow_WhenInMemoryOutboxRegisteredInDevelopment_Does } [Fact] - public void ValidateSensitiveFlow_WhenDurableOutboxHasNoPublisher_ReportsWarning() + public void ValidateSensitiveFlow_WhenDurableOutboxHasNoPublisher_ReportsError() { var services = new ServiceCollection(); services.AddSingleton(); - services.AddSensitiveFlowValidation(o => o.RequireAuditStore = false); + services.AddSensitiveFlowValidation(o => + { + o.RequireAuditStore = false; + o.FailOnError = false; + }); var report = services.BuildServiceProvider().ValidateSensitiveFlow(); - report.Diagnostics.Should().Contain(d => d.Code == "SF-CONFIG-014"); + report.Diagnostics.Should().Contain(d => + d.Code == "SF-CONFIG-014" && d.Severity == SensitiveFlowDiagnosticSeverity.Error); + } + + [Fact] + public void ValidateSensitiveFlow_WhenFailOnErrorEnabled_ThrowsOnErrorDiagnostic() + { + var services = new ServiceCollection(); + services.AddSingleton(); + services.AddSensitiveFlowValidation(o => + { + o.RequireAuditStore = false; + o.FailOnError = true; + }); + + Action act = () => services.BuildServiceProvider().ValidateSensitiveFlow(); + + act.Should().Throw() + .Which.Code.Should().Be("SF-CONFIG-FAIL"); } [Fact] diff --git a/tools/migrations/README.md b/tools/migrations/README.md new file mode 100644 index 0000000..f2b1bfb --- /dev/null +++ b/tools/migrations/README.md @@ -0,0 +1,68 @@ +# SensitiveFlow database schemas + +Idempotent SQL scripts for the four SensitiveFlow stores (audit records, outbox, +token mappings, audit snapshots) across SQLite, SQL Server, and PostgreSQL. + +The library packages do **not** create production schema on their own. Apply +these scripts (or your own EF Core migrations) before starting the app in +production. + +## Layout + +``` +tools/migrations/ +├── sqlite/ # Local development, samples +├── sqlserver/ # SQL Server / Azure SQL +└── postgres/ # PostgreSQL / Aurora / Supabase / Neon +``` + +Each provider folder contains numbered scripts you can run in order: + +| Script | Required if you use ... | +|---------------------|----------------------------------------------------------| +| `01-audit.sql` | `SensitiveFlow.Audit.EFCore` (always recommended) | +| `02-outbox.sql` | `SensitiveFlow.Audit.EFCore.Outbox` (`EnableOutbox()`) | +| `03-tokens.sql` | `SensitiveFlow.TokenStore.EFCore` (pseudonymization) | +| `04-snapshots.sql` | `SensitiveFlow.Audit.Snapshots.EFCore` (point-in-time) | + +All scripts are idempotent (`CREATE TABLE IF NOT EXISTS` / `IF OBJECT_ID IS NULL`), +so it is safe to re-run them during deployment. + +## Applying the scripts + +### SQLite + +```powershell +sqlite3 mydb.db ".read tools/migrations/sqlite/01-audit.sql" +sqlite3 mydb.db ".read tools/migrations/sqlite/02-outbox.sql" +sqlite3 mydb.db ".read tools/migrations/sqlite/03-tokens.sql" +``` + +### SQL Server + +```powershell +sqlcmd -S server -d database -i tools/migrations/sqlserver/01-audit.sql +sqlcmd -S server -d database -i tools/migrations/sqlserver/02-outbox.sql +sqlcmd -S server -d database -i tools/migrations/sqlserver/03-tokens.sql +``` + +### PostgreSQL + +```bash +psql -d mydb -f tools/migrations/postgres/01-audit.sql +psql -d mydb -f tools/migrations/postgres/02-outbox.sql +psql -d mydb -f tools/migrations/postgres/03-tokens.sql +``` + +## EF Core migrations + +If you prefer EF Core migrations, generate them against your own +`DbContext` once you have added the SensitiveFlow EF Core packages: + +```bash +dotnet ef migrations add SensitiveFlowInitial --context AuditDbContext +dotnet ef database update --context AuditDbContext +``` + +The included scripts mirror what EF Core would emit for the default table +names and `null` schema (so they remain provider-agnostic by default). diff --git a/tools/migrations/postgres/01-audit.sql b/tools/migrations/postgres/01-audit.sql new file mode 100644 index 0000000..08cc38b --- /dev/null +++ b/tools/migrations/postgres/01-audit.sql @@ -0,0 +1,24 @@ +-- SensitiveFlow Audit schema for PostgreSQL +-- Idempotent: safe to run multiple times. + +CREATE TABLE IF NOT EXISTS "SensitiveFlow_AuditRecords" ( + "Id" BIGSERIAL PRIMARY KEY, + "RecordId" VARCHAR(64) NOT NULL, + "DataSubjectId" VARCHAR(256) NOT NULL, + "Entity" VARCHAR(256) NOT NULL, + "Field" VARCHAR(256) NOT NULL, + "Operation" INTEGER NOT NULL, + "Timestamp" TIMESTAMP NOT NULL, + "ActorId" VARCHAR(256), + "IpAddressToken" VARCHAR(128), + "Details" VARCHAR(2048) +); + +CREATE UNIQUE INDEX IF NOT EXISTS "IX_SensitiveFlow_AuditRecords_RecordId" + ON "SensitiveFlow_AuditRecords" ("RecordId"); + +CREATE INDEX IF NOT EXISTS "IX_SensitiveFlow_AuditRecords_Timestamp" + ON "SensitiveFlow_AuditRecords" ("Timestamp"); + +CREATE INDEX IF NOT EXISTS "IX_SensitiveFlow_AuditRecords_DataSubjectId_Timestamp" + ON "SensitiveFlow_AuditRecords" ("DataSubjectId", "Timestamp"); diff --git a/tools/migrations/postgres/02-outbox.sql b/tools/migrations/postgres/02-outbox.sql new file mode 100644 index 0000000..9fabae1 --- /dev/null +++ b/tools/migrations/postgres/02-outbox.sql @@ -0,0 +1,28 @@ +-- SensitiveFlow Outbox schema for PostgreSQL +-- Default deployment uses the "public" schema. If you need a dedicated schema, +-- create it (CREATE SCHEMA sensitiveflow;) and pass the schema name to +-- AuditOutboxEntryConfiguration("AuditOutboxEntries", "sensitiveflow"). + +CREATE TABLE IF NOT EXISTS "AuditOutboxEntries" ( + "Id" UUID NOT NULL PRIMARY KEY, + "AuditRecordId" VARCHAR(256) NOT NULL, + "Payload" TEXT NOT NULL, + "Attempts" INTEGER NOT NULL DEFAULT 0, + "EnqueuedAt" TIMESTAMPTZ NOT NULL, + "EnqueuedAtTicks" BIGINT NOT NULL, + "LastAttemptAt" TIMESTAMPTZ, + "LastError" VARCHAR(512), + "IsProcessed" BOOLEAN NOT NULL DEFAULT FALSE, + "ProcessedAt" TIMESTAMPTZ, + "IsDeadLettered" BOOLEAN NOT NULL DEFAULT FALSE, + "DeadLetterReason" VARCHAR(512) +); + +CREATE INDEX IF NOT EXISTS "IX_AuditOutboxEntries_IsProcessed_IsDeadLettered" + ON "AuditOutboxEntries" ("IsProcessed", "IsDeadLettered"); + +CREATE INDEX IF NOT EXISTS "IX_AuditOutboxEntries_IsDeadLettered" + ON "AuditOutboxEntries" ("IsDeadLettered"); + +CREATE INDEX IF NOT EXISTS "IX_AuditOutboxEntries_EnqueuedAtTicks" + ON "AuditOutboxEntries" ("EnqueuedAtTicks"); diff --git a/tools/migrations/postgres/03-tokens.sql b/tools/migrations/postgres/03-tokens.sql new file mode 100644 index 0000000..3e611b0 --- /dev/null +++ b/tools/migrations/postgres/03-tokens.sql @@ -0,0 +1,13 @@ +-- SensitiveFlow Token Store schema for PostgreSQL + +CREATE TABLE IF NOT EXISTS "SensitiveFlow_TokenMappings" ( + "Id" BIGSERIAL PRIMARY KEY, + "Value" VARCHAR(512) NOT NULL, + "Token" VARCHAR(128) NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS "IX_SensitiveFlow_TokenMappings_Value" + ON "SensitiveFlow_TokenMappings" ("Value"); + +CREATE INDEX IF NOT EXISTS "IX_SensitiveFlow_TokenMappings_Token" + ON "SensitiveFlow_TokenMappings" ("Token"); diff --git a/tools/migrations/postgres/04-snapshots.sql b/tools/migrations/postgres/04-snapshots.sql new file mode 100644 index 0000000..03cf88a --- /dev/null +++ b/tools/migrations/postgres/04-snapshots.sql @@ -0,0 +1,23 @@ +-- SensitiveFlow Audit Snapshots schema for PostgreSQL + +CREATE TABLE IF NOT EXISTS "SensitiveFlow_AuditSnapshots" ( + "Id" BIGSERIAL PRIMARY KEY, + "SnapshotId" VARCHAR(64) NOT NULL, + "DataSubjectId" VARCHAR(256) NOT NULL, + "Aggregate" VARCHAR(256) NOT NULL, + "AggregateId" VARCHAR(256) NOT NULL, + "Timestamp" TIMESTAMP NOT NULL, + "ActorId" VARCHAR(256), + "IpAddressToken" VARCHAR(128), + "BeforeJson" TEXT, + "AfterJson" TEXT +); + +CREATE UNIQUE INDEX IF NOT EXISTS "IX_SensitiveFlow_AuditSnapshots_SnapshotId" + ON "SensitiveFlow_AuditSnapshots" ("SnapshotId"); + +CREATE INDEX IF NOT EXISTS "IX_SensitiveFlow_AuditSnapshots_Aggregate_Timestamp" + ON "SensitiveFlow_AuditSnapshots" ("Aggregate", "AggregateId", "Timestamp"); + +CREATE INDEX IF NOT EXISTS "IX_SensitiveFlow_AuditSnapshots_DataSubjectId_Timestamp" + ON "SensitiveFlow_AuditSnapshots" ("DataSubjectId", "Timestamp"); diff --git a/tools/migrations/sqlite/01-audit.sql b/tools/migrations/sqlite/01-audit.sql new file mode 100644 index 0000000..b3eee75 --- /dev/null +++ b/tools/migrations/sqlite/01-audit.sql @@ -0,0 +1,24 @@ +-- SensitiveFlow Audit schema for SQLite +-- Idempotent: safe to run multiple times. + +CREATE TABLE IF NOT EXISTS "SensitiveFlow_AuditRecords" ( + "Id" INTEGER NOT NULL CONSTRAINT "PK_SensitiveFlow_AuditRecords" PRIMARY KEY AUTOINCREMENT, + "RecordId" TEXT NOT NULL, + "DataSubjectId" TEXT NOT NULL, + "Entity" TEXT NOT NULL, + "Field" TEXT NOT NULL, + "Operation" INTEGER NOT NULL, + "Timestamp" TEXT NOT NULL, + "ActorId" TEXT NULL, + "IpAddressToken" TEXT NULL, + "Details" TEXT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS "IX_SensitiveFlow_AuditRecords_RecordId" + ON "SensitiveFlow_AuditRecords" ("RecordId"); + +CREATE INDEX IF NOT EXISTS "IX_SensitiveFlow_AuditRecords_Timestamp" + ON "SensitiveFlow_AuditRecords" ("Timestamp"); + +CREATE INDEX IF NOT EXISTS "IX_SensitiveFlow_AuditRecords_DataSubjectId_Timestamp" + ON "SensitiveFlow_AuditRecords" ("DataSubjectId", "Timestamp"); diff --git a/tools/migrations/sqlite/02-outbox.sql b/tools/migrations/sqlite/02-outbox.sql new file mode 100644 index 0000000..e1bd126 --- /dev/null +++ b/tools/migrations/sqlite/02-outbox.sql @@ -0,0 +1,26 @@ +-- SensitiveFlow Outbox schema for SQLite +-- Note: no schema prefix — SQLite does not support schemas. + +CREATE TABLE IF NOT EXISTS "AuditOutboxEntries" ( + "Id" TEXT NOT NULL CONSTRAINT "PK_AuditOutboxEntries" PRIMARY KEY, + "AuditRecordId" TEXT NOT NULL, + "Payload" TEXT NOT NULL, + "Attempts" INTEGER NOT NULL DEFAULT 0, + "EnqueuedAt" TEXT NOT NULL, + "EnqueuedAtTicks" INTEGER NOT NULL, + "LastAttemptAt" TEXT NULL, + "LastError" TEXT NULL, + "IsProcessed" INTEGER NOT NULL DEFAULT 0, + "ProcessedAt" TEXT NULL, + "IsDeadLettered" INTEGER NOT NULL DEFAULT 0, + "DeadLetterReason" TEXT NULL +); + +CREATE INDEX IF NOT EXISTS "IX_AuditOutboxEntries_IsProcessed_IsDeadLettered" + ON "AuditOutboxEntries" ("IsProcessed", "IsDeadLettered"); + +CREATE INDEX IF NOT EXISTS "IX_AuditOutboxEntries_IsDeadLettered" + ON "AuditOutboxEntries" ("IsDeadLettered"); + +CREATE INDEX IF NOT EXISTS "IX_AuditOutboxEntries_EnqueuedAtTicks" + ON "AuditOutboxEntries" ("EnqueuedAtTicks"); diff --git a/tools/migrations/sqlite/03-tokens.sql b/tools/migrations/sqlite/03-tokens.sql new file mode 100644 index 0000000..ca3ac8d --- /dev/null +++ b/tools/migrations/sqlite/03-tokens.sql @@ -0,0 +1,13 @@ +-- SensitiveFlow Token Store schema for SQLite + +CREATE TABLE IF NOT EXISTS "SensitiveFlow_TokenMappings" ( + "Id" INTEGER NOT NULL CONSTRAINT "PK_SensitiveFlow_TokenMappings" PRIMARY KEY AUTOINCREMENT, + "Value" TEXT NOT NULL, + "Token" TEXT NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS "IX_SensitiveFlow_TokenMappings_Value" + ON "SensitiveFlow_TokenMappings" ("Value"); + +CREATE INDEX IF NOT EXISTS "IX_SensitiveFlow_TokenMappings_Token" + ON "SensitiveFlow_TokenMappings" ("Token"); diff --git a/tools/migrations/sqlite/04-snapshots.sql b/tools/migrations/sqlite/04-snapshots.sql new file mode 100644 index 0000000..11ebce0 --- /dev/null +++ b/tools/migrations/sqlite/04-snapshots.sql @@ -0,0 +1,23 @@ +-- SensitiveFlow Audit Snapshots schema for SQLite + +CREATE TABLE IF NOT EXISTS "SensitiveFlow_AuditSnapshots" ( + "Id" INTEGER NOT NULL CONSTRAINT "PK_SensitiveFlow_AuditSnapshots" PRIMARY KEY AUTOINCREMENT, + "SnapshotId" TEXT NOT NULL, + "DataSubjectId" TEXT NOT NULL, + "Aggregate" TEXT NOT NULL, + "AggregateId" TEXT NOT NULL, + "Timestamp" TEXT NOT NULL, + "ActorId" TEXT NULL, + "IpAddressToken" TEXT NULL, + "BeforeJson" TEXT NULL, + "AfterJson" TEXT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS "IX_SensitiveFlow_AuditSnapshots_SnapshotId" + ON "SensitiveFlow_AuditSnapshots" ("SnapshotId"); + +CREATE INDEX IF NOT EXISTS "IX_SensitiveFlow_AuditSnapshots_Aggregate_Timestamp" + ON "SensitiveFlow_AuditSnapshots" ("Aggregate", "AggregateId", "Timestamp"); + +CREATE INDEX IF NOT EXISTS "IX_SensitiveFlow_AuditSnapshots_DataSubjectId_Timestamp" + ON "SensitiveFlow_AuditSnapshots" ("DataSubjectId", "Timestamp"); diff --git a/tools/migrations/sqlserver/01-audit.sql b/tools/migrations/sqlserver/01-audit.sql new file mode 100644 index 0000000..a409c17 --- /dev/null +++ b/tools/migrations/sqlserver/01-audit.sql @@ -0,0 +1,34 @@ +-- SensitiveFlow Audit schema for SQL Server +-- Idempotent: safe to run multiple times. + +IF OBJECT_ID(N'[SensitiveFlow_AuditRecords]', N'U') IS NULL +BEGIN + CREATE TABLE [SensitiveFlow_AuditRecords] ( + [Id] BIGINT IDENTITY(1,1) NOT NULL CONSTRAINT [PK_SensitiveFlow_AuditRecords] PRIMARY KEY, + [RecordId] NVARCHAR(64) NOT NULL, + [DataSubjectId] NVARCHAR(256) NOT NULL, + [Entity] NVARCHAR(256) NOT NULL, + [Field] NVARCHAR(256) NOT NULL, + [Operation] INT NOT NULL, + [Timestamp] DATETIME2(7) NOT NULL, + [ActorId] NVARCHAR(256) NULL, + [IpAddressToken] NVARCHAR(128) NULL, + [Details] NVARCHAR(2048) NULL + ); +END +GO + +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'IX_SensitiveFlow_AuditRecords_RecordId') + CREATE UNIQUE INDEX [IX_SensitiveFlow_AuditRecords_RecordId] + ON [SensitiveFlow_AuditRecords] ([RecordId]); +GO + +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'IX_SensitiveFlow_AuditRecords_Timestamp') + CREATE INDEX [IX_SensitiveFlow_AuditRecords_Timestamp] + ON [SensitiveFlow_AuditRecords] ([Timestamp]); +GO + +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'IX_SensitiveFlow_AuditRecords_DataSubjectId_Timestamp') + CREATE INDEX [IX_SensitiveFlow_AuditRecords_DataSubjectId_Timestamp] + ON [SensitiveFlow_AuditRecords] ([DataSubjectId], [Timestamp]); +GO diff --git a/tools/migrations/sqlserver/02-outbox.sql b/tools/migrations/sqlserver/02-outbox.sql new file mode 100644 index 0000000..98e4c86 --- /dev/null +++ b/tools/migrations/sqlserver/02-outbox.sql @@ -0,0 +1,38 @@ +-- SensitiveFlow Outbox schema for SQL Server +-- The default deployment uses the default schema (dbo). If you need a dedicated +-- schema, create it via CREATE SCHEMA and update your DbContext model builder +-- to pass it through AuditOutboxEntryConfiguration("AuditOutboxEntries", "your_schema"). + +IF OBJECT_ID(N'[AuditOutboxEntries]', N'U') IS NULL +BEGIN + CREATE TABLE [AuditOutboxEntries] ( + [Id] UNIQUEIDENTIFIER NOT NULL CONSTRAINT [PK_AuditOutboxEntries] PRIMARY KEY, + [AuditRecordId] NVARCHAR(256) NOT NULL, + [Payload] NVARCHAR(MAX) NOT NULL, + [Attempts] INT NOT NULL CONSTRAINT [DF_AuditOutboxEntries_Attempts] DEFAULT 0, + [EnqueuedAt] DATETIMEOFFSET(7) NOT NULL, + [EnqueuedAtTicks] BIGINT NOT NULL, + [LastAttemptAt] DATETIMEOFFSET(7) NULL, + [LastError] NVARCHAR(512) NULL, + [IsProcessed] BIT NOT NULL CONSTRAINT [DF_AuditOutboxEntries_IsProcessed] DEFAULT 0, + [ProcessedAt] DATETIMEOFFSET(7) NULL, + [IsDeadLettered] BIT NOT NULL CONSTRAINT [DF_AuditOutboxEntries_IsDeadLettered] DEFAULT 0, + [DeadLetterReason] NVARCHAR(512) NULL + ); +END +GO + +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'IX_AuditOutboxEntries_IsProcessed_IsDeadLettered') + CREATE INDEX [IX_AuditOutboxEntries_IsProcessed_IsDeadLettered] + ON [AuditOutboxEntries] ([IsProcessed], [IsDeadLettered]); +GO + +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'IX_AuditOutboxEntries_IsDeadLettered') + CREATE INDEX [IX_AuditOutboxEntries_IsDeadLettered] + ON [AuditOutboxEntries] ([IsDeadLettered]); +GO + +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'IX_AuditOutboxEntries_EnqueuedAtTicks') + CREATE INDEX [IX_AuditOutboxEntries_EnqueuedAtTicks] + ON [AuditOutboxEntries] ([EnqueuedAtTicks]); +GO diff --git a/tools/migrations/sqlserver/03-tokens.sql b/tools/migrations/sqlserver/03-tokens.sql new file mode 100644 index 0000000..863b5df --- /dev/null +++ b/tools/migrations/sqlserver/03-tokens.sql @@ -0,0 +1,21 @@ +-- SensitiveFlow Token Store schema for SQL Server + +IF OBJECT_ID(N'[SensitiveFlow_TokenMappings]', N'U') IS NULL +BEGIN + CREATE TABLE [SensitiveFlow_TokenMappings] ( + [Id] BIGINT IDENTITY(1,1) NOT NULL CONSTRAINT [PK_SensitiveFlow_TokenMappings] PRIMARY KEY, + [Value] NVARCHAR(512) NOT NULL, + [Token] NVARCHAR(128) NOT NULL + ); +END +GO + +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'IX_SensitiveFlow_TokenMappings_Value') + CREATE UNIQUE INDEX [IX_SensitiveFlow_TokenMappings_Value] + ON [SensitiveFlow_TokenMappings] ([Value]); +GO + +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'IX_SensitiveFlow_TokenMappings_Token') + CREATE INDEX [IX_SensitiveFlow_TokenMappings_Token] + ON [SensitiveFlow_TokenMappings] ([Token]); +GO diff --git a/tools/migrations/sqlserver/04-snapshots.sql b/tools/migrations/sqlserver/04-snapshots.sql new file mode 100644 index 0000000..505748e --- /dev/null +++ b/tools/migrations/sqlserver/04-snapshots.sql @@ -0,0 +1,33 @@ +-- SensitiveFlow Audit Snapshots schema for SQL Server + +IF OBJECT_ID(N'[SensitiveFlow_AuditSnapshots]', N'U') IS NULL +BEGIN + CREATE TABLE [SensitiveFlow_AuditSnapshots] ( + [Id] BIGINT IDENTITY(1,1) NOT NULL CONSTRAINT [PK_SensitiveFlow_AuditSnapshots] PRIMARY KEY, + [SnapshotId] NVARCHAR(64) NOT NULL, + [DataSubjectId] NVARCHAR(256) NOT NULL, + [Aggregate] NVARCHAR(256) NOT NULL, + [AggregateId] NVARCHAR(256) NOT NULL, + [Timestamp] DATETIME2(7) NOT NULL, + [ActorId] NVARCHAR(256) NULL, + [IpAddressToken] NVARCHAR(128) NULL, + [BeforeJson] NVARCHAR(MAX) NULL, + [AfterJson] NVARCHAR(MAX) NULL + ); +END +GO + +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'IX_SensitiveFlow_AuditSnapshots_SnapshotId') + CREATE UNIQUE INDEX [IX_SensitiveFlow_AuditSnapshots_SnapshotId] + ON [SensitiveFlow_AuditSnapshots] ([SnapshotId]); +GO + +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'IX_SensitiveFlow_AuditSnapshots_Aggregate_Timestamp') + CREATE INDEX [IX_SensitiveFlow_AuditSnapshots_Aggregate_Timestamp] + ON [SensitiveFlow_AuditSnapshots] ([Aggregate], [AggregateId], [Timestamp]); +GO + +IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = N'IX_SensitiveFlow_AuditSnapshots_DataSubjectId_Timestamp') + CREATE INDEX [IX_SensitiveFlow_AuditSnapshots_DataSubjectId_Timestamp] + ON [SensitiveFlow_AuditSnapshots] ([DataSubjectId], [Timestamp]); +GO From 73370ebeae6f031d7ddfd922863d785ed862047d Mon Sep 17 00:00:00 2001 From: Mateus Silva Date: Mon, 11 May 2026 23:46:07 -0300 Subject: [PATCH 14/23] feat: Improve error messaging and diagnostics in token stores and audit handling --- docs/audit.md | 2 +- docs/policies-discovery-health.md | 15 +++++-- samples/Console.Sample/Program.cs | 2 +- samples/Redis.Sample/RedisTokenStore.cs | 2 +- .../Decorators/BufferedAuditStore.cs | 44 ++++++++++++++++++- .../SensitiveDataAuditInterceptor.cs | 10 ++++- .../Stores/EfCoreTokenStore.cs | 19 ++++++-- 7 files changed, 82 insertions(+), 12 deletions(-) diff --git a/docs/audit.md b/docs/audit.md index 1ea7e40..58af3f1 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -214,7 +214,7 @@ builder.Services.AddAuditOutbox(); | --- | --- | --- | --- | --- | | `InMemoryAuditOutbox` | tests/local development | best effort / at-most-once | no | Deprecated for production. Diagnostics emit `SF-CONFIG-013` outside Development, and the audit-outbox health check reports `Degraded`. | | `SensitiveFlow.Audit.EFCore.Outbox` | production durable delivery | at-least-once | yes | EF Core storage, dispatcher retries, and dead-letter state. | -| custom `IDurableAuditOutbox` | application-specific backend | defined by your implementation | should be yes | Register at least one `IAuditOutboxPublisher`; otherwise diagnostics emit `SF-CONFIG-014`. | +| custom `IDurableAuditOutbox` | application-specific backend | defined by your implementation | should be yes | Register at least one `IAuditOutboxPublisher`; otherwise diagnostics emit `SF-CONFIG-014` as an **Error** (validation throws unless `FailOnError = false`). | Dispatcher defaults: diff --git a/docs/policies-discovery-health.md b/docs/policies-discovery-health.md index 1b504c7..d39ec62 100644 --- a/docs/policies-discovery-health.md +++ b/docs/policies-discovery-health.md @@ -126,14 +126,21 @@ builder.Services.AddSensitiveFlowValidation(options => var result = app.Services.ValidateSensitiveFlow(); ``` -Example warnings: +Example diagnostics: + +**Errors** (cause `ValidateSensitiveFlow()` to throw `SensitiveFlowConfigurationException` +when `SensitiveFlowValidationOptions.FailOnError` is `true`, the default): + +- `SF-CONFIG-001`: no `IAuditStore` registration found (when `RequireAuditStore = true`). +- `SF-CONFIG-002`: no durable `ITokenStore` registration found (when `RequireTokenStore = true`). +- `SF-CONFIG-014`: durable audit outbox registered without any `IAuditOutboxPublisher` — the dispatcher would fail every poll. + +**Warnings** (reported but never throw): -- `SF-CONFIG-001`: no `IAuditStore` registration found. -- `SF-CONFIG-002`: no durable `ITokenStore` registration found. - `SF-CONFIG-003`: `IPseudonymizer` registered without `ITokenStore`. - `SF-CONFIG-009`: EF Core audit interceptor registered without `IAuditStore`. - `SF-CONFIG-010`: retention annotations found without `RetentionExecutor` or handlers. - `SF-CONFIG-011`: ASP.NET Core services registered but `UseSensitiveFlowAudit()` was not marked. - `SF-CONFIG-012`: middleware observed an authenticated user before it ran, which can indicate it was placed after authentication. - `SF-CONFIG-013`: in-memory audit outbox registered outside Development. -- `SF-CONFIG-014`: durable audit outbox registered without any `IAuditOutboxPublisher`. +- `SF-CONFIG-FAIL`: code on the exception thrown by `ValidateSensitiveFlow()` when one or more errors are present. diff --git a/samples/Console.Sample/Program.cs b/samples/Console.Sample/Program.cs index f03c7d2..682887d 100644 --- a/samples/Console.Sample/Program.cs +++ b/samples/Console.Sample/Program.cs @@ -367,7 +367,7 @@ public async Task ResolveTokenAsync(string token, CancellationToken canc if (mapping is null) { - throw new KeyNotFoundException($"Token '{token}' not found in the store."); + throw new KeyNotFoundException("Token not found in the store."); } return mapping.Value; diff --git a/samples/Redis.Sample/RedisTokenStore.cs b/samples/Redis.Sample/RedisTokenStore.cs index 59c02c1..49ba0a2 100644 --- a/samples/Redis.Sample/RedisTokenStore.cs +++ b/samples/Redis.Sample/RedisTokenStore.cs @@ -82,7 +82,7 @@ public async Task ResolveTokenAsync(string token, CancellationToken canc var value = await _db.StringGetAsync(TokenToValuePrefix + token).ConfigureAwait(false); if (!value.HasValue) { - throw new KeyNotFoundException($"Token '{token}' not found in the store."); + throw new KeyNotFoundException("Token not found in the store."); } return value.ToString(); } diff --git a/src/SensitiveFlow.Audit/Decorators/BufferedAuditStore.cs b/src/SensitiveFlow.Audit/Decorators/BufferedAuditStore.cs index 833670c..ccafb27 100644 --- a/src/SensitiveFlow.Audit/Decorators/BufferedAuditStore.cs +++ b/src/SensitiveFlow.Audit/Decorators/BufferedAuditStore.cs @@ -147,8 +147,31 @@ public Task> QueryByDataSubjectAsync( => _inner.QueryByDataSubjectAsync(dataSubjectId, from, to, skip, take, cancellationToken); /// + /// + /// Prefer await using / . This synchronous overload + /// signals the worker to complete and waits up to + /// for the pending batch to flush. Records still in flight after the timeout are abandoned + /// (counted in .); + /// this overload never blocks indefinitely so it is safe in synchronous shutdown paths. + /// public void Dispose() - => DisposeAsync().AsTask().GetAwaiter().GetResult(); + { + _channel.Writer.TryComplete(); + if (!_worker.Wait(_options.ShutdownTimeout)) + { + // Worker did not finish in time. Count the still-pending records as dropped + // so health/metrics reflect the loss instead of silently swallowing them. + var abandoned = _channel.Reader.Count; + if (abandoned > 0) + { + Interlocked.Add(ref _droppedCount, abandoned); + DroppedItemsCounter.Add(abandoned); + _logger?.LogWarning( + "BufferedAuditStore.Dispose timed out after {Timeout}; {Abandoned} records were not flushed.", + _options.ShutdownTimeout, abandoned); + } + } + } /// public async ValueTask DisposeAsync() @@ -180,6 +203,18 @@ private async Task ProcessAsync() await FlushAsync(batch).ConfigureAwait(false); } } + + // Channel was completed. Drain any records that landed between WaitToReadAsync + // returning false and TryComplete (the channel guarantees they remain readable + // after Complete until drained). + while (_channel.Reader.TryRead(out var record)) + { + batch.Add(record); + } + if (batch.Count > 0) + { + await FlushAsync(batch).ConfigureAwait(false); + } } catch (Exception ex) { @@ -225,6 +260,13 @@ public sealed class BufferedAuditStoreOptions /// Maximum number of queued audit records flushed to the inner store at once. Default 100. public int MaxBatchSize { get; set; } = 100; + + /// + /// Maximum time the synchronous waits for the + /// background worker to drain. Records still in flight after this timeout are counted as + /// dropped instead of blocking shutdown indefinitely. Default 5 seconds. + /// + public TimeSpan ShutdownTimeout { get; set; } = TimeSpan.FromSeconds(5); } /// diff --git a/src/SensitiveFlow.EFCore/Interceptors/SensitiveDataAuditInterceptor.cs b/src/SensitiveFlow.EFCore/Interceptors/SensitiveDataAuditInterceptor.cs index 8379c71..e5aab03 100644 --- a/src/SensitiveFlow.EFCore/Interceptors/SensitiveDataAuditInterceptor.cs +++ b/src/SensitiveFlow.EFCore/Interceptors/SensitiveDataAuditInterceptor.cs @@ -85,11 +85,19 @@ public override async ValueTask SavedChangesAsync( } /// + /// + /// ⚠ DEADLOCK RISK: Flushing audit records requires async I/O. To avoid the + /// classic sync-over-async deadlock on ASP.NET Core's thread-pool, we run the flush + /// via , which detaches from any captured + /// . This is still a synchronous wait — prefer + /// SaveChangesAsync in any concurrent host. + /// public override int SavedChanges(SaveChangesCompletedEventData eventData, int result) { if (eventData.Context is not null) { - FlushAuditRecordsAsync(eventData.Context, CancellationToken.None).GetAwaiter().GetResult(); + var context = eventData.Context; + Task.Run(() => FlushAuditRecordsAsync(context, CancellationToken.None)).GetAwaiter().GetResult(); } return base.SavedChanges(eventData, result); diff --git a/src/SensitiveFlow.TokenStore.EFCore/Stores/EfCoreTokenStore.cs b/src/SensitiveFlow.TokenStore.EFCore/Stores/EfCoreTokenStore.cs index b9ed81d..f856a3b 100644 --- a/src/SensitiveFlow.TokenStore.EFCore/Stores/EfCoreTokenStore.cs +++ b/src/SensitiveFlow.TokenStore.EFCore/Stores/EfCoreTokenStore.cs @@ -86,11 +86,21 @@ public async Task GetOrCreateTokenAsync(string value, CancellationToken ctx.Entry(local).State = EntityState.Detached; } + // Use FirstOrDefault: an extremely rare interleaving with token erasure + // could delete the winner between the conflict and our re-read. var winner = await set .AsNoTracking() - .FirstAsync(t => t.Value == value, cancellationToken) + .FirstOrDefaultAsync(t => t.Value == value, cancellationToken) .ConfigureAwait(false); - return winner.Token; + + if (winner is not null) + { + return winner.Token; + } + + // Winner was deleted before we could observe it (e.g. concurrent erasure). + // Fall through to a fresh attempt rather than crashing the caller. + return await GetOrCreateTokenAsync(value, cancellationToken).ConfigureAwait(false); } } @@ -115,7 +125,10 @@ public async Task ResolveTokenAsync(string token, CancellationToken canc throw SchemaErrorTranslator.Translate(ex, typeof(TContext).Name); } + // Do NOT include the token in the exception message — exception text is commonly + // captured by logs/APM and the token is the pseudonymization key. A generic + // message preserves the failure signal without leaking the lookup key. return mapping?.Value - ?? throw new KeyNotFoundException($"Token '{token}' not found in the store."); + ?? throw new KeyNotFoundException("Token not found in the store."); } } From 5e15c0e2062616686272f20c240770927a97dd84 Mon Sep 17 00:00:00 2001 From: Mateus Silva Date: Tue, 12 May 2026 00:04:37 -0300 Subject: [PATCH 15/23] feat: Update version to preview 3 and enhance audit middleware with non-durable options --- Directory.Build.props | 2 +- .../AspNetCoreServiceCollectionExtensions.cs | 9 ++- .../SensitiveFlowAuditMiddleware.cs | 70 +++++++++++++++++-- .../Decorators/RetryingAuditStore.cs | 21 +++++- .../Outbox/InMemoryAuditOutbox.cs | 2 +- .../Interfaces/INonDurableAuditOutbox.cs | 15 ++++ .../SensitiveFlowConfigurationValidator.cs | 5 +- .../Checks/AuditOutboxHealthCheck.cs | 5 +- .../AuditOutboxHealthCheckTests.cs | 2 +- 9 files changed, 112 insertions(+), 19 deletions(-) create mode 100644 src/SensitiveFlow.Core/Interfaces/INonDurableAuditOutbox.cs diff --git a/Directory.Build.props b/Directory.Build.props index 52f8b84..4c52bee 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -20,7 +20,7 @@ true snupkg true - 1.0.0-preview.2 + 1.0.0-preview.3 diff --git a/src/SensitiveFlow.AspNetCore/Extensions/AspNetCoreServiceCollectionExtensions.cs b/src/SensitiveFlow.AspNetCore/Extensions/AspNetCoreServiceCollectionExtensions.cs index e23edc4..a4cd41c 100644 --- a/src/SensitiveFlow.AspNetCore/Extensions/AspNetCoreServiceCollectionExtensions.cs +++ b/src/SensitiveFlow.AspNetCore/Extensions/AspNetCoreServiceCollectionExtensions.cs @@ -15,11 +15,18 @@ public static class AspNetCoreServiceCollectionExtensions /// Registers as the scoped , /// backed by the HTTP context accessor. /// - public static IServiceCollection AddSensitiveFlowAspNetCore(this IServiceCollection services) + public static IServiceCollection AddSensitiveFlowAspNetCore( + this IServiceCollection services, + Action? configureMiddleware = null) { services.AddHttpContextAccessor(); services.AddScoped(); services.AddSingleton(); + + var middlewareOptions = new SensitiveFlowAuditMiddlewareOptions(); + configureMiddleware?.Invoke(middlewareOptions); + services.AddSingleton(middlewareOptions); + return services; } diff --git a/src/SensitiveFlow.AspNetCore/Middleware/SensitiveFlowAuditMiddleware.cs b/src/SensitiveFlow.AspNetCore/Middleware/SensitiveFlowAuditMiddleware.cs index 3920a77..2572960 100644 --- a/src/SensitiveFlow.AspNetCore/Middleware/SensitiveFlowAuditMiddleware.cs +++ b/src/SensitiveFlow.AspNetCore/Middleware/SensitiveFlowAuditMiddleware.cs @@ -9,6 +9,11 @@ namespace SensitiveFlow.AspNetCore; /// in so HttpAuditContext can read it /// without accessing raw personal data downstream. /// +/// +/// CORS preflight (OPTIONS) and well-known health/liveness paths are skipped so the +/// token store is not hammered by traffic that has no meaningful actor. Tune the skip list +/// via if your app uses non-standard paths. +/// public sealed class SensitiveFlowAuditMiddleware { /// Key used to store the pseudonymized IP token in . @@ -16,14 +21,17 @@ public sealed class SensitiveFlowAuditMiddleware private readonly RequestDelegate _next; private readonly SensitiveFlowAspNetCorePipelineDiagnostics? _diagnostics; + private readonly SensitiveFlowAuditMiddlewareOptions _options; /// Initializes a new instance of . public SensitiveFlowAuditMiddleware( RequestDelegate next, - SensitiveFlowAspNetCorePipelineDiagnostics? diagnostics = null) + SensitiveFlowAspNetCorePipelineDiagnostics? diagnostics = null, + SensitiveFlowAuditMiddlewareOptions? options = null) { _next = next; _diagnostics = diagnostics; + _options = options ?? new SensitiveFlowAuditMiddlewareOptions(); } /// Pseudonymizes the remote IP and stores the token before passing to the next middleware. @@ -34,14 +42,62 @@ public async Task InvokeAsync(HttpContext context, IPseudonymizer pseudonymizer) _diagnostics?.MarkAuthenticatedUserObserved(); } - var ip = context.Connection.RemoteIpAddress?.ToString(); - if (!string.IsNullOrEmpty(ip)) + if (ShouldPseudonymize(context)) { - context.Items[IpTokenKey] = await pseudonymizer.PseudonymizeAsync( - ip, - context.RequestAborted); + var ip = context.Connection.RemoteIpAddress?.ToString(); + if (!string.IsNullOrEmpty(ip)) + { + context.Items[IpTokenKey] = await pseudonymizer + .PseudonymizeAsync(ip, context.RequestAborted) + .ConfigureAwait(false); + } } - await _next(context); + await _next(context).ConfigureAwait(false); } + + private bool ShouldPseudonymize(HttpContext context) + { + // Skip CORS preflight — no actor information to capture, and these fire on every + // cross-origin call. + if (HttpMethods.IsOptions(context.Request.Method)) + { + return false; + } + + if (_options.SkipPaths.Count == 0) + { + return true; + } + + var path = context.Request.Path; + foreach (var skip in _options.SkipPaths) + { + if (path.StartsWithSegments(skip, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + } + + return true; + } +} + +/// +/// Options controlling request-gating behavior. +/// +public sealed class SensitiveFlowAuditMiddlewareOptions +{ + /// + /// Path prefixes (case-insensitive) for which the middleware skips IP pseudonymization. + /// Defaults cover common health/liveness/readiness probes. Add app-specific prefixes + /// (e.g. /metrics) to avoid storing tokens for synthetic traffic. + /// + public IList SkipPaths { get; } = new List + { + "/health", + "/healthz", + "/livez", + "/readyz", + }; } diff --git a/src/SensitiveFlow.Audit/Decorators/RetryingAuditStore.cs b/src/SensitiveFlow.Audit/Decorators/RetryingAuditStore.cs index d235522..e294d64 100644 --- a/src/SensitiveFlow.Audit/Decorators/RetryingAuditStore.cs +++ b/src/SensitiveFlow.Audit/Decorators/RetryingAuditStore.cs @@ -92,11 +92,20 @@ private async Task RetryAsync(Func operation, string op !cancellationToken.IsCancellationRequested) { attempt++; + + // Apply equal-jitter backoff: half of the computed delay is fixed, half is random. + // This breaks the synchronized retry waves that cause "thundering herd" when many + // instances recover from the same upstream failure simultaneously. + var jitterFactor = _options.JitterFactor; + var jittered = jitterFactor <= 0 + ? delay + : TimeSpan.FromTicks((long)(delay.Ticks * (1.0 - jitterFactor + (Random.Shared.NextDouble() * jitterFactor * 2.0)))); + _logger?.LogWarning(ex, "Audit store {Operation} attempt {Attempt}/{MaxAttempts} failed; retrying in {DelayMs}ms.", - operationName, attempt, _options.MaxAttempts, delay.TotalMilliseconds); + operationName, attempt, _options.MaxAttempts, jittered.TotalMilliseconds); - await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + await Task.Delay(jittered, cancellationToken).ConfigureAwait(false); delay = TimeSpan.FromTicks(Math.Min( (long)(delay.Ticks * _options.BackoffMultiplier), @@ -121,6 +130,14 @@ public sealed class RetryingAuditStoreOptions /// Multiplier applied to the delay after each failed attempt. Default 2.0. public double BackoffMultiplier { get; set; } = 2.0; + /// + /// Random jitter applied to each backoff delay, expressed as a fraction in [0, 1). + /// Each delay is multiplied by a random value in [1 - JitterFactor, 1 + JitterFactor) + /// to prevent synchronized retry waves ("thundering herd") across replicas. Set to 0 + /// to disable jitter. Default 0.25 (±25%). + /// + public double JitterFactor { get; set; } = 0.25; + /// /// Predicate that decides whether a thrown exception should trigger a retry. /// Defaults to retrying on any exception that is not a diff --git a/src/SensitiveFlow.Audit/Outbox/InMemoryAuditOutbox.cs b/src/SensitiveFlow.Audit/Outbox/InMemoryAuditOutbox.cs index 292bce4..0893a6f 100644 --- a/src/SensitiveFlow.Audit/Outbox/InMemoryAuditOutbox.cs +++ b/src/SensitiveFlow.Audit/Outbox/InMemoryAuditOutbox.cs @@ -9,7 +9,7 @@ namespace SensitiveFlow.Audit.Outbox; /// It is not durable and should not be used as the only production outbox. /// [Obsolete("InMemoryAuditOutbox is for tests/local development only. Use AddEfCoreAuditOutbox() or AddAuditOutbox() for durable production delivery.", error: false)] -public sealed class InMemoryAuditOutbox : IAuditOutbox +public sealed class InMemoryAuditOutbox : INonDurableAuditOutbox { private readonly ConcurrentQueue _records = new(); diff --git a/src/SensitiveFlow.Core/Interfaces/INonDurableAuditOutbox.cs b/src/SensitiveFlow.Core/Interfaces/INonDurableAuditOutbox.cs new file mode 100644 index 0000000..010b061 --- /dev/null +++ b/src/SensitiveFlow.Core/Interfaces/INonDurableAuditOutbox.cs @@ -0,0 +1,15 @@ +namespace SensitiveFlow.Core.Interfaces; + +/// +/// Marker interface for implementations that are +/// not durable across process restarts (e.g. in-memory queues used in tests or +/// local development). +/// +/// +/// Diagnostics, health checks, and startup validators key off this marker to warn when a +/// non-durable outbox is configured outside Development. Application code should +/// never depend on this interface directly — it carries no behavior, only intent. +/// +public interface INonDurableAuditOutbox : IAuditOutbox +{ +} diff --git a/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs index eb17359..e558fd9 100644 --- a/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs +++ b/src/SensitiveFlow.Diagnostics/Validation/SensitiveFlowConfigurationValidator.cs @@ -156,10 +156,9 @@ private static void AddOutboxDiagnostics( { var outbox = provider.GetService(); var environment = provider.GetService(); - if (outbox?.GetType().FullName == "SensitiveFlow.Audit.Outbox.InMemoryAuditOutbox" - && environment?.IsDevelopment() == false) + if (outbox is INonDurableAuditOutbox && environment?.IsDevelopment() == false) { - diagnostics.Add(Warning("SF-CONFIG-013", "In-memory audit outbox is configured outside Development. Use AddEfCoreAuditOutbox() or AddAuditOutbox().")); + diagnostics.Add(Warning("SF-CONFIG-013", "A non-durable audit outbox is configured outside Development. Use AddEfCoreAuditOutbox() or AddAuditOutbox().")); } var durableOutbox = provider.GetService(); diff --git a/src/SensitiveFlow.HealthChecks/Checks/AuditOutboxHealthCheck.cs b/src/SensitiveFlow.HealthChecks/Checks/AuditOutboxHealthCheck.cs index c68a24e..2f3f54a 100644 --- a/src/SensitiveFlow.HealthChecks/Checks/AuditOutboxHealthCheck.cs +++ b/src/SensitiveFlow.HealthChecks/Checks/AuditOutboxHealthCheck.cs @@ -24,10 +24,9 @@ public Task CheckHealthAsync( { cancellationToken.ThrowIfCancellationRequested(); - var isInMemory = _outbox.GetType().FullName == "SensitiveFlow.Audit.Outbox.InMemoryAuditOutbox"; - if (isInMemory && _environment?.IsDevelopment() == false) + if (_outbox is INonDurableAuditOutbox && _environment?.IsDevelopment() == false) { - return Task.FromResult(HealthCheckResult.Degraded("In-memory audit outbox is configured outside Development.")); + return Task.FromResult(HealthCheckResult.Degraded("A non-durable audit outbox is configured outside Development.")); } return Task.FromResult(HealthCheckResult.Healthy("Audit outbox resolved.")); diff --git a/tests/SensitiveFlow.HealthChecks.Tests/AuditOutboxHealthCheckTests.cs b/tests/SensitiveFlow.HealthChecks.Tests/AuditOutboxHealthCheckTests.cs index 9ec7183..0272307 100644 --- a/tests/SensitiveFlow.HealthChecks.Tests/AuditOutboxHealthCheckTests.cs +++ b/tests/SensitiveFlow.HealthChecks.Tests/AuditOutboxHealthCheckTests.cs @@ -52,7 +52,7 @@ public async Task CheckHealthAsync_WithInMemoryOutbox_InProduction_ReturnsDegrad var result = await check.CheckHealthAsync(new HealthCheckContext()); result.Status.Should().Be(HealthStatus.Degraded); - result.Description.Should().Contain("In-memory audit outbox"); + result.Description.Should().Contain("non-durable audit outbox"); } [Fact] From 35fb846dfc8f9d09e0ec756654f2c51048d2880a Mon Sep 17 00:00:00 2001 From: Mateus Silva Date: Tue, 12 May 2026 07:20:51 -0300 Subject: [PATCH 16/23] docs: rewrite README with value-first approach and tangible examples - Lead with 'Why SensitiveFlow?' using real pain points (data exposed everywhere with no visibility) - Show concrete benefits: audit trail, log redaction, JSON masking, pseudonymization, retention, export, erasure - Simplify Quick Start: 1. Install & annotate, 2. Register, 3. See the magic, 4. Verify - Add 'What You Get' feature table showing examples and actual value - Add 'Going Deeper' for advanced features with doc links - Add 'Samples' section showing how to run examples with simple 'dotnet run' - Remove dated setup details (now in docs/getting-started.md) - Focus on communicating what makes this library valuable vs. doing it manually Co-Authored-By: Claude Opus 4.7 --- README.md | 191 +++++++++--------- RELEASE.md | 35 ++-- docs/analyzers.md | 2 +- samples/Console.Sample/Console.Sample.csproj | 1 + .../MinimalApi.Sample.csproj | 1 + .../QuickStart.Sample.csproj | 1 + samples/Redis.Sample/Redis.Sample.csproj | 1 + samples/WebApi.Sample/WebApi.Sample.csproj | 1 + .../AnalyzerReleases.Shipped.md | 9 + .../AnalyzerReleases.Unshipped.md | 10 +- 10 files changed, 129 insertions(+), 123 deletions(-) diff --git a/README.md b/README.md index 481563c..3b2e48b 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,23 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) [![.NET](https://img.shields.io/badge/.NET-8%20%7C%209%20%7C%2010-blue)](https://dotnet.microsoft.com) -**SensitiveFlow** is a .NET library that brings observability and control to sensitive data. It focuses on runtime behavior -- automatic auditing, log redaction, and masking -- not compliance paperwork. +**SensitiveFlow** is a .NET library that brings observability and control to sensitive data through automatic auditing, log redaction, JSON masking, and pseudonymization. Mark your data once, get automatic protection everywhere. > **Important:** SensitiveFlow helps reduce accidental exposure of sensitive data, but it does not guarantee legal compliance or complete data protection by itself. You are responsible for how you use these primitives in your application. ## Why SensitiveFlow? -Sensitive data flows through your application on every request: EF Core saves, HTTP responses, log lines. SensitiveFlow makes that flow visible and controlled at the infrastructure level, without requiring manual instrumentation. +Sensitive data flows through your application constantly: EF Core database saves, HTTP JSON responses, structured logs, error messages. Without SensitiveFlow, that data is exposed everywhere—in logs, in responses, in backups, in error traces—often with no visibility into where it leaked or why. + +SensitiveFlow gives you: + +- **Automatic audit trail** — Every database change to a sensitive field is logged with who changed it, when, what changed, and why (from your request context). No manual instrumentation. +- **Log redaction** — Your structured logger scrubs sensitive values *before* they reach any sink. A developer who accidentally logs a customer's email will never see it in the logs; it's redacted automatically. +- **JSON response masking** — ASP.NET Core responses automatically mask sensitive fields in your JSON output without changing your serialization code. +- **Reversible pseudonymization** — Replace real values with tokens while keeping them queryable. Trade a customer's email for `token_abc123` in logs and analytics, then recover the original when needed. +- **Data retention & erasure** — Declare retention policies on fields; SensitiveFlow will anonymize or delete expired records on schedule. +- **Data subject export** — Generate privacy reports showing all data associated with a customer in your system. +- **Health checks** — Monitor whether your audit infrastructure is working; if your audit database fails, you'll know immediately instead of silently losing records. ## Packages @@ -45,23 +55,14 @@ Sensitive data flows through your application on every request: EF Core saves, H ## Quick Start -### 1. Install the composition package +### 1. Install and annotate ```bash dotnet add package SensitiveFlow.AspNetCore.EFCore dotnet add package Microsoft.EntityFrameworkCore.SqlServer # or your EF Core provider ``` -This single package brings in the full recommended stack: Core, Audit, EF Core audit -stores and outbox, Token store, Anonymization, JSON and logging redaction, Retention, -Diagnostics, and Health Checks. Database provider packages stay app-owned, so the -same setup works with SQL Server, PostgreSQL, SQLite, MySQL, or any EF Core provider. - -The repository samples target .NET 10. The library packages remain multi-targeted -where supported, but the documented runnable examples are intentionally kept on -the current .NET sample baseline. - -### 2. Annotate your model +Then mark your sensitive fields: ```csharp using SensitiveFlow.Core.Attributes; @@ -70,7 +71,7 @@ using SensitiveFlow.Core.Enums; public class Customer { public Guid Id { get; set; } - public string DataSubjectId { get; set; } = string.Empty; + public string UserId { get; set; } = string.Empty; [PersonalData(Category = DataCategory.Identification)] public string Name { get; set; } = string.Empty; @@ -78,127 +79,121 @@ public class Customer [PersonalData(Category = DataCategory.Contact)] public string Email { get; set; } = string.Empty; - [SensitiveData(Category = SensitiveDataCategory.Other)] + [SensitiveData(Category = SensitiveDataCategory.Financial)] [RetentionData(Years = 5, Policy = RetentionPolicy.AnonymizeOnExpiration)] public string TaxId { get; set; } = string.Empty; } ``` -`DataSubjectId` or `UserId` is required for EF Core audit correlation. - -### 3. Register SensitiveFlow +### 2. Register and start ```csharp using SensitiveFlow.AspNetCore.EFCore.Extensions; +// Register SensitiveFlow builder.Services.AddSensitiveFlowWeb(options => { options.UseProfile(SensitiveFlowProfile.Balanced); - - // Provider-agnostic: use any EF Core provider referenced by your app. options.UseEfCoreStores( - audit => audit.UseSqlServer(builder.Configuration.GetConnectionString("Audit")!), - tokens => tokens.UseSqlServer(builder.Configuration.GetConnectionString("Tokens")!)); - - // You can also configure each store independently: - // options.UseEfCoreAuditStore(audit => audit.UseNpgsql(...)); - // options.UseEfCoreTokenStore(tokens => tokens.UseSqlite(...)); - - // Enable the features your app needs + audit => audit.UseSqlServer("Server=.;Database=Audit;..."), + tokens => tokens.UseSqlServer("Server=.;Database=Tokens;...")); + options.EnableEfCoreAudit(); options.EnableAspNetCoreContext(); options.EnableJsonRedaction(); options.EnableLoggingRedaction(); options.EnableValidation(); options.EnableHealthChecks(); - - // Production-grade features (opt-in) - // options.EnableOutbox(); - // options.EnableDiagnostics(); - // options.EnableAuditStoreRetry(); - // options.EnableCachingTokenStore(); - // options.EnableRetention().EnableRetentionExecutor(); - // options.EnableDataSubjectExport().EnableDataSubjectErasure(); }); -``` -### 4. Wire your DbContext and middleware - -```csharp -// DbContext setup -builder.Services.AddDbContext((sp, options) => - options.UseSqlServer(connectionString) +// Wire the interceptor and middleware +builder.Services.AddDbContext((sp, opt) => + opt.UseSqlServer(connectionString) .AddInterceptors(sp.GetRequiredService())); var app = builder.Build(); - -// Place before UseAuthentication so the pseudonymized IP token -// is available throughout the request pipeline. app.UseSensitiveFlow(); - -// Map health checks (registers under /health by default) app.MapHealthChecks("/health"); ``` -That's it. Every `SaveChanges` on a field annotated with `[PersonalData]` or -`[SensitiveData]` now produces an `AuditRecord` automatically, HTTP responses are -JSON-redacted, logs are scrubbed, and health checks monitor the infrastructure. +### 3. See the magic -Schema is explicit: databases must already contain the app tables and the -SensitiveFlow audit/token/outbox tables before the first write. SensitiveFlow does -not create tables automatically, even in samples, because schema creation belongs -to migrations or deployment tooling. +Every time your code saves a customer, SensitiveFlow automatically: -The hosted audit outbox dispatcher does not create schema either. If its durable -table is missing, it logs the infrastructure failure and suspends polling by -default (`SuspendOnInfrastructureFailure = true`) instead of stopping the host. +- **Creates an audit record** showing who changed what field, when, and from what to what +- **Scrubs logs** — if a developer logs `email: {customer.Email}`, the actual email never appears in logs +- **Masks JSON responses** — API endpoints return `"email": "[REDACTED]"` instead of the real address +- **Tracks IP addresses** without storing them raw — instead stores a pseudonymized token +- **Validates retention policies** — knows when data should expire or be anonymized -The `WebApi.Sample` is intentionally different for DX: it creates its local -SQLite demo schema on startup so the UI and routes are immediately testable. Use -migrations or deployment-owned scripts in real applications. +No additional code needed. SensitiveFlow works at the infrastructure level. -### Optional: policies, reports, and diagnostics +### 4. Verify it works -```csharp -// Custom policy overrides -builder.Services.AddSensitiveFlowWeb(options => -{ - options.UseProfile(SensitiveFlowProfile.Strict); - // ... store config, features ... -}); +Check the `/health` endpoint to ensure your audit and token infrastructure are connected: -// Discovery report (CLI or at startup) -var report = SensitiveDataDiscovery.Scan(typeof(Customer).Assembly); -File.WriteAllText("sensitiveflow-report.md", report.ToMarkdown()); +```bash +curl http://localhost:5000/health +``` -// Startup validation -var startupReport = app.Services.ValidateSensitiveFlow(); +Success means your audit database is reachable and healthy. + +### Schema setup + +SensitiveFlow does not create tables automatically. Before your first write, create: + +- Your application tables (via EF Core migrations) +- SensitiveFlow audit/token tables (via [SQL scripts](tools/migrations/) or EF Core) + +See [Database Providers](docs/database-providers.md) for per-provider setup and schema details. The samples create schema automatically on startup for convenience; production apps should use migrations or deployment scripts. + +## What You Get + +| Feature | Example | Why | +|---------|---------|-----| +| **Audit trail** | `AuditRecord: customer.Email changed from john@old.com → john@new.com by UserId=abc at 2025-06-15T10:30:00Z` | Know who changed sensitive data and when; required for GDPR/LGPD investigation | +| **Log redaction** | `logger.LogInformation("User: {Name}, Email: {Email}", name, email)` → logs show `Email: [REDACTED]` | Prevent sensitive data in logs, error messages, or monitoring streams | +| **JSON masking** | `GET /api/customer/123` → `{"name":"John","email":"[REDACTED]","salary":"[REDACTED]"}` | API responses never leak sensitive fields even if you forget to scrub them | +| **IP pseudonymization** | IP `203.0.113.42` → token `ip_token_xyz` in logs; recoverable via token store | Audit shows request origin without storing raw IPs in plaintext | +| **Retention policies** | `[RetentionData(Years=5, AnonymizeOnExpiration)]` → field anonymized after 5 years automatically | Comply with "don't keep data longer than needed" requirement | +| **Data export** | Single API call to list all data associated with a user | GDPR/LGPD "right to access" — user can request and download their data | +| **Data erasure** | Schedule erasure or call API to fully delete user's data | GDPR/LGPD "right to be forgotten" — clean removal from all tables | + +## Going Deeper + +Once the basics work, explore: + +- **[Durable Outbox](docs/audit.md#outbox)** — guaranteed audit delivery even if database is temporarily unavailable +- **[Retry & Backoff](docs/audit.md#retry)** — automatic exponential backoff with jitter for transient failures +- **[Profiles](docs/policies.md)** — switch from `Balanced` to `Strict` (redact everything) or `Permissive` (trust some fields) +- **[Custom policies](docs/policies.md)** — define your own classification rules +- **[OpenTelemetry](docs/diagnostics.md)** — emit audit events as spans and metrics for observability +- **[Caching token store](docs/anonymization.md#caching)** — speed up pseudonymization lookups with in-memory cache +- **[Data subject reports](docs/anonymization.md#export)** — customize what fields appear in privacy reports + +See the [full documentation](docs/README.md) for all features. + +## Samples + +The repository includes runnable examples. After cloning: + +```bash +cd samples/QuickStart.Sample +dotnet run + +cd ../WebApi.Sample +dotnet run ``` -### Defaults - -| Area | Default | -|------|---------| -| Profile | `SensitiveFlowProfile.Balanced` | -| JSON redaction mode | `JsonRedactionMode.Mask` | -| Redaction marker | `[REDACTED]` | -| Logging | `[Sensitive]` markers redacted; annotated members redacted unless `MaskInLogs()` | -| Retention anonymization | `[ANONYMIZED]` | -| Audit outbox (when enabled) | `PollInterval=1s`, `BatchSize=100`, `MaxAttempts=5`, `BackoffStrategy=Exponential` | -| Outbox infrastructure failures | Dispatcher suspends polling by default with `SuspendOnInfrastructureFailure=true` | -| Data-subject export | Raw annotated values by default; use `[Redaction(Export = ...)]` to override | -| Health checks | `sensitiveflow-audit-store`, `sensitiveflow-token-store`, `sensitiveflow-audit-outbox` | - -### Advanced: fine-grained package-by-package setup - -The composition layer is the recommended path. If you need precise control over every -registration, install individual packages and register each service manually. See the -[Advanced Composition](docs/package-reference.md) guide for the full package-by-package -setup matrix and individual registration calls. - -> **Do not use in-memory stores in production.** Audit records and token mappings must -> survive process restarts. Losing audit history defeats accountability; losing token -> mappings makes pseudonymized data irrecoverable. +Each sample demonstrates different configurations: QuickStart keeps it minimal, WebApi shows outbox and diagnostics, MinimalApi shows routing integration, Console shows standalone usage, Redis shows distributed token store. + +## Advanced: Package-by-package setup + +The composition layer (`SensitiveFlow.AspNetCore.EFCore`) is recommended. For teams needing fine-grained control over which packages to install: + +See [Package Reference](docs/package-reference.md) for the full setup matrix and manual registration calls for each service. + +> **Do not use in-memory stores in production.** Audit records and token mappings must survive process restarts. ## Documentation diff --git a/RELEASE.md b/RELEASE.md index 3be247e..05287f6 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -10,20 +10,23 @@ While in preview (pre-v1), the version format is: MAJOR.MINOR.PATCH-preview.N ``` -| Version | Meaning | -|---------|---------| -| `1.0.0-preview.1` | First preview with initial feature set | -| `1.0.0-preview.2` | Bug fixes, metadata, or doc updates — no API changes | -| `1.0.0-preview.3` | New features or minor API changes | -| `1.0.0` | First stable release | +| Version | Meaning | +|-------------------|--------------------------------------------------------| +| `1.0.0-preview.1` | First preview with initial feature set | +| `1.0.0-preview.2` | Bug fixes, metadata, or doc updates — no API changes | +| `1.0.0-preview.3` | Hardening: DB compatibility, fail-fast validation, DX | +| `1.0.0` | First stable release | **While in preview:** -- Bug fixes / metadata / docs → bump preview number (e.g., `1.0.0-preview.1` → `1.0.0-preview.2`) -- New features or breaking changes → bump preview number (e.g., `1.0.0-preview.2` → `1.0.0-preview.3`) +- Bug fixes / metadata / docs → bump preview number (e.g., `preview.1` → `preview.2`) +- New features or breaking changes → bump preview number (e.g., `preview.2` → `preview.3`) - Do **not** republish the same version — NuGet rejects overwrites ## How to publish a release +The examples below use `` as a placeholder for the version you're shipping +(e.g. `1.0.0` for the stable release or `1.0.1` for the next patch). + ### 1. Ensure everything is merged to `main` ```bash @@ -36,7 +39,7 @@ git pull origin main Edit `Directory.Build.props`: ```xml -1.0.0-preview.2 + ``` ### 3. Update CHANGELOG.md @@ -47,23 +50,23 @@ Move changes from `[Unreleased]` to the new version section. ```bash git add Directory.Build.props CHANGELOG.md -git commit -m "release: v1.0.0-preview.2" -git tag v1.0.0-preview.2 +git commit -m "release: v" +git tag v git push origin main -git push origin v1.0.0-preview.2 +git push origin v ``` ### 5. Create GitHub Release 1. Go to **Releases → Draft a new release** -2. Choose the tag: `v1.0.0-preview.2` -3. Title: `SensitiveFlow v1.0.0-preview.2` +2. Choose the tag: `v` +3. Title: `SensitiveFlow v` 4. Description: copy the relevant section from `CHANGELOG.md` -5. Check **"Set as a pre-release"** (while in preview) +5. Check **"Set as a pre-release"** *only* if the tag still carries a `-preview.N` suffix 6. Publish The `release.yml` workflow will automatically build, test (including container tests), pack, and push to NuGet. ## Current version -`1.0.0-preview.1` — already published. Next will be `1.0.0-preview.2`. +`1.0.0-preview.3` — latest published preview. Next release: `1.0.0` (first stable). diff --git a/docs/analyzers.md b/docs/analyzers.md index eefcd70..670a211 100644 --- a/docs/analyzers.md +++ b/docs/analyzers.md @@ -14,7 +14,7 @@ dotnet add package SensitiveFlow.Analyzers In application projects, keep analyzers private to that project: ```xml - + ``` During local development of this repository, reference the analyzer project as an diff --git a/samples/Console.Sample/Console.Sample.csproj b/samples/Console.Sample/Console.Sample.csproj index e998eeb..7c511d1 100644 --- a/samples/Console.Sample/Console.Sample.csproj +++ b/samples/Console.Sample/Console.Sample.csproj @@ -3,6 +3,7 @@ Exe net10.0 + enable enable false diff --git a/samples/MinimalApi.Sample/MinimalApi.Sample.csproj b/samples/MinimalApi.Sample/MinimalApi.Sample.csproj index f87ca7c..c00921b 100644 --- a/samples/MinimalApi.Sample/MinimalApi.Sample.csproj +++ b/samples/MinimalApi.Sample/MinimalApi.Sample.csproj @@ -2,6 +2,7 @@ net10.0 + enable enable false diff --git a/samples/QuickStart.Sample/QuickStart.Sample.csproj b/samples/QuickStart.Sample/QuickStart.Sample.csproj index efd7e1e..a2630c0 100644 --- a/samples/QuickStart.Sample/QuickStart.Sample.csproj +++ b/samples/QuickStart.Sample/QuickStart.Sample.csproj @@ -2,6 +2,7 @@ net10.0 + enable enable false diff --git a/samples/Redis.Sample/Redis.Sample.csproj b/samples/Redis.Sample/Redis.Sample.csproj index 29851ba..94afdd8 100644 --- a/samples/Redis.Sample/Redis.Sample.csproj +++ b/samples/Redis.Sample/Redis.Sample.csproj @@ -3,6 +3,7 @@ Exe net10.0 + enable enable false diff --git a/samples/WebApi.Sample/WebApi.Sample.csproj b/samples/WebApi.Sample/WebApi.Sample.csproj index 3a40c98..c5637d9 100644 --- a/samples/WebApi.Sample/WebApi.Sample.csproj +++ b/samples/WebApi.Sample/WebApi.Sample.csproj @@ -2,6 +2,7 @@ net10.0 + enable enable false diff --git a/src/SensitiveFlow.Analyzers/AnalyzerReleases.Shipped.md b/src/SensitiveFlow.Analyzers/AnalyzerReleases.Shipped.md index 8b13789..60a7e7b 100644 --- a/src/SensitiveFlow.Analyzers/AnalyzerReleases.Shipped.md +++ b/src/SensitiveFlow.Analyzers/AnalyzerReleases.Shipped.md @@ -1 +1,10 @@ +## Release 1.0.0 +### New Rules + +Rule ID | Category | Severity | Notes +--------|----------|----------|------ +SF0001 | Privacy | Warning | Sensitive member logged directly without sanitization. +SF0002 | Privacy | Warning | Sensitive member returned directly in HTTP response. +SF0003 | Privacy | Warning | Entity with sensitive data missing DataSubjectId/UserId. +SF0004 | Privacy | Info | Property name suggests PII but is not annotated. diff --git a/src/SensitiveFlow.Analyzers/AnalyzerReleases.Unshipped.md b/src/SensitiveFlow.Analyzers/AnalyzerReleases.Unshipped.md index 2d4aa08..cbe0609 100644 --- a/src/SensitiveFlow.Analyzers/AnalyzerReleases.Unshipped.md +++ b/src/SensitiveFlow.Analyzers/AnalyzerReleases.Unshipped.md @@ -1,8 +1,2 @@ -### New Rules - -Rule ID | Category | Severity | Notes ---------|----------|----------|------ -SF0001 | Privacy | Warning | Sensitive member logged directly without sanitization. -SF0002 | Privacy | Warning | Sensitive member returned directly in HTTP response. -SF0003 | Privacy | Warning | Entity with sensitive data missing DataSubjectId/UserId. -SF0004 | Privacy | Info | Property name suggests PII but is not annotated. +; Shipped analyzer release tracking. New rules added to a release version go here, +; then move to AnalyzerReleases.Shipped.md once the release is published. From 71ac9c86a0b8d00172791805dd009c1590431092 Mon Sep 17 00:00:00 2001 From: Mateus Silva Date: Tue, 12 May 2026 07:56:00 -0300 Subject: [PATCH 17/23] feat: Refactor sample applications to manage employee data, update sensitive data categories, and enhance documentation --- CHANGELOG.md | 4 +- README.md | 20 +-- SECURITY.md | 2 +- docs/anonymization.md | 7 + docs/attributes.md | 7 +- docs/audit.md | 14 +- docs/backends-example.md | 2 +- docs/package-reference.md | 29 +++-- samples/Console.Sample/Program.cs | 2 +- .../Infrastructure/SampleDbContext.cs | 2 +- samples/QuickStart.Sample/AppDbContext.cs | 2 +- .../Controllers/CustomersController.cs | 121 +++++++++--------- .../Controllers/SensitiveFlowController.cs | 2 +- .../Infrastructure/SampleDbContext.cs | 17 ++- samples/WebApi.Sample/Program.cs | 59 +++++---- .../Enums/SensitiveDataCategory.cs | 6 + 16 files changed, 164 insertions(+), 132 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5bc20b..f9bf33a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security -- Durable outbox prevents audit record loss on process restart or failure — critical for compliance. +- Durable outbox prevents audit record loss on process restart or failure. - Transactional outbox pattern ensures audit writes and outbox enqueuing happen atomically. ## [1.0.0-preview.2] - 2026-05-10 @@ -44,7 +44,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `SECURITY.md` with supported versions table, vulnerability reporting process, and security scope. - `CODE_OF_CONDUCT.md` (Contributor Covenant 2.1). - `RELEASE.md` with full versioning scheme and publishing process. -- Legal disclaimer in README: "SensitiveFlow helps reduce accidental exposure of sensitive data, but it does not guarantee legal compliance or complete data protection by itself." +- Disclaimer in README: "SensitiveFlow is a tool to help you manage sensitive data safely, not a guarantee of complete data protection by itself." - New README badges: Container Tests, NuGet Downloads, .NET 8 | 9 | 10. ### Changed diff --git a/README.md b/README.md index 3b2e48b..eff5f71 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,9 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) [![.NET](https://img.shields.io/badge/.NET-8%20%7C%209%20%7C%2010-blue)](https://dotnet.microsoft.com) -**SensitiveFlow** is a .NET library that brings observability and control to sensitive data through automatic auditing, log redaction, JSON masking, and pseudonymization. Mark your data once, get automatic protection everywhere. +**SensitiveFlow** is a .NET library that brings observability and control to sensitive data flows through automatic auditing, log redaction, JSON masking, and pseudonymization. Mark your data once, get automatic protection everywhere. -> **Important:** SensitiveFlow helps reduce accidental exposure of sensitive data, but it does not guarantee legal compliance or complete data protection by itself. You are responsible for how you use these primitives in your application. +> **Important:** SensitiveFlow is a tool to help you manage sensitive data safely, not a guarantee of complete data protection by itself. You are responsible for how you use these primitives in your application and for ensuring they meet your requirements. ## Why SensitiveFlow? @@ -151,13 +151,13 @@ See [Database Providers](docs/database-providers.md) for per-provider setup and | Feature | Example | Why | |---------|---------|-----| -| **Audit trail** | `AuditRecord: customer.Email changed from john@old.com → john@new.com by UserId=abc at 2025-06-15T10:30:00Z` | Know who changed sensitive data and when; required for GDPR/LGPD investigation | -| **Log redaction** | `logger.LogInformation("User: {Name}, Email: {Email}", name, email)` → logs show `Email: [REDACTED]` | Prevent sensitive data in logs, error messages, or monitoring streams | -| **JSON masking** | `GET /api/customer/123` → `{"name":"John","email":"[REDACTED]","salary":"[REDACTED]"}` | API responses never leak sensitive fields even if you forget to scrub them | -| **IP pseudonymization** | IP `203.0.113.42` → token `ip_token_xyz` in logs; recoverable via token store | Audit shows request origin without storing raw IPs in plaintext | -| **Retention policies** | `[RetentionData(Years=5, AnonymizeOnExpiration)]` → field anonymized after 5 years automatically | Comply with "don't keep data longer than needed" requirement | -| **Data export** | Single API call to list all data associated with a user | GDPR/LGPD "right to access" — user can request and download their data | -| **Data erasure** | Schedule erasure or call API to fully delete user's data | GDPR/LGPD "right to be forgotten" — clean removal from all tables | +| **Audit trail** | `AuditRecord: salary changed from $50k → $60k by manager@company.com at 2025-06-15T10:30:00Z` | Know who changed sensitive data and when; essential for investigation and accountability | +| **Log redaction** | `logger.LogInformation("Processing user: {Email}", email)` → logs show `Email: [REDACTED]` | Prevent accidental leaks in logs, error messages, monitoring systems, or backups | +| **JSON masking** | `GET /api/employee/123` → `{"name":"John","salary":"[REDACTED]","ssn":"[REDACTED]"}` | API responses never expose sensitive fields even if you forget to scrub them in code | +| **IP pseudonymization** | IP `203.0.113.42` → token `ip_token_xyz` in audit logs; recoverable via token store | Audit shows request origin without storing raw IPs in plaintext | +| **Retention policies** | `[RetentionData(Years=7, AnonymizeOnExpiration)]` → field automatically anonymized after 7 years | Control how long sensitive data stays in your systems; delete when no longer needed | +| **Data export** | Single API call to collect all data associated with a user | Generate comprehensive reports for users; useful for transparency and debugging | +| **Data erasure** | Schedule erasure or call API to fully remove user's data from all tables | Clean removal of user data when requested; useful for testing and cleanup | ## Going Deeper @@ -204,7 +204,7 @@ See [Package Reference](docs/package-reference.md) for the full setup matrix and ## Design Principles -- **Runtime behavior over compliance paperwork** -- instruments what actually happens, not what should happen. +- **Observable behavior over aspirational goals** -- instruments what actually happens with sensitive data, not what should happen in theory. - **Explicit metadata over implicit heuristics** -- every classification is opt-in via attributes. - **Composition over lock-in** -- each module is optional and independently testable. - **Safe defaults** -- the IP address is never stored raw; the log redactor strips sensitive values before they reach any sink. diff --git a/SECURITY.md b/SECURITY.md index 2d3dacb..7f940b7 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -42,4 +42,4 @@ Security-sensitive areas include: ## Non-goals -SensitiveFlow helps reduce accidental exposure of sensitive data, but it does not guarantee legal compliance or complete data protection by itself. You are responsible for how you use these primitives in your application. +SensitiveFlow helps reduce accidental exposure of sensitive data, but it is a tool, not a guarantee of complete data protection by itself. You are responsible for how you use these primitives in your application and for ensuring the configuration meets your requirements. diff --git a/docs/anonymization.md b/docs/anonymization.md index ab6880a..0e61941 100644 --- a/docs/anonymization.md +++ b/docs/anonymization.md @@ -249,6 +249,13 @@ public string InternalRiskNote { get; set; } = string.Empty; `IDataSubjectExporter` is the read-side counterpart of `IDataSubjectErasureService`. Given an entity, it returns a dictionary keyed by property name with every annotated value (`[PersonalData]`, `[SensitiveData]`, `[RetentionData]`) — useful for satisfying portability requests where a user asks for a copy of the personal data the application holds about them. +SensitiveFlow provides two interfaces for export: + +- **`IDataSubjectExporter`** (Core) — generic interface you can implement for custom type-specific export logic +- **`IDataSubjectExporter`** (Anonymization) — concrete non-generic implementation that works with any `object` + +Use the non-generic version for simplicity (it reflects over the object at runtime): + ```csharp services.AddDataSubjectExport(); diff --git a/docs/attributes.md b/docs/attributes.md index bbb3e6d..271f2c5 100644 --- a/docs/attributes.md +++ b/docs/attributes.md @@ -22,9 +22,9 @@ public string Email { get; set; } | `Identification` | Name, CPF, RG, passport | | `Contact` | Email, phone, address | | `Financial` | Bank account, income | +| `JudicialData` | Court records, legal proceedings | | `Behavioral` | Browsing history, preferences | | `Location` | GPS, IP (raw), address | -| `Professional` | Employer, job title | | `Other` | Anything not covered above | ## SensitiveDataAttribute @@ -74,9 +74,10 @@ public string ContractData { get; set; } | Value | Description | |-------|-------------| -| `AnonymizeOnExpiration` | Replace the value with an anonymized placeholder | +| `AnonymizeOnExpiration` | Replace the value with an anonymized placeholder (mask/tokenize) | | `DeleteOnExpiration` | Remove the record or nullify the field | -| `BlockOnExpiration` | Block further processing of the field | +| `BlockOnExpiration` | Prevent further access or processing of the field when expired | +| `NotifyOwner` | Trigger a notification workflow when data expires (custom handler required) | ### Calendar-accurate arithmetic diff --git a/docs/audit.md b/docs/audit.md index 58af3f1..f2dd30f 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -29,13 +29,15 @@ public sealed class AuditRecord | Value | Description | |-------|-------------| -| `Access` | Field was read | +| `Access` | Field was read or accessed | | `Create` | Record was inserted | | `Update` | Field value was changed | | `Delete` | Record was removed | -| `Export` | Data was exported | -| `Anonymize` | Data was anonymized | -| `Pseudonymize` | Data was pseudonymized | +| `Export` | Data was exported for subject access requests | +| `Anonymize` | Data was anonymized or pseudonymized | +| `Pseudonymize` | Data was pseudonymized (tokenized) | +| `Share` | Data was shared with a third party | +| `Revoke` | Data sharing was revoked or consent withdrawn | ## IAuditStore @@ -155,7 +157,7 @@ public string InternalNote { get; set; } = string.Empty; ## Audit outbox -`SensitiveFlow.Audit` ships a concrete in-memory outbox for tests/local development. Production systems must use a durable audit outbox to ensure reliable delivery to downstream systems (e.g., SIEM, compliance dashboards, data lakes). +`SensitiveFlow.Audit` ships a concrete in-memory outbox for tests/local development. Production systems must use a durable audit outbox to ensure reliable delivery to downstream systems (e.g., SIEM, data lakes, audit dashboards). ### In-memory outbox (tests only) @@ -164,7 +166,7 @@ builder.Services.AddAuditStore(); builder.Services.AddInMemoryAuditOutbox(); // Deprecated – use for tests/dev only ``` -⚠️ `InMemoryAuditOutbox` is **deprecated for production**. It loses all enqueued records on process restart and is not suitable for compliance/audit scenarios. The `SensitiveFlowConfigurationValidator` will emit `SF-CONFIG-013` if it detects an in-memory outbox outside a Development environment. +⚠️ `InMemoryAuditOutbox` is **deprecated for production**. It loses all enqueued records on process restart. The `SensitiveFlowConfigurationValidator` will emit `SF-CONFIG-013` if it detects an in-memory outbox outside a Development environment. ### Durable outbox with EF Core (production-ready) diff --git a/docs/backends-example.md b/docs/backends-example.md index 07e11bb..7e59594 100644 --- a/docs/backends-example.md +++ b/docs/backends-example.md @@ -349,7 +349,7 @@ builder.Services.AddSensitiveFlowEFCore(); - Redis: `EXPIRE` on keys - DynamoDB: `TTL` attribute - MongoDB: TTL index -- Ensures compliance with retention policies +- Enables enforcement of retention policies ### 4. **Partition by DataSubjectId (Optional)** - Improves query performance for data-subject access and erasure requests diff --git a/docs/package-reference.md b/docs/package-reference.md index 59a6e38..9efd7eb 100644 --- a/docs/package-reference.md +++ b/docs/package-reference.md @@ -10,7 +10,7 @@ This document summarizes each SensitiveFlow package individually: purpose, prima | `SensitiveFlow.Core` | You annotate models or implement contracts. | Add attributes to models. | None by itself; it does not enforce behavior. | | `SensitiveFlow.Audit` | You register custom audit stores or decorators. | `AddAuditStore()` or first-party EF store plus optional retry/buffer. | In-memory/buffered data can be lost before durable write. | | `SensitiveFlow.Audit.EFCore` | You want first-party SQL audit storage. | `AddEfCoreAuditStore(...)`; create/migrate audit table. | Audit DB must be durable and backed up independently. | -| `SensitiveFlow.Audit.EFCore.Outbox` | You need durable, reliable delivery of audit records to downstream systems (SIEM, data lakes, compliance dashboards). | `AddEfCoreAuditOutbox()` and register `IAuditOutboxPublisher` implementations. | Outbox table must be monitored for dead-lettered entries (failed delivery after max retries). | +| `SensitiveFlow.Audit.EFCore.Outbox` | You need durable, reliable delivery of audit records to downstream systems (SIEM, data lakes, audit dashboards). | `AddEfCoreAuditOutbox()` and register `IAuditOutboxPublisher` implementations. | Outbox table must be monitored for dead-lettered entries (failed delivery after max retries). | | `SensitiveFlow.Audit.Snapshots.EFCore` | You want durable aggregate-level audit snapshots. | `AddEfCoreAuditSnapshotStore(...)`; create/migrate snapshot table. | Snapshots can be large; monitor storage growth. | | `SensitiveFlow.TokenStore.EFCore` | You want first-party SQL token storage for reversible pseudonymization. | `AddEfCoreTokenStore(...)`; create/migrate token table. | Losing token mappings makes pseudonymized data irrecoverable. | | `SensitiveFlow.EFCore` | You want automatic audit on `SaveChanges`. | `AddSensitiveFlowEFCore()` and `AddInterceptors(...)`. | Missing interceptor means no automatic audit. | @@ -37,17 +37,18 @@ Primary APIs: - `AddSensitiveFlowWeb(options => { ... })` - `UseSensitiveFlow()` (middleware — wraps `UseSensitiveFlowAudit()`) - `SensitiveFlowWebOptions` (fluent builder) - - `UseProfile(SensitiveFlowProfile profile)` - - `UseEfCoreStores(configureAuditStore, configureTokenStore)` (provider-agnostic shorthand) - - `UseEfCoreAuditStore(Action)` - - `UseEfCoreTokenStore(Action)` - - `EnableOutbox()`, `EnableDiagnostics()`, `EnableAuditStoreRetry()` - - `EnableCachingTokenStore()` - - `EnableDataSubjectExport()`, `EnableDataSubjectErasure()` - - `EnableLoggingRedaction()`, `EnableJsonRedaction()` - - `EnableEfCoreAudit()`, `EnableAspNetCoreContext()` - - `EnableValidation()`, `EnableHealthChecks()` - - `EnableRetention()`, `EnableRetentionExecutor()` + - `UseProfile(SensitiveFlowProfile profile)` — set security profile (Permissive, Balanced, Strict) + - `ConfigurePolicies(Action)` — override individual category policies beyond the profile + - `UseEfCoreStores(configureAuditStore, configureTokenStore)` — provider-agnostic shorthand for both stores + - `UseEfCoreAuditStore(Action)` — explicitly configure audit DB + - `UseEfCoreTokenStore(Action)` — explicitly configure token DB + - `EnableOutbox()`, `EnableDiagnostics()`, `EnableAuditStoreRetry()` — outbox/observability + - `EnableCachingTokenStore()` — in-memory cache for pseudonymization lookups + - `EnableDataSubjectExport()`, `EnableDataSubjectErasure()` — data subject rights + - `EnableLoggingRedaction()`, `EnableJsonRedaction()` — output protection + - `EnableEfCoreAudit()`, `EnableAspNetCoreContext()` — audit source enrichment + - `EnableValidation()`, `EnableHealthChecks()` — runtime checks + - `EnableRetention()`, `EnableRetentionExecutor()` — data lifecycle management Install when: @@ -188,7 +189,7 @@ Operational notes: Purpose: -- First-party EF Core-backed **durable audit outbox** for reliable, transactional delivery of audit records to downstream systems (SIEM, data lakes, compliance dashboards, Kafka, webhooks, etc.). +- First-party EF Core-backed **durable audit outbox** for reliable, transactional delivery of audit records to downstream systems (SIEM, data lakes, audit dashboards, Kafka, webhooks, etc.). Primary APIs: @@ -202,7 +203,7 @@ Primary APIs: Install when: - You need **guaranteed, at-least-once delivery** of audit records to a remote system. -- Your business/compliance requirements demand that no audit record is lost if the application crashes. +- Your business requirements demand that no audit record is lost if the application crashes. Recommended setup: diff --git a/samples/Console.Sample/Program.cs b/samples/Console.Sample/Program.cs index 682887d..cf00b87 100644 --- a/samples/Console.Sample/Program.cs +++ b/samples/Console.Sample/Program.cs @@ -231,7 +231,7 @@ public sealed class Customer [PersonalData(Category = DataCategory.Contact)] public string Email { get; set; } = string.Empty; - [SensitiveData(Category = SensitiveDataCategory.Other)] + [SensitiveData(Category = SensitiveDataCategory.Financial)] [RetentionData(Years = 5, Policy = RetentionPolicy.AnonymizeOnExpiration)] public string TaxId { get; set; } = string.Empty; diff --git a/samples/MinimalApi.Sample/Infrastructure/SampleDbContext.cs b/samples/MinimalApi.Sample/Infrastructure/SampleDbContext.cs index eb43923..2d331c8 100644 --- a/samples/MinimalApi.Sample/Infrastructure/SampleDbContext.cs +++ b/samples/MinimalApi.Sample/Infrastructure/SampleDbContext.cs @@ -15,7 +15,7 @@ public sealed class Customer [PersonalData(Category = DataCategory.Contact)] public string Email { get; set; } = string.Empty; - [SensitiveData(Category = SensitiveDataCategory.Other)] + [SensitiveData(Category = SensitiveDataCategory.Financial)] [RetentionData(Years = 5, Policy = RetentionPolicy.AnonymizeOnExpiration)] public string TaxId { get; set; } = string.Empty; diff --git a/samples/QuickStart.Sample/AppDbContext.cs b/samples/QuickStart.Sample/AppDbContext.cs index 2dbc717..2e073f6 100644 --- a/samples/QuickStart.Sample/AppDbContext.cs +++ b/samples/QuickStart.Sample/AppDbContext.cs @@ -15,7 +15,7 @@ public sealed class Customer [PersonalData(Category = DataCategory.Contact)] public string Email { get; set; } = string.Empty; - [SensitiveData(Category = SensitiveDataCategory.Other)] + [SensitiveData(Category = SensitiveDataCategory.Financial)] public string TaxId { get; set; } = string.Empty; public DateTimeOffset CreatedAt { get; set; } diff --git a/samples/WebApi.Sample/Controllers/CustomersController.cs b/samples/WebApi.Sample/Controllers/CustomersController.cs index 41f1cbc..9b019be 100644 --- a/samples/WebApi.Sample/Controllers/CustomersController.cs +++ b/samples/WebApi.Sample/Controllers/CustomersController.cs @@ -12,8 +12,8 @@ namespace WebApi.Sample.Controllers; [ApiController] -[Route("customers")] -public sealed class CustomersController : ControllerBase +[Route("employees")] +public sealed class EmployeesController : ControllerBase { private readonly SampleDbContext _db; private readonly IAuditStore _auditStore; @@ -21,16 +21,16 @@ public sealed class CustomersController : ControllerBase private readonly IDataSubjectExporter _exporter; private readonly IDataSubjectErasureService _erasure; private readonly RetentionExecutor _retention; - private readonly ILogger _logger; + private readonly ILogger _logger; - public CustomersController( + public EmployeesController( SampleDbContext db, IAuditStore auditStore, IAuditContext auditContext, IDataSubjectExporter exporter, IDataSubjectErasureService erasure, RetentionExecutor retention, - ILogger logger) + ILogger logger) { _db = db; _auditStore = auditStore; @@ -44,126 +44,128 @@ public CustomersController( [HttpGet] public async Task List(CancellationToken ct) { - var customers = await _db.Customers + var employees = await _db.Employees .Take(100) .ToListAsync(ct); - return Ok(customers + return Ok(employees .OrderByDescending(c => c.CreatedAt) .Take(20) .Select(ToResponse)); } [HttpPost] - public async Task Create([FromBody] CreateCustomerRequest request, CancellationToken ct) + public async Task Create([FromBody] CreateEmployeeRequest request, CancellationToken ct) { - var customer = new Customer + var employee = new Employee { DataSubjectId = Guid.NewGuid().ToString(), - Name = request.Name, + FullName = request.FullName, Email = request.Email, - TaxId = request.TaxId, Phone = request.Phone, + AnnualSalary = request.AnnualSalary, + Department = request.Department, CreatedAt = DateTimeOffset.UtcNow, }; - _db.Customers.Add(customer); + _db.Employees.Add(employee); await _db.SaveChangesAsync(ct); - _logger.LogInformation("Customer created - name: {Name}, email: {[Sensitive]Email}", - customer.Name.MaskName(), - customer.Email.MaskEmail()); + _logger.LogInformation("Employee created - name: {Name}, email: {[Sensitive]Email}, salary: {[Sensitive]Salary}", + employee.FullName.MaskName(), + employee.Email.MaskEmail(), + employee.AnnualSalary.ToString().MaskEmail()); - return CreatedAtAction(nameof(Get), new { id = customer.DataSubjectId }, ToResponse(customer)); + return CreatedAtAction(nameof(Get), new { id = employee.DataSubjectId }, ToResponse(employee)); } [HttpGet("{id}")] public async Task Get(string id, CancellationToken ct) { - var customer = await FindCustomerAsync(id, ct); + var employee = await FindEmployeeAsync(id, ct); - if (customer is null) + if (employee is null) { return NotFound(); } await _auditStore.AppendAsync(new AuditRecord { - DataSubjectId = customer.DataSubjectId, - Entity = nameof(Customer), + DataSubjectId = employee.DataSubjectId, + Entity = nameof(Employee), Field = "*", Operation = AuditOperation.Access, ActorId = _auditContext.ActorId, IpAddressToken = _auditContext.IpAddressToken, }, ct); - _logger.LogInformation("Customer {Id} accessed - email: {[Sensitive]Email}", - id.SanitizeForLog(), customer.Email.MaskEmail()); + _logger.LogInformation("Employee {Id} accessed - email: {[Sensitive]Email}", + id.SanitizeForLog(), employee.Email.MaskEmail()); - return Ok(ToResponse(customer)); + return Ok(ToResponse(employee)); } [HttpGet("{id}/audit")] public async Task GetAudit(string id, CancellationToken ct) { - var customer = await FindCustomerAsync(id, ct); - if (customer is null) + var employee = await FindEmployeeAsync(id, ct); + if (employee is null) { return NotFound(); } - var records = await _auditStore.QueryByDataSubjectAsync(customer.DataSubjectId, cancellationToken: ct); + var records = await _auditStore.QueryByDataSubjectAsync(employee.DataSubjectId, cancellationToken: ct); return Ok(records); } [HttpGet("{id}/json")] public async Task GetWithJsonRedaction(string id, CancellationToken ct) { - var customer = await FindCustomerAsync(id, ct); + var employee = await FindEmployeeAsync(id, ct); - return customer is null ? NotFound() : Ok(customer); + return employee is null ? NotFound() : Ok(employee); } [HttpGet("{id}/export")] public async Task Export(string id, CancellationToken ct) { - var customer = await FindCustomerAsync(id, ct, asNoTracking: true); + var employee = await FindEmployeeAsync(id, ct, asNoTracking: true); - if (customer is null) + if (employee is null) { return NotFound(); } await _auditStore.AppendAsync(new AuditRecord { - DataSubjectId = customer.DataSubjectId, - Entity = nameof(Customer), + DataSubjectId = employee.DataSubjectId, + Entity = nameof(Employee), Field = "*", Operation = AuditOperation.Export, ActorId = _auditContext.ActorId, IpAddressToken = _auditContext.IpAddressToken, }, ct); - return Ok(_exporter.Export(customer)); + return Ok(_exporter.Export(employee)); } [HttpPost("{id}/erase")] public async Task Erase(string id, CancellationToken ct) { - var customer = await FindCustomerAsync(id, ct); + var employee = await FindEmployeeAsync(id, ct); - if (customer is null) + if (employee is null) { return NotFound(); } - var changed = _erasure.Erase(customer); + var changed = _erasure.Erase(employee); await _db.SaveChangesAsync(ct); await _auditStore.AppendAsync(new AuditRecord { - DataSubjectId = customer.DataSubjectId, - Entity = nameof(Customer), + DataSubjectId = employee.DataSubjectId, + Entity = nameof(Employee), Field = "*", Operation = AuditOperation.Anonymize, ActorId = _auditContext.ActorId, @@ -177,10 +179,10 @@ await _auditStore.AppendAsync(new AuditRecord [HttpPost("/retention/run")] public async Task RunRetention(CancellationToken ct) { - var customers = await _db.Customers.ToListAsync(ct); + var employees = await _db.Employees.ToListAsync(ct); var report = await _retention.ExecuteAsync( - customers, - entity => ((Customer)entity).CreatedAt, + employees, + entity => ((Employee)entity).CreatedAt, ct); await _db.SaveChangesAsync(ct); @@ -201,10 +203,10 @@ public async Task RunRetention(CancellationToken ct) [HttpPost("/retention/dry-run")] public async Task DryRunRetention(CancellationToken ct) { - var customers = await _db.Customers.AsNoTracking().ToListAsync(ct); + var employees = await _db.Employees.AsNoTracking().ToListAsync(ct); var report = await _retention.DryRunAsync( - customers, - entity => ((Customer)entity).CreatedAt, + employees, + entity => ((Employee)entity).CreatedAt, ct); return Ok(new @@ -220,29 +222,32 @@ public async Task DryRunRetention(CancellationToken ct) }); } - private static CustomerResponse ToResponse(Customer c) => new( - c.DataSubjectId, - c.Name.MaskName(), - c.Email.MaskEmail(), - c.Phone.MaskPhone()); + private static EmployeeResponse ToResponse(Employee e) => new( + e.DataSubjectId, + e.FullName.MaskName(), + e.Email.MaskEmail(), + e.Phone.MaskPhone(), + e.AnnualSalary); - private Task FindCustomerAsync(string id, CancellationToken ct, bool asNoTracking = false) + private Task FindEmployeeAsync(string id, CancellationToken ct, bool asNoTracking = false) { - var query = asNoTracking ? _db.Customers.AsNoTracking() : _db.Customers; + var query = asNoTracking ? _db.Employees.AsNoTracking() : _db.Employees; return int.TryParse(id, out var numericId) ? query.FirstOrDefaultAsync(c => c.Id == numericId, ct) : query.FirstOrDefaultAsync(c => c.DataSubjectId == id, ct); } } -public sealed record CreateCustomerRequest( - string Name, +public sealed record CreateEmployeeRequest( + string FullName, string Email, - string TaxId, - string Phone); + string Phone, + decimal AnnualSalary, + string Department); -public sealed record CustomerResponse( +public sealed record EmployeeResponse( string DataSubjectId, - string Name, + string FullName, string Email, - string Phone); + string Phone, + decimal AnnualSalary); diff --git a/samples/WebApi.Sample/Controllers/SensitiveFlowController.cs b/samples/WebApi.Sample/Controllers/SensitiveFlowController.cs index 5c699dc..0321b62 100644 --- a/samples/WebApi.Sample/Controllers/SensitiveFlowController.cs +++ b/samples/WebApi.Sample/Controllers/SensitiveFlowController.cs @@ -19,7 +19,7 @@ public SensitiveFlowController(IServiceProvider services) [HttpGet("discovery")] public ContentResult Discovery() { - var report = SensitiveDataDiscovery.Scan(typeof(Customer).Assembly); + var report = SensitiveDataDiscovery.Scan(typeof(Employee).Assembly); return Content(report.ToMarkdown(), "text/markdown"); } diff --git a/samples/WebApi.Sample/Infrastructure/SampleDbContext.cs b/samples/WebApi.Sample/Infrastructure/SampleDbContext.cs index 910cdf2..4e27689 100644 --- a/samples/WebApi.Sample/Infrastructure/SampleDbContext.cs +++ b/samples/WebApi.Sample/Infrastructure/SampleDbContext.cs @@ -4,24 +4,27 @@ namespace WebApi.Sample.Infrastructure; -public sealed class Customer +public sealed class Employee { public int Id { get; set; } public string DataSubjectId { get; set; } = string.Empty; [PersonalData(Category = DataCategory.Identification)] - public string Name { get; set; } = string.Empty; + public string FullName { get; set; } = string.Empty; [PersonalData(Category = DataCategory.Contact)] public string Email { get; set; } = string.Empty; - [SensitiveData(Category = SensitiveDataCategory.Other)] - [RetentionData(Years = 5, Policy = RetentionPolicy.AnonymizeOnExpiration)] - public string TaxId { get; set; } = string.Empty; - [PersonalData(Category = DataCategory.Contact)] public string Phone { get; set; } = string.Empty; + [SensitiveData(Category = SensitiveDataCategory.Financial)] + [RetentionData(Years = 7, Policy = RetentionPolicy.AnonymizeOnExpiration)] + public decimal AnnualSalary { get; set; } + + [SensitiveData(Category = SensitiveDataCategory.Other)] + public string Department { get; set; } = string.Empty; + public DateTimeOffset CreatedAt { get; set; } } @@ -29,5 +32,5 @@ public sealed class SampleDbContext : DbContext { public SampleDbContext(DbContextOptions options) : base(options) { } - public DbSet Customers => Set(); + public DbSet Employees => Set(); } diff --git a/samples/WebApi.Sample/Program.cs b/samples/WebApi.Sample/Program.cs index 21b4033..103118b 100644 --- a/samples/WebApi.Sample/Program.cs +++ b/samples/WebApi.Sample/Program.cs @@ -125,7 +125,7 @@ - SensitiveFlow Web API Sample + SensitiveFlow Employee Management Sample