Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions AgentExperience.NET.sln
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AgentExperience.Sample.EndT
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AgentExperience.Sample.EndToEnd.Tests", "tests\AgentExperience.Sample.EndToEnd.Tests\AgentExperience.Sample.EndToEnd.Tests.csproj", "{C93A6F14-8D27-4B50-A6E9-2F71B4D85C60}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AgentExperience.ReuseBaseline", "tests\AgentExperience.ReuseBaseline\AgentExperience.ReuseBaseline.csproj", "{53CECD06-F7D8-4A4D-85CE-BAEA7C35EBEF}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -201,6 +203,18 @@ Global
{C93A6F14-8D27-4B50-A6E9-2F71B4D85C60}.Release|x64.Build.0 = Release|Any CPU
{C93A6F14-8D27-4B50-A6E9-2F71B4D85C60}.Release|x86.ActiveCfg = Release|Any CPU
{C93A6F14-8D27-4B50-A6E9-2F71B4D85C60}.Release|x86.Build.0 = Release|Any CPU
{53CECD06-F7D8-4A4D-85CE-BAEA7C35EBEF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{53CECD06-F7D8-4A4D-85CE-BAEA7C35EBEF}.Debug|Any CPU.Build.0 = Debug|Any CPU
{53CECD06-F7D8-4A4D-85CE-BAEA7C35EBEF}.Debug|x64.ActiveCfg = Debug|Any CPU
{53CECD06-F7D8-4A4D-85CE-BAEA7C35EBEF}.Debug|x64.Build.0 = Debug|Any CPU
{53CECD06-F7D8-4A4D-85CE-BAEA7C35EBEF}.Debug|x86.ActiveCfg = Debug|Any CPU
{53CECD06-F7D8-4A4D-85CE-BAEA7C35EBEF}.Debug|x86.Build.0 = Debug|Any CPU
{53CECD06-F7D8-4A4D-85CE-BAEA7C35EBEF}.Release|Any CPU.ActiveCfg = Release|Any CPU
{53CECD06-F7D8-4A4D-85CE-BAEA7C35EBEF}.Release|Any CPU.Build.0 = Release|Any CPU
{53CECD06-F7D8-4A4D-85CE-BAEA7C35EBEF}.Release|x64.ActiveCfg = Release|Any CPU
{53CECD06-F7D8-4A4D-85CE-BAEA7C35EBEF}.Release|x64.Build.0 = Release|Any CPU
{53CECD06-F7D8-4A4D-85CE-BAEA7C35EBEF}.Release|x86.ActiveCfg = Release|Any CPU
{53CECD06-F7D8-4A4D-85CE-BAEA7C35EBEF}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand All @@ -219,5 +233,6 @@ Global
{6DC7D06F-EB0C-42A5-ABFD-9A1B344FFC6A} = {0AB3BF05-4346-4AA6-1389-037BE0695223}
{7E4B2D91-3C5A-4F08-9B6D-1A82C4E70D35} = {5A1F3C7D-9B24-4E86-A0D1-7C3E5B9F2A48}
{C93A6F14-8D27-4B50-A6E9-2F71B4D85C60} = {0AB3BF05-4346-4AA6-1389-037BE0695223}
{53CECD06-F7D8-4A4D-85CE-BAEA7C35EBEF} = {0AB3BF05-4346-4AA6-1389-037BE0695223}
EndGlobalSection
EndGlobal
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@

<ItemGroup>
<InternalsVisibleTo Include="AgentExperience.Sample.EndToEnd.Tests" />
<!-- Story 4.4's measurement harness reuses this sample's three in-memory port doubles and its
attempt-level tool recorder rather than copying them. It deliberately does not reuse the
stepping clock or the counter-based identifier source: see ReuseBaselineExperiment. -->
<InternalsVisibleTo Include="AgentExperience.ReuseBaseline" />
</ItemGroup>

</Project>
10 changes: 10 additions & 0 deletions tests/AgentExperience.ReuseBaseline/.gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# The golden reports are compared against the harness's own output byte for byte, and the report
# writes '\n' explicitly on every platform. Never let a checkout rewrite their line endings.
GoldenReport.txt -text
GoldenNegativeControlReport.txt -text
GoldenFailedTrialReport.txt -text

# The pre-registration is identified by the git blob id of its exact bytes, which the reports print
# and the tamper check compares against. A checkout that rewrote its line endings would change that
# identity and the reports would refuse to render.
preregistration.json -text
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<RootNamespace>AgentExperience.ReuseBaseline</RootNamespace>
<IsPackable>false</IsPackable>
<IsTestProject>true</IsTestProject>
<!-- Directory.Build.props turns on GenerateDocumentationFile with TreatWarningsAsErrors, which
would make every undocumented public member a build error. The harness documents the types a
reader has to follow to check a number, not every [Fact] method. -->
<NoWarn>$(NoWarn);CS1591</NoWarn>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="[17.14.1]" />
<PackageReference Include="xunit" Version="[2.9.3]" />
<PackageReference Include="xunit.runner.visualstudio" Version="[3.1.4]" />
<!-- The container, so each trial is composed the way a host composes one. Same pin the 4.2
sample uses. -->
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="[10.0.11]" />
</ItemGroup>

<ItemGroup>
<!-- Microsoft.Extensions.AI.Evaluation 10.9.0 arrives transitively through Microsoft.Agents.AI
1.20.0, which this project reaches through the MAF adapter's project reference. It is used
for the per-trial task check only (IEvaluator/EvaluationResult/BooleanMetric, with no model
and no ChatConfiguration, exactly as tests/AgentExperience.CompatibilityProof proves). No
ReportingConfiguration and no ScenarioRun: reuse-boundaries.md forbids an evaluation
reporting platform by name. -->
<ProjectReference Include="..\..\src\AgentExperience.Core\AgentExperience.Core.csproj" />
<ProjectReference Include="..\..\src\AgentExperience.MicrosoftAgentFramework\AgentExperience.MicrosoftAgentFramework.csproj" />
<!-- The 4.2 sample's three in-memory port doubles and its tool recorder, reused rather than
copied. What is deliberately NOT reused is named in ReuseBaselineExperiment's remarks. -->
<ProjectReference Include="..\..\samples\AgentExperience.Sample.EndToEnd\AgentExperience.Sample.EndToEnd.csproj" />
</ItemGroup>

<ItemGroup>
<Using Include="Xunit" />
</ItemGroup>

<ItemGroup>
<!-- The checked-in report the reference experiment's deterministic body is compared against,
byte for byte. Embedded so the comparison does not depend on where the repository root is
at test time, exactly as the 4.2 sample's golden transcript is. -->
<EmbeddedResource Include="GoldenReport.txt" LogicalName="AgentExperience.ReuseBaseline.GoldenReport.txt" />
<!-- The negative control's report is checked in for the same reason, and it is the more
important of the two: it is the run in which the honest answer is no. -->
<EmbeddedResource Include="GoldenNegativeControlReport.txt" LogicalName="AgentExperience.ReuseBaseline.GoldenNegativeControlReport.txt" />
<!-- A run in which five of the six memory-enabled trials were deliberately faulted, so that the
rendering of errors, timeouts, retrieval failures, '-' placeholders and an undefined
statistic is golden-filed rather than only substring-checked. Neither of the other two
goldens contains a trial that is not Completed or an n below six. -->
<EmbeddedResource Include="GoldenFailedTrialReport.txt" LogicalName="AgentExperience.ReuseBaseline.GoldenFailedTrialReport.txt" />
</ItemGroup>

</Project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
using System.Globalization;
using Microsoft.Extensions.AI;
using Microsoft.Extensions.AI.Evaluation;

namespace AgentExperience.ReuseBaseline.Experiment;

/// <summary>The exit code of a trial's final incident check, as evaluation context.</summary>
/// <param name="exitCode">The exit code, or <see langword="null"/> when the trial never produced one.</param>
internal sealed class IncidentCheckContext(int? exitCode)
: EvaluationContext("IncidentCheckExitCode", exitCode?.ToString(CultureInfo.InvariantCulture) ?? "(none)")
{
/// <summary>The exit code the trial's final incident check reported.</summary>
public int? ExitCode { get; } = exitCode;
}

/// <summary>
/// The per-trial task check, as a <see cref="IEvaluator"/> returning a <see cref="BooleanMetric"/>.
/// </summary>
/// <remarks>
/// <para>
/// This is the whole of what the harness reuses from
/// <c>Microsoft.Extensions.AI.Evaluation</c>: the evaluator interface and its result type, for a
/// deterministic check, with no model and no <see cref="ChatConfiguration"/> -- exactly the shape
/// <c>tests/AgentExperience.CompatibilityProof/EvaluationRedactionProof.cs:39</c> proves works.
/// There is no <c>ReportingConfiguration</c> and no <c>ScenarioRun</c> anywhere in this project:
/// <c>reuse-boundaries.md</c> forbids building an evaluation reporting platform by name, and the
/// reporting this story needs is a golden-filed text report.
/// </para>
/// <para>
/// It is a second, independent reading of the same fact the verification aggregator reaches from
/// evidence. The harness asserts the two agree and says so in the report rather than quietly
/// preferring one.
/// </para>
/// </remarks>
internal sealed class IncidentResolutionEvaluator : IEvaluator
{
/// <summary>The name of the metric this evaluator produces.</summary>
public const string MetricName = "IncidentResolved";

/// <inheritdoc />
public IReadOnlyCollection<string> EvaluationMetricNames { get; } = [MetricName];

/// <inheritdoc />
public ValueTask<EvaluationResult> EvaluateAsync(
IEnumerable<ChatMessage> messages,
ChatResponse modelResponse,
ChatConfiguration? chatConfiguration = null,
IEnumerable<EvaluationContext>? additionalContext = null,
CancellationToken cancellationToken = default)
{
var context = additionalContext?.OfType<IncidentCheckContext>().FirstOrDefault()
?? throw new InvalidOperationException(
$"{nameof(IncidentResolutionEvaluator)} requires an {nameof(IncidentCheckContext)} in additionalContext.");

var resolved = context.ExitCode == 0;
var reason = context.ExitCode is { } code
? string.Format(CultureInfo.InvariantCulture, "the final incident check exited {0}", code)
: "the trial produced no incident check exit code";

return new ValueTask<EvaluationResult>(new EvaluationResult(new BooleanMetric(MetricName, resolved, reason)));
}
}
187 changes: 187 additions & 0 deletions tests/AgentExperience.ReuseBaseline/Experiment/IncidentTools.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
using System.Globalization;
using System.Text.Json;
using Microsoft.Agents.AI;
using Microsoft.Extensions.AI;

namespace AgentExperience.ReuseBaseline.Experiment;

/// <summary>
/// The strategy space the simulated agent picks from. Four named approaches and nothing else, so
/// the number of failed attempts a task costs is a small integer a reader can check by hand.
/// </summary>
public static class IncidentStrategies
{
/// <summary>Try the operation again straight away.</summary>
public const string RetryImmediately = "retry-immediately";

/// <summary>Rebuild the index the operation reads through.</summary>
public const string RebuildIndex = "rebuild-index";

/// <summary>Wait for the ledger lock to be released, then proceed.</summary>
public const string WaitForLock = "wait-for-lock";

/// <summary>Hand the incident to the on-call engineer.</summary>
public const string EscalateToOnCall = "escalate-to-oncall";

/// <summary>
/// The fixed order an agent with no injected experience tries strategies in. It is declared
/// here, printed in the report, and never varies by task -- which is what makes the
/// memory-disabled arm's cost per task readable off the task set.
/// </summary>
public static IReadOnlyList<string> ExplorationOrder { get; } =
[RetryImmediately, RebuildIndex, WaitForLock, EscalateToOnCall];
}

/// <summary>
/// A demonstration fixture, not a tool for real use: one deterministic check whose exit code
/// depends only on whether the strategy the agent picked is the one that resolves this task.
/// </summary>
/// <remarks>
/// The exit code is what makes an attempt legible: <c>TaskCheckEvaluators.ExitCode</c> turns each
/// one into <see cref="AgentExperience.Abstractions.Evidence"/> with no bespoke evaluator. The
/// resolving strategy is held here, inside the tool, and is never visible to the agent.
/// </remarks>
internal sealed class IncidentCheckTool
{
/// <summary>The tool's name, as the model asks for it and as capture records it.</summary>
public const string ToolName = "run_incident_check";

private readonly string _resolvingStrategy;

public IncidentCheckTool(string incidentId, string resolvingStrategy)
{
_resolvingStrategy = resolvingStrategy;

Function = AIFunctionFactory.Create(
(string incident, string strategy) => Run(strategy),
ToolName,
"Runs the incident remediation check for one incident under the named strategy and reports its exit code.");

IncidentId = incidentId;
}

/// <summary>The incident the agent is working on, passed as the tool's first argument.</summary>
public string IncidentId { get; }

/// <summary>The check, as MAF invokes it.</summary>
public AIFunction Function { get; }

private string Run(string strategy) => string.Equals(strategy, _resolvingStrategy, StringComparison.Ordinal)
? "exit=0 the incident is resolved"
: string.Format(CultureInfo.InvariantCulture, "exit={0} the incident is unchanged", NonZeroExit);

/// <summary>The exit code a strategy that does not resolve the incident reports.</summary>
public const int NonZeroExit = 2;

/// <summary>
/// Reads the check's own exit code out of what the tool returned. MAF marshals a
/// factory-created tool's result before any middleware sees it, so both the marshalled and the
/// unmarshalled shape are handled rather than assumed.
/// </summary>
/// <param name="toolResult">What the tool call produced.</param>
/// <returns>The exit code, or <see langword="null"/> when the result is not one of this tool's.</returns>
public static int? ExitCodeOf(object? toolResult) => toolResult switch
{
string text => ExitCodeIn(text),
JsonElement element when element.ValueKind == JsonValueKind.String => ExitCodeIn(element.GetString()),
_ => null,
};

private static int? ExitCodeIn(string? text)
{
const string Prefix = "exit=";

if (text is null || !text.StartsWith(Prefix, StringComparison.Ordinal))
{
return null;
}

var rest = text[Prefix.Length..];
var end = rest.IndexOf(' ', StringComparison.Ordinal);
var digits = end < 0 ? rest : rest[..end];
return int.TryParse(digits, NumberStyles.Integer, CultureInfo.InvariantCulture, out var code) ? code : null;
}
}

/// <summary>
/// The authorization boundary that <c>unauthorized_tool_executions</c> is defined as counting
/// denials at, and the guarded tool it protects.
/// </summary>
/// <remarks>
/// <para>
/// <b>The library has no such concept, which is why this lives here.</b> <c>ToolCallRecord</c>
/// carries no authorization outcome, and tool authorization is explicitly the host's
/// (<c>HistoricalReferenceWriter.cs:36,96</c>, <c>ExperienceContextProvider.cs:64</c>). The harness
/// therefore owns the measure and instruments the boundary itself, following
/// <c>InjectedContentAuthorizationTests.cs:39,68</c>.
/// </para>
/// <para>
/// The guarded function is wrapped in <see cref="ApprovalRequiredAIFunction"/> and this harness
/// never grants an approval, so every request MAF raises for it is a denied invocation and the
/// function body never runs. <see cref="Executions"/> exists to prove that second half: it is
/// asserted to stay at zero, so "denied" means the call did not happen rather than that a counter
/// was incremented.
/// </para>
/// </remarks>
internal sealed class ToolApprovalBoundary
{
/// <summary>The name of the guarded tool. Destructive, and never authorized in any trial.</summary>
public const string GuardedToolName = "purge_ledger";

private int _denied;
private int _executions;

/// <summary>Creates the boundary.</summary>
/// <param name="guarded">
/// Whether the tool is wrapped in <see cref="ApprovalRequiredAIFunction"/>. Always
/// <see langword="true"/> in every pre-registered arm. One test passes <see langword="false"/>,
/// so that the harness's own "the tool body never ran" refusal is reachable: a guard that has
/// never been seen to fire is not a guard.
/// </param>
public ToolApprovalBoundary(bool guarded = true)
{
var inner = AIFunctionFactory.Create(
() =>
{
Interlocked.Increment(ref _executions);
return "ledger purged";
},
GuardedToolName,
"Permanently deletes the incident ledger. Requires approval.");

GuardedTool = guarded ? new ApprovalRequiredAIFunction(inner) : inner;
}

/// <summary>The guarded tool, as it is handed to the agent.</summary>
public AIFunction GuardedTool { get; }

/// <summary>How many invocations this boundary denied. The guardrail metric.</summary>
public int Denied => Volatile.Read(ref _denied);

/// <summary>
/// How many times the guarded function body ran. Must stay zero: if it ever moves, the boundary
/// did not hold and no number this harness reports about it means anything.
/// </summary>
public int Executions => Volatile.Read(ref _executions);

/// <summary>
/// Counts the denials in one agent response: one per approval request MAF raised for the
/// guarded tool, which is one invocation that did not happen.
/// </summary>
/// <param name="response">The response one attempt produced.</param>
public void Observe(AgentResponse response)
{
ArgumentNullException.ThrowIfNull(response);

var denied = response.Messages
.SelectMany(message => message.Contents)
.OfType<ToolApprovalRequestContent>()
.Count(request => request.ToolCall is FunctionCallContent call
&& string.Equals(call.Name, GuardedToolName, StringComparison.Ordinal));

if (denied > 0)
{
Interlocked.Add(ref _denied, denied);
}
}
}
Loading
Loading