feat(datasetAttributes): always call attributes with context + updated evals

packages/mcp-core/src/tools/support/search-events/utils.ts

@@ -1273,6 +1273,7 @@ export function createDatasetAttributesTool(options: {
             attributeTypes,
             substringMatch,
             query,
+            context: true,
           },
         );
 

packages/mcp-server-evals/src/evals/natural-language-search.eval.ts

@@ -72,7 +72,7 @@ describeEval("natural-language-search-events", {
           value.includes("metric.type:distribution"),
         fields: (value: unknown) =>
           Array.isArray(value) &&
-          value.includes("environment") &&
+          value.includes("sentry.environment") &&
           value.includes(
             "p95(value,http.request.duration,distribution,millisecond)",
           ),

packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts

@@ -6,12 +6,11 @@ import { StructuredOutputScorer } from "./utils/structuredOutputScorer";
 import "../setup-env";
 
 // Compares the attribute the agent picks when the trace-items attributes
-// endpoint is called with and without `expand=context`. The with-context span
+// endpoint is called with and without `expand=context`. The with-context
 // fixtures list the replacement for each deprecated attribute (e.g.
 // user.geo.region for geo.region), so only that path should use it.
 //
-// The datasetAttributes tool does not request context itself, so this service
-// forces it on or off for every attributes request.
+// This service forces the context on or off for every attributes request.
 class AttributeContextApiService extends SentryApiService {
   constructor(private readonly attributeContext: boolean) {
     super({ accessToken: "test-token" });
@@ -130,6 +129,90 @@ describeEval("search-events-agent-attributes-without-context", {
             !value.includes("kiosk.firmware"),
         },
       },
+      {
+        // Without context the logs attributes only list code.filepath
+        input:
+          "Count error logs grouped by source code file path over the last 24 hours",
+        expectedTools: [
+          {
+            name: "datasetAttributes",
+            arguments: {
+              dataset: "logs",
+            },
+          },
+        ],
+        expected: {
+          dataset: "logs",
+          fields: (value: unknown) =>
+            Array.isArray(value) &&
+            value.includes("code.filepath") &&
+            !value.includes("code.file.path"),
+        },
+      },
+      {
+        // Without context the metrics attributes only list http.method
+        input:
+          "In metrics, show p95 http.request.duration grouped by HTTP method over the last 24 hours",
+        expectedTools: [
+          {
+            name: "datasetAttributes",
+            arguments: {
+              dataset: "metrics",
+            },
+          },
+        ],
+        expected: {
+          dataset: "metrics",
+          fields: (value: unknown) =>
+            Array.isArray(value) &&
+            value.includes("http.method") &&
+            !value.includes("http.request.method"),
+        },
+      },
+      {
+        // worker.queue_wait and worker.queue_wait_time are made-up attributes.
+        // Without briefs the agent can only match on the name, and
+        // worker.queue_wait_time matches "worker queue wait time"
+        input:
+          "From logs, what is the average worker queue wait time jobs actually experienced over the last 24 hours?",
+        expectedTools: [
+          {
+            name: "datasetAttributes",
+            arguments: {
+              dataset: "logs",
+            },
+          },
+        ],
+        expected: {
+          dataset: "logs",
+          fields: (value: unknown) =>
+            Array.isArray(value) &&
+            value.includes("avg(worker.queue_wait_time)") &&
+            !value.includes("avg(worker.queue_wait)"),
+        },
+      },
+      {
+        // fleet.region and fleet.deployment_region are made-up attributes.
+        // Without briefs the agent can only match on the name, and
+        // fleet.deployment_region matches "fleet deployment region"
+        input:
+          "In metrics, show p95 http.request.duration grouped by the fleet deployment region each server is currently running in over the last 24 hours",
+        expectedTools: [
+          {
+            name: "datasetAttributes",
+            arguments: {
+              dataset: "metrics",
+            },
+          },
+        ],
+        expected: {
+          dataset: "metrics",
+          fields: (value: unknown) =>
+            Array.isArray(value) &&
+            value.includes("fleet.deployment_region") &&
+            !value.includes("fleet.region"),
+        },
+      },
     ];
   },
   task: async (input) => {
@@ -294,6 +377,91 @@ describeEval("search-events-agent-attributes-with-context", {
             !value.includes("kiosk.firmware_revision"),
         },
       },
+      {
+        // Context marks code.filepath as deprecated in favor of code.file.path
+        input:
+          "Count error logs grouped by source code file path over the last 24 hours",
+        expectedTools: [
+          {
+            name: "datasetAttributes",
+            arguments: {
+              dataset: "logs",
+            },
+          },
+        ],
+        expected: {
+          dataset: "logs",
+          fields: (value: unknown) =>
+            Array.isArray(value) &&
+            value.includes("code.file.path") &&
+            !value.includes("code.filepath"),
+        },
+      },
+      {
+        // Context marks http.method as deprecated in favor of
+        // http.request.method
+        input:
+          "In metrics, show p95 http.request.duration grouped by HTTP method over the last 24 hours",
+        expectedTools: [
+          {
+            name: "datasetAttributes",
+            arguments: {
+              dataset: "metrics",
+            },
+          },
+        ],
+        expected: {
+          dataset: "metrics",
+          fields: (value: unknown) =>
+            Array.isArray(value) &&
+            value.includes("http.request.method") &&
+            !value.includes("http.method"),
+        },
+      },
+      {
+        // worker.queue_wait_time matches "worker queue wait time" by name, but
+        // its brief says it is a configured limit. Only worker.queue_wait's
+        // brief describes the measured wait
+        input:
+          "From logs, what is the average worker queue wait time jobs actually experienced over the last 24 hours?",
+        expectedTools: [
+          {
+            name: "datasetAttributes",
+            arguments: {
+              dataset: "logs",
+            },
+          },
+        ],
+        expected: {
+          dataset: "logs",
+          fields: (value: unknown) =>
+            Array.isArray(value) &&
+            value.includes("avg(worker.queue_wait)") &&
+            !value.includes("avg(worker.queue_wait_time)"),
+        },
+      },
+      {
+        // fleet.deployment_region matches "fleet deployment region" by name,
+        // but its brief says it is the region the server moves to next. Only
+        // fleet.region's brief describes where the server runs now
+        input:
+          "In metrics, show p95 http.request.duration grouped by the fleet deployment region each server is currently running in over the last 24 hours",
+        expectedTools: [
+          {
+            name: "datasetAttributes",
+            arguments: {
+              dataset: "metrics",
+            },
+          },
+        ],
+        expected: {
+          dataset: "metrics",
+          fields: (value: unknown) =>
+            Array.isArray(value) &&
+            value.includes("fleet.region") &&
+            !value.includes("fleet.deployment_region"),
+        },
+      },
     ];
   },
   task: async (input) => {

packages/mcp-server-evals/src/evals/search-events-agent.eval.ts

@@ -121,7 +121,7 @@ describeEval("search-events-agent", {
           query: (value: unknown) =>
             typeof value === "string" &&
             [
-              'transaction:"VPN connections"',
+              'sentry.segment.name:"VPN connections"',
               "tags[type]:Unified",
               "tags[country]:CN",
             ].every((token) => value.includes(token)),

packages/mcp-server-evals/src/evals/search-issue-events-agent.eval.ts

@@ -73,7 +73,7 @@ describeEval("search-issue-events-agent", {
       },
       {
         // Combined filters with time range
-        input: "Production events from yesterday with specific release",
+        input: "Production events from yesterday with the 5.1 release",
         expectedTools: [],
         expected: {
           query:

packages/mcp-server-mocks/src/fixtures/trace-items-attributes-logs-number-with-context.json

@@ -0,0 +1,60 @@
+[
+  {
+    "key": "severity_number",
+    "name": "severity_number",
+    "context": {
+      "isConvention": false,
+      "brief": "The numeric severity level of the log.",
+      "isDeprecated": false,
+      "examples": [17]
+    }
+  },
+  {
+    "key": "sentry.observed_timestamp_nanos",
+    "name": "sentry.observed_timestamp_nanos",
+    "context": {
+      "isConvention": false,
+      "brief": "When Sentry received the log, in nanoseconds since the Unix epoch.",
+      "isDeprecated": false
+    }
+  },
+  {
+    "key": "timestamp",
+    "name": "timestamp",
+    "context": {
+      "isConvention": false,
+      "brief": "When the log was emitted.",
+      "isDeprecated": false
+    }
+  },
+  {
+    "key": "custom.duration",
+    "name": "custom.duration",
+    "context": {}
+  },
+  {
+    "key": "custom.bytes",
+    "name": "custom.bytes",
+    "context": {}
+  },
+  {
+    "key": "worker.queue_wait",
+    "name": "worker.queue_wait",
+    "context": {
+      "isConvention": true,
+      "brief": "How long the job waited in the worker queue before a worker picked it up, in milliseconds.",
+      "isDeprecated": false,
+      "examples": [412]
+    }
+  },
+  {
+    "key": "worker.queue_wait_time",
+    "name": "worker.queue_wait_time",
+    "context": {
+      "isConvention": true,
+      "brief": "The maximum time a job may wait in the worker queue before it is dropped, as configured on the worker. It is a limit, not a measured wait.",
+      "isDeprecated": false,
+      "examples": [30000]
+    }
+  }
+]

packages/mcp-server-mocks/src/fixtures/trace-items-attributes-logs-number.json

@@ -18,5 +18,13 @@
   {
     "key": "custom.bytes",
     "name": "Custom Bytes"
+  },
+  {
+    "key": "worker.queue_wait",
+    "name": "worker.queue_wait"
+  },
+  {
+    "key": "worker.queue_wait_time",
+    "name": "worker.queue_wait_time"
   }
 ]