pkg/ccl/testccl/workload/schemachange/schemachange_test: TestWorkload failed

[cockroach-teamcity]

pkg/ccl/testccl/workload/schemachange/schemachange_test.TestWorkload failed with artifacts on master @ 2c1cb12eef85880e19dd598df808d0b0baea4170:

    ]
   }
  ],
  "previousStatements": [
   "CREATE DOMAIN public.domain_w1_71 AS BOOL NOT NULL",
   "CREATE TABLE public.table_w1_72 (col72_w1_73 BOX2D NOT NULL, col72_w1_74 INT2 NOT NULL, col72_w1_75 REGTYPE NULL, col72_w1_76 BIT(48) NOT NULL, col72_w1_77 GEOMETRY NOT NULL, col72_w1_78 INT4 NOT NULL, col72_w1_79 REGROLE, col72_w1_80 INTERVAL NULL, col72_w1_81 TIMESTAMPTZ NOT NULL, col72_w1_82 INT8 NULL AS (col72_w1_78 + col72_w1_74) VIRTUAL, col72_w1_83 STRING AS (CASE WHEN col72_w1_79 IS NULL THEN NULL ELSE e'T#\\x7f\\x19':::STRING END) STORED, PRIMARY KEY (col72_w1_78, col72_w1_76 DESC, col72_w1_73), INDEX (col72_w1_78 DESC) STORING (col72_w1_77, col72_w1_79, col72_w1_80), UNIQUE (col72_w1_73 ASC) STORING (col72_w1_77, col72_w1_79, col72_w1_80, col72_w1_81, col72_w1_83) PARTITION BY LIST (col72_w1_73) (PARTITION table72_part_0 VALUES IN (('BOX(-0.6038336753750194 -1.5397034959189264,0.7342316904885146 -0.5302682240666662)':::BOX2D,), ('BOX(0.4861566185095332 -1.2171030498323039,0.624552865551701 0.17556359783476383)':::BOX2D,), ('BOX(-1.4107576357816232 -0.7670186899455276,0.7603181334180087 0.22372050135769886)':::BOX2D,), ('BOX(-0.34214268823104477 -1.628055115129217,0.11305464325248038 0.11799799260756999)':::BOX2D,), ('BOX(-0.4972694650074221 -1.1929055001000926,0.9233921307644493 -0.04881975673176431)':::BOX2D,), ('BOX(-0.8166142467822013 -0.19230681911579883,0.1625905035952112 0.917028788051816)':::BOX2D,), ('BOX(0.724489445970615 -1.5642363005177962,1.3007026657016194 -0.06090931172291064)':::BOX2D,), ('BOX(-0.5456301320432351 0.5761027288367004,0.3373226297908951 0.7227473662728912)':::BOX2D,), ('BOX(-2.0459449102722735 -0.3883415063315757,0.07293018799008649 1.246247121053115)':::BOX2D,), ('BOX(-0.4998152265613246 -2.5530326080474275,-0.15739225568268989 -0.06898007129947037)':::BOX2D,)), PARTITION table72_part_1 VALUES IN (('BOX(-10 -10,10 10)':::BOX2D,), ('BOX(-1.4272640194567818 -0.22868976600042146,1.020756484065495 0.7463826872722316)':::BOX2D,), ('BOX(-0.8554247217637755 -0.2628231656526522,0.5488270680179064 0.4995786264444332)':::BOX2D,), ('BOX(-0.41502464662505134 -1.8048774286579046,-0.12747495158847189 -0.04826577232974383)':::BOX2D,), ('BOX(-0.5150173118931283 -1.5331147014938868,0.5742618771718209 0.531185198906077)':::BOX2D,), ('BOX(1.406102559645861 -0.04835179961417235,1.574558581520662 0.9052284515790258)':::BOX2D,), ('BOX(-0.7082023141903635 -0.37439187323391293,1.0095482444723496 0.3640697838707646)':::BOX2D,), ('BOX(-0.00018214889847034677 -1.2114359052906272,1.2402872591703757 -0.7400147773645124)':::BOX2D,), ('BOX(-0.5639178814824641 0.15812415788041773,0.5188795532591393 0.38847532735459994)':::BOX2D,))), INVERTED INDEX (col72_w1_77), INDEX (col72_w1_80 ASC, col72_w1_82 ASC) STORING (col72_w1_77, col72_w1_79, col72_w1_81, col72_w1_83), INDEX (col72_w1_79 ASC) NOT VISIBLE) WITH ('schema_locked' = true)",
   "CREATE TABLE public.table_w1_84 (col84_w1_85 INT2 NOT NULL, col84_w1_86 PG_LSN NOT NULL, PRIMARY KEY (col84_w1_85), UNIQUE (col84_w1_85 ASC, col84_w1_86 ASC), UNIQUE (col84_w1_86, col84_w1_85 ASC), UNIQUE (col84_w1_85 DESC, col84_w1_86), UNIQUE (col84_w1_86 DESC), INDEX (col84_w1_85 ASC), FAMILY (col84_w1_85), FAMILY (col84_w1_86))",
   "ALTER FUNCTION public.udf_w1_43(BYTES, TIMESTAMPTZ, VARCHAR, INET, INET, JSONB, VARBIT, INTERVAL, TIMETZ, NAME, PG_LSN, LTREE, BIT(0), BIT(0), FLOAT8, BPCHAR, INT8, INT8, GEOMETRY, CITEXT, INT2, INT2, DECIMAL, GEOGRAPHY, TIME, UUID, REFCURSOR, \"char\", STRING, STRING, STRING, TSQUERY, TSQUERY, TSVECTOR, BOX2D, DATE, DATE, TIMESTAMP, BOOL, BOOL) SET SCHEMA public"
  ]
 }
}
    schema_change_external_test.go:183: 
        	Error Trace:	pkg/ccl/testccl/workload/schemachange/schema_change_external_test.go:183
        	            				pkg/ccl/testccl/workload/schemachange/schema_change_external_test.go:41
        	Error:      	Received unexpected error:
        	            	***UNEXPECTED ERROR; Received an unexpected execution error.: ERROR: function udf_w1_43(bytes,timestamptz,varchar,inet,inet,jsonb,varbit,interval,timetz,name,pg_lsn,ltree,bit,bit,float,bpchar,int,int,geometry,citext,int2,int2,decimal,geography,time,uuid,refcursor,"char",string,string,string,tsquery,tsquery,tsvector,box2d,date,date,timestamp,bool,bool) does not exist (SQLSTATE 42883)
        	            	(1) forced error mark
        	            	  | "fatal error when running txn"
        	            	  | github.com/cockroachdb/errors/withstack/*withstack.withStack::
        	            	Wraps: (2)
        	            	Wraps: (3) attached stack trace
        	            	  -- stack trace:
        	            	  | github.com/cockroachdb/cockroach/pkg/workload/schemachange.(*opStmt).executeStmt
        	            	  | 	pkg/workload/schemachange/operation_generator.go:3838
        	            	  | github.com/cockroachdb/cockroach/pkg/workload/schemachange.(*schemaChangeWorker).runInTxn
        	            	  | 	pkg/workload/schemachange/schemachange.go:676
        	            	  | github.com/cockroachdb/cockroach/pkg/workload/schemachange.(*schemaChangeWorker).run
        	            	  | 	pkg/workload/schemachange/schemachange.go:801
        	            	  | pkg/ccl/testccl/workload/schemachange/schemachange_test_test.runSchemaChangeWorkload.runSchemaChangeWorkload.func5.func7
        	            	  | 	pkg/ccl/testccl/workload/schemachange/schema_change_external_test.go:172
        	            	  | golang.org/x/sync/errgroup.(*Group).Go.func1
        	            	  | 	external/org_golang_x_sync/errgroup/errgroup.go:93
        	            	  | runtime.goexit
        	            	  | 	src/runtime/asm_s390x.s:864
        	            	Wraps: (4) ***UNEXPECTED ERROR; Received an unexpected execution error.
        	            	Wraps: (5) ERROR: function udf_w1_43(bytes,timestamptz,varchar,inet,inet,jsonb,varbit,interval,timetz,name,pg_lsn,ltree,bit,bit,float,bpchar,int,int,geometry,citext,int2,int2,decimal,geography,time,uuid,refcursor,"char",string,string,string,tsquery,tsquery,tsvector,box2d,date,date,timestamp,bool,bool) does not exist (SQLSTATE 42883)
        	            	Error types: (1) *markers.withMark (2) *schemachange.ErrorState (3) *withstack.withStack (4) *errutil.withPrefix (5) *pgconn.PgError
        	Test:       	TestWorkload
Schema Workload Stats
Total Schema Statements Executed = 15
Total Schema Statements Succeeded = 15
Total Schema Statement Expected Failures = 0
Total Transactions Committed = 4
Total Transactions Rolled Back = 11
Total Transactions Executed = 15
    schema_change_external_test.go:136: backup, tracing data, and system table dumps in /artifacts/tmp/_tmp/e1bc47aa61a233e9c8ddfa2c4c769198/logTestWorkload2156981962
    schema_change_external_test.go:145: running DROP with use_declarative_schema_changer = off
    panic.go:694: -- test log scope end --
test logs left over in: /artifacts/tmp/_tmp/e1bc47aa61a233e9c8ddfa2c4c769198/logTestWorkload2156981962
--- FAIL: TestWorkload (15.83s)

Help

See also: How To Investigate a Go Test Failure (internal)

/cc @cockroachlabs/sql-foundations

Improve this report!

Jira issue: CRDB-68166

[himanshu-shrivastava-crl]

/investigate

[github-actions]

Investigation: pkg/ccl/testccl/workload/schemachange/schemachange_test.TestWorkload

Investigated failure: issue body
Failure SHA: 2c1cb12eef85880e19dd598df808d0b0baea4170
Confidence: moderate

What This Test Does

TestWorkload spins up a 3-node in-memory cluster and runs the
schemachange workload against it (2 concurrent workers, 100 iterations
each). The workload randomly generates schema-change statements, wraps
them in transactions, and — crucially — predicts the expected outcome of
each statement (success or a specific pgcode). A statement that fails
with an error code the generator did not predict is treated as a test
failure ("***UNEXPECTED ERROR").

Where the Failure Occurs

The workload generated a statement referencing user-defined function
udf_w1_43(...) and the server returned:

ERROR: function udf_w1_43(bytes,timestamptz,varchar,...,bool,bool) does not exist (SQLSTATE 42883)

42883 (UndefinedFunction) was not in the statement's expected/potential
error set, so opStmt.executeStmt marked it fatal:
operation_generator.go:3835-3841,
which propagates up through
schemachange.go:676
and fails g.Wait() at
schema_change_external_test.go:183.

The key detail is in the dumped previousStatements: the last
statement executed in the same transaction before the failure was

ALTER FUNCTION public.udf_w1_43(...) SET SCHEMA public

i.e. an ALTER FUNCTION on the same function had just run (and
succeeded) earlier in the transaction, yet the subsequent generated
statement referencing that function reported it as non-existent.

Analysis

This is almost certainly NOT an s390x-specific bug, despite the
s390x-test-failure label. The identical error signature was reported
one day earlier in #174888 (closed 2026-09-10) failing on arm64
and EngFlow (x86) builders — see Related Issues.
The s390x label here reflects only which builder happened to hit it.

Primary hypothesis (moderate confidence): the workload's descriptor
introspection returns stale function identity within a multi-statement
transaction after an earlier ALTER FUNCTION in the same txn.

The workload generates and executes one operation at a time inside a
transaction (schemachange.go:623-699).
Each op is built from a fresh introspection query
(descJSONQuery/functionDescsQuery, which read system.descriptor
and derive function names from the schema descriptor's functions
map — query_util.go:24-49).

The smoking gun is the sibling failure #174888: its last prior
statement was ALTER FUNCTION public.udf_w0_90(...) RENAME TO udf_w1_189,
and the very next op then referenced the old name udf_w0_90 and got
does not exist. For the introspection to hand the generator the old
name after a rename committed to the txn's uncommitted state, the
introspection query must be reading function identity that does not yet
reflect the earlier same-transaction ALTER FUNCTION. The generator then
builds a statement against a stale name/overload signature that no longer
resolves at execution time.

For this issue (#175015) the earlier statement was SET SCHEMA public
(a no-op schema move). The failing reference used the same 40-argument
signature. The common thread across both occurrences is: an
ALTER FUNCTION runs earlier in the transaction, and a later generated
op re-derives that function's identity from introspection that is
inconsistent with the pending change, yielding an unexpected
UndefinedFunction.

Two sub-possibilities I could not fully disambiguate without artifacts:

• Workload bug — the introspection view (descJSONQuery) doesn't
  correctly reflect uncommitted function DDL, so the generator predicts
  success against a function identity that is gone. Fix would live in the
  workload (correct the introspection, or add UndefinedFunction to the
  potential-error set when a same-txn ALTER FUNCTION touched the
  function).
• Product bug — CRDB descriptor/leasing behavior leaves a
  just-altered function transiently unresolvable by overload resolution
  within the same txn while system.descriptor still shows it. This
  would be the more serious outcome and is worth ruling in/out with the
  descriptor dumps in the artifacts.

Note: the generator already treats UndefinedFunction as
rollback-worthy when it occurs during op generation
(schemachange.go:644-652),
but not when it occurs during execution of a statement the generator
believed would succeed — which is exactly this case.

Recommendation: SQL Foundations should compare this against #174888
(same failure mode). Since #174888 was closed 2026-09-10 17:46 UTC while
this failure ran against a SHA built 2026-09-10 13:40 UTC, the failure
predates whatever closed #174888 — this may simply be a re-occurrence
of an unfixed bug (I found no PR referencing #174888). Confirm whether a
fix actually landed; if not, reopen/dedupe. To root-cause, inspect the
ns-desc-json / descriptor dumps that the test writes on failure (see
schema_change_external_test.go:119-136).

Related Issues and PRs

• pkg/ccl/testccl/workload/schemachange/schemachange_test: TestWorkload failed #174888 (CLOSED 2026-09-10) — same failure mode, identical
  ***UNEXPECTED ERROR ... function udf_wN_M(...) does not exist (42883),
  with a preceding same-txn ALTER FUNCTION ... RENAME. Failed on arm64
  and EngFlow — establishes this is architecture-independent. No fixing
  PR references it; closure may have been a dedupe/stale close.
• The schemachange_test: TestWorkload failed issue title recurs
  frequently (pkg/ccl/testccl/workload/schemachange/schemachange_test: TestWorkload failed #174888, pkg/ccl/testccl/workload/schemachange/schemachange_test: TestWorkload failed #173504, pkg/ccl/testccl/workload/schemachange/schemachange_test: TestWorkload failed #172621, pkg/ccl/testccl/workload/schemachange/schemachange_test: TestWorkload failed #172599, pkg/ccl/testccl/workload/schemachange/schemachange_test: TestWorkload failed #172592, pkg/ccl/testccl/workload/schemachange/schemachange_test: TestWorkload failed #172061,
  pkg/ccl/testccl/workload/schemachange/schemachange_test: TestWorkload failed #170696, pkg/ccl/testccl/workload/schemachange/schemachange_test: TestWorkload failed #170145, ... — 20+ occurrences in the last several months),
  but these are a mix of different root causes. The UDF
  "does not exist after same-txn ALTER FUNCTION" signature specifically
  matches pkg/ccl/testccl/workload/schemachange/schemachange_test: TestWorkload failed #174888.

Timeline

• Failure SHA 2c1cb12 authored/built 2026-09-10 13:40 UTC.
• Sibling issue pkg/ccl/testccl/workload/schemachange/schemachange_test: TestWorkload failed #174888 (same signature) closed 2026-09-10 17:46 UTC.
• This issue (pkg/ccl/testccl/workload/schemachange/schemachange_test: TestWorkload failed #175015) filed 2026-09-10 19:43 UTC.
• Recent commits under pkg/workload/schemachange/ in the weeks prior
  focus on dynamic SQL inside procedures and domain constraints; none
  obviously target function-rename/set-schema introspection staleness.

Detailed evidence and investigation notes

Log Excerpts

Failing txn (from issue body), last successful statement then the fatal error:

"previousStatements": [
   ...
   "ALTER FUNCTION public.udf_w1_43(BYTES, TIMESTAMPTZ, VARCHAR, ..., BOOL, BOOL) SET SCHEMA public"
]
...
***UNEXPECTED ERROR; Received an unexpected execution error.:
ERROR: function udf_w1_43(bytes,timestamptz,varchar,...,bool,bool) does not exist (SQLSTATE 42883)

Total Schema Statements Executed = 15 / Succeeded = 15
Total Transactions Committed = 4 / Rolled Back = 11
--- FAIL: TestWorkload (15.83s)

Sibling #174888 (same signature, different prior op):

"previousStatements": [
   "ALTER FUNCTION public.udf_w0_90(BPCHAR, BOOL, ..., CITEXT) RENAME TO udf_w1_189"
]
...
ERROR: function udf_w0_90(bpchar,bool,...,citext) does not exist (SQLSTATE 42883)

Code References

• Unexpected-error classification (fatal path):
  operation_generator.go:3835-3841.
• Per-op generation loop inside a txn: schemachange.go:623-699.
• UndefinedFunction tolerated only at generation time, not execution:
  schemachange.go:644-652.
• ALTER FUNCTION ... SET SCHEMA generator + introspection:
  operation_generator.go:5820-5932.
• Descriptor introspection view that supplies function identity:
  query_util.go:24-49 (descJSONQuery), query_util.go:95
  (functionDescsQuery).
• Failure-time descriptor/namespace dumps for debugging:
  schema_change_external_test.go:119-136.

Things Ruled Out

• Endianness / s390x-specific behavior. Ruled out: pkg/ccl/testccl/workload/schemachange/schemachange_test: TestWorkload failed #174888 shows the
  identical failure on arm64 and EngFlow x86. The workload uses a seeded
  RNG but the failure is about server-side function resolution, not
  RNG-divergent generation.
• A function actually dropped earlier in the txn. The
  previousStatements show no DROP FUNCTION; the only prior op on the
  function was ALTER FUNCTION ... SET SCHEMA public (a no-op schema
  move), which succeeded.

Other Failure Occurrences

Only one failure comment on this issue (the body itself; the sole other
comment is the /investigate trigger). No TeamCity/EngFlow artifacts
were downloaded for this run (see Tooling Feedback).

Tooling Feedback

• I did not download the TeamCity artifacts (build 21509412 on
  Cockroach_Ci_TestsIbmCloudLinuxS390x_UnitTestsS390x). The
  ns-desc-json and raw descriptor dumps the test writes on failure
  would let a future run confirm whether the just-altered function
  descriptor was actually present/resolvable in the txn's view — the
  single most useful artifact for distinguishing "workload introspection
  bug" from "product resolution bug". Prioritizing this download would
  strengthen the root-cause call.
• gh search prs does not support a combined --state all, and its
  JSON schema lacks mergedAt/mergeCommit; confirming whether a PR
  fixed pkg/ccl/testccl/workload/schemachange/schemachange_test: TestWorkload failed #174888 required manual cross-checking. A direct
  "what closed issue N" lookup would have been faster.
• git log -S over a blobless clone timed out (on-demand blob fetch);
  plain git log -- <path> worked fine.

---

Investigation run: https://github.com/cockroachdb/cockroach/actions/runs/34636856767

Was this investigation helpful? Leave a 👍 or 👎 on this comment.

---

Generated by claude-opus-4-8 at high effort. Use /investigate fable-5 max to go deeper.

Workflow run · Was this investigation helpful? Leave a 👍 or 👎 on this comment.

[shghasemi]

investigate on this issue is somewhat incorrect. This is the same issue as #174888 that was fixed and closed by https://github.com/cockroachlabs/cockroach/pull/4560 right before this failure. The failing SHA doesn't have the fix.

The investigate claims that the function signature was correct. This is wrong.

"sql": "CREATE FUNCTION public.udf_w1_43(val_43 VARCHAR, val_5 BOX2D, val_38 TSQUERY, val_35 TIMESTAMP, val_12 FLOAT8, val_6 CHAR, val_41 UUID, val_19 INT8, val_39 TSVECTOR, val_21 JSONB, val_7 BYTES, val_42 VARBIT, val_15 INET, val_22 LTREE, val_11 DECIMAL, val_14 GEOMETRY, val_4 BOOL, val_34 TIME, val_33 STRING, val_0 \"char\", val_16 INT8, val_36 TIMESTAMPTZ, val_27 REFCURSOR, val_20 INTERVAL, val_23 NAME, val_10 DATE, val_17 INT2, val_37 TIMETZ, val_9 STRING, val_26 PG_LSN, val_8 CITEXT, val_3 BIT, val_13 GEOGRAPHY, val_61 INET DEFAULT '231.14.147.253':::INET, val_50 BOOL DEFAULT true, val_49 BIT DEFAULT B'1000010100100100101101000010011001', val_84 TSQUERY DEFAULT e'!\\'\\u00F1D\\'\\'k\\U00004E16\\' \u0026 \\'UQpIUTySs\\'':::TSQUERY, val_55 STRING DEFAULT e'\\U0002213A' COLLATE en, val_56 DATE DEFAULT '1997-04-13':::DATE, val_63 INT2 DEFAULT 4599:::INT8)\n\tRETURNS INT8\n\tLANGUAGE SQL\n\tAS $$ SELECT NULL WHERE ((SELECT count(*) FROM public.table_w1_1 LIMIT 0) = 0) $$"
"sql": "ALTER FUNCTION public.udf_w1_43(BYTES, TIMESTAMPTZ, VARCHAR, INET, INET, JSONB, VARBIT, INTERVAL, TIMETZ, NAME, PG_LSN, LTREE, BIT(0), BIT(0), FLOAT8, BPCHAR, INT8, INT8, GEOMETRY, CITEXT, INT2, INT2, DECIMAL, GEOGRAPHY, TIME, UUID, REFCURSOR, \"char\", STRING, STRING, STRING, TSQUERY, TSQUERY, TSVECTOR, BOX2D, DATE, DATE, TIMESTAMP, BOOL, BOOL) SET SCHEMA public"
ERROR: function udf_w1_43(bytes,timestamptz,varchar,inet,inet,jsonb,varbit,interval,timetz,name,pg_lsn,ltree,bit,bit,float,bpchar,int,int,geometry,citext,int2,int2,decimal,geography,time,uuid,refcursor,\"char\",string,string,string,tsquery,tsquery,tsvector,box2d,date,date,timestamp,bool,bool) does not exist (SQLSTATE 42883)",

We can clearly see that the ALTER was called with a wrong signature.

This is only a test flake. Closing it because it's resolved.