diff --git a/src/prism.c b/src/prism.c
index 1f62ea5add..166f52c98a 100644
--- a/src/prism.c
+++ b/src/prism.c
@@ -17459,18 +17459,6 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm
 
                     return UP(pm_pinned_variable_node_create(parser, &operator, variable));
                 }
-                case PM_TOKEN_NUMBERED_REFERENCE: {
-                    parser_lex(parser);
-                    pm_node_t *variable = UP(pm_numbered_reference_read_node_create(parser, &parser->previous));
-
-                    return UP(pm_pinned_variable_node_create(parser, &operator, variable));
-                }
-                case PM_TOKEN_BACK_REFERENCE: {
-                    parser_lex(parser);
-                    pm_node_t *variable = UP(pm_back_reference_read_node_create(parser, &parser->previous));
-
-                    return UP(pm_pinned_variable_node_create(parser, &operator, variable));
-                }
                 case PM_TOKEN_PARENTHESIS_LEFT: {
                     bool previous_pattern_matching_newlines = parser->pattern_matching_newlines;
                     parser->pattern_matching_newlines = false;
diff --git a/test/prism/errors/pinned_back_reference.txt b/test/prism/errors/pinned_back_reference.txt
new file mode 100644
index 0000000000..7d1ef9ea03
--- /dev/null
+++ b/test/prism/errors/pinned_back_reference.txt
@@ -0,0 +1,4 @@
+1 in ^$&
+     ^ expected a pattern expression after the `^` pin operator
+      ^~ unexpected back reference, expecting end-of-input
+
diff --git a/test/prism/errors/pinned_numbered_reference.txt b/test/prism/errors/pinned_numbered_reference.txt
new file mode 100644
index 0000000000..3d6639ab54
--- /dev/null
+++ b/test/prism/errors/pinned_numbered_reference.txt
@@ -0,0 +1,4 @@
+1 in ^$1
+     ^ expected a pattern expression after the `^` pin operator
+      ^~ unexpected numbered reference, expecting end-of-input
+