feat(studio-ui): propose output fields from what the steps read

apps/studio-ui/src/record-editor/field-row.component.tsx

@@ -1,7 +1,8 @@
 import { useState } from 'react'
 import { Badge, Box, Button, Checkbox, HStack, IconButton, Input, NativeSelect, Stack, Text } from '@chakra-ui/react'
 import type { FieldTraceView } from '@opencraw/studio'
-import { PILL_DRAG_MIME } from '../steps-outline'
+import { PILL_DRAG_MIME, Pill } from '../steps-outline'
+import { typeColor } from './field-type-color.mapper'
 import { FIELD_NAME_PATTERN, FIELD_TYPES, MISSING_POLICIES } from './field-spec.model'
 import type { FieldSpecJson, FromRuleJson } from './field-spec.model'
 import { TransformChain } from './transform-chain.component'
@@ -16,6 +17,8 @@ export interface FieldRowProps {
   scopeIds:           string[]
   /** The selected sample record's trace for this field, when it was mapped and a sample has run. */
   trace?:             FieldTraceView
+  /** The selected sample record's value for this field, shown beside its source so a mapping reads as "pill → value" without running anything else; `undefined` before a sample has run. */
+  value?:             unknown
   /** `true` when the selected record's value for this field is `null` (the missing-value policy applied) — the row's Why? affordance. */
   missing?:           boolean
   /** How many of the last sample's records this field rejected, and why (its most recent reason); `undefined` when none did. */
@@ -41,7 +44,7 @@ export interface FieldRowProps {
  * Steps tab), the transform chain with real trace values, key/required, and
  * an expanded row for `nullable`/`default`/`onMissing`/validation options.
  */
-export function FieldRow ({ name, field, rule, scopeIds, trace, missing, rejected, depth = 0, stepId, highlighted = false, onHoverStepId, onFieldChange, onRuleChange, onRename, onRemove, onExplainMissing, onExplainRejected }: FieldRowProps) {
+export function FieldRow ({ name, field, rule, scopeIds, trace, value, missing, rejected, depth = 0, stepId, highlighted = false, onHoverStepId, onFieldChange, onRuleChange, onRename, onRemove, onExplainMissing, onExplainRejected }: FieldRowProps) {
   const [expanded, setExpanded] = useState(false)
   const [nameText, setNameText] = useState(name)
   const [nameError, setNameError] = useState<string>()
@@ -72,6 +75,16 @@ export function FieldRow ({ name, field, rule, scopeIds, trace, missing, rejecte
       onMouseEnter={stepId === undefined || onHoverStepId === undefined ? undefined : () => { onHoverStepId(stepId) }}
       onMouseLeave={stepId === undefined || onHoverStepId === undefined ? undefined : () => { onHoverStepId(undefined) }}
     >
+      <HStack data-testid={`${name}-summary`} gap={2} mb={2} align='center' wrap='wrap' fontSize='xs'>
+        <Badge size='sm' colorPalette={typeColor(field.type)}>{field.type}</Badge>
+        {generated
+          ? <Badge size='sm' colorPalette='gray'>generated</Badge>
+          : (stepId === undefined
+              ? <Badge size='sm' colorPalette='orange'>{rule === undefined ? 'not mapped' : 'from a template'}</Badge>
+              : <><Text color='fg.muted'>←</Text><Pill name={stepId} draggable={false} /></>)}
+        {value !== undefined && <Text color='fg.muted' truncate maxW='260px' title={previewOf(value)}>= {previewOf(value)}</Text>}
+      </HStack>
+
       <HStack gap={2} align='center' wrap='wrap'>
         <Stack gap={0}>
           <Input size='xs' width='140px' value={nameText} onChange={event => { setNameText(event.target.value) }} onBlur={commitName} />
@@ -226,6 +239,13 @@ export function FieldRow ({ name, field, rule, scopeIds, trace, missing, rejecte
   )
 }
 
+/** A value as one short line: text as it is, anything else as JSON, cut at 80 characters. */
+function previewOf (value: unknown): string {
+  const text = typeof value === 'string' ? value : JSON.stringify(value)
+
+  return text.length > 80 ? `${text.slice(0, 79)}…` : text
+}
+
 /** A small read-only badge for a `key`/`required` field, used by rows that only summarise a nested field rather than editing it fully. Exported for `record-editor.component.tsx`'s each-item header. */
 export function FieldBadges ({ field }: { field: FieldSpecJson }) {
   return (

apps/studio-ui/src/record-editor/field-suggestions.component.tsx

@@ -0,0 +1,52 @@
+import { Badge, Box, Button, HStack, Stack, Text } from '@chakra-ui/react'
+import { Pill } from '../steps-outline'
+import { typeColor } from './field-type-color.mapper'
+import type { FieldSuggestion } from './suggest-fields.algorithm'
+
+export interface FieldSuggestionsProps {
+  suggestions: FieldSuggestion[]
+  /** Adds one proposal: its field and the rule that reads its pill. */
+  onAdd:       (suggestion: FieldSuggestion) => void
+  onAddAll:    () => void
+}
+
+/**
+ * What the steps read that no output field uses yet, one chip each: the pill,
+ * the field it would become and its guessed type. A click adds the field and
+ * its mapping in one go, so building the output is picking, not typing.
+ * Renders nothing when everything read is already used.
+ */
+export function FieldSuggestions ({ suggestions, onAdd, onAddAll }: FieldSuggestionsProps) {
+  if (suggestions.length === 0) return null
+
+  return (
+    <Box data-testid='field-suggestions' borderWidth='1px' borderStyle='dashed' borderRadius='md' p={2} bg='bg.subtle'>
+      <HStack justify='space-between' mb={1}>
+        <Text fontSize='xs' fontWeight='medium'>Read by the steps, not in the output yet</Text>
+        <Button size='2xs' variant='outline' onClick={onAddAll}>Add all ({suggestions.length})</Button>
+      </HStack>
+      <Stack direction='row' wrap='wrap' gap={2}>
+        {suggestions.map(suggestion => (
+          <HStack
+            key={suggestion.source}
+            data-testid={`suggestion-${suggestion.source}`}
+            gap={1}
+            borderWidth='1px'
+            borderRadius='full'
+            pl={1}
+            pr={1}
+            py='1px'
+            bg='bg'
+            title={`${suggestion.name}: ${suggestion.field.type}, because ${suggestion.reason}`}
+          >
+            <Pill name={suggestion.source} draggable={false} />
+            <Text fontSize='xs' color='fg.muted'>→</Text>
+            <Text fontSize='xs' fontWeight='medium'>{suggestion.name}</Text>
+            <Badge size='sm' colorPalette={typeColor(suggestion.field.type)}>{suggestion.field.type}</Badge>
+            <Button size='2xs' variant='ghost' aria-label={`Add ${suggestion.name}`} onClick={() => { onAdd(suggestion) }}>＋</Button>
+          </HStack>
+        ))}
+      </Stack>
+    </Box>
+  )
+}

apps/studio-ui/src/record-editor/field-type-color.mapper.ts

@@ -0,0 +1,25 @@
+const TYPE_COLORS: Record<string, string> = {
+  string:   'gray',
+  number:   'blue',
+  integer:  'blue',
+  boolean:  'green',
+  date:     'purple',
+  datetime: 'purple',
+  currency: 'orange',
+  url:      'cyan',
+  enum:     'pink',
+  array:    'teal',
+  object:   'teal',
+  json:     'yellow',
+}
+
+/**
+ * The colour palette a field type's badge is drawn with, so a record's shape
+ * reads at a glance: text grey, numbers blue, dates purple, links cyan.
+ *
+ * @param type - A field type.
+ * @returns A Chakra colour palette name; grey for an unknown type.
+ */
+export function typeColor (type: string): string {
+  return TYPE_COLORS[type] ?? 'gray'
+}

apps/studio-ui/src/record-editor/index.ts

@@ -7,3 +7,9 @@ export type { TransformChainProps, TransformJson } from './transform-chain.compo
 export { scopeIdsOf } from './scope-ids.algorithm'
 export { FIELD_NAME_PATTERN, FIELD_TYPES, GENERATED_VALUES, MISSING_POLICIES, fieldToStepIdOf, isEachRule } from './field-spec.model'
 export type { EachRuleJson, FieldSpecJson, FromRuleJson, MappingRuleJson } from './field-spec.model'
+
+export { FieldSuggestions } from './field-suggestions.component'
+export type { FieldSuggestionsProps } from './field-suggestions.component'
+export { suggestFields } from './suggest-fields.algorithm'
+export type { FieldSuggestion } from './suggest-fields.algorithm'
+export { typeColor } from './field-type-color.mapper'

apps/studio-ui/src/record-editor/record-editor.component.tsx

@@ -5,9 +5,12 @@ import type { PreviewRecord } from '../preview'
 import type { RejectedRecord } from '../studio-store'
 import { useStudioUiStore } from '../studio-store'
 import { FieldRow } from './field-row.component'
+import { FieldSuggestions } from './field-suggestions.component'
 import type { FieldSpecJson, MappingRuleJson } from './field-spec.model'
 import { isEachRule, MISSING_POLICIES } from './field-spec.model'
 import { scopeIdsOf } from './scope-ids.algorithm'
+import { suggestFields } from './suggest-fields.algorithm'
+import type { FieldSuggestion } from './suggest-fields.algorithm'
 
 export interface RecordEditorProps {
   /** The selected input recipe (its `mapping`), or `undefined` before one is selected. */
@@ -75,6 +78,7 @@ export function RecordEditor ({ inputRecipe, outputRecipe, records, rejected, on
   }
 
   const scopeIds = scopeIdsOf(input)
+  const suggestions = suggestFields(input, output.fields, input.mapping)
   const record = records[recordIndex] as PreviewRecord | undefined
   const dirty = JSON.stringify(output) !== savedOutputJson || JSON.stringify(input) !== savedInputJson
 
@@ -102,6 +106,17 @@ export function RecordEditor ({ inputRecipe, outputRecipe, records, rejected, on
     else mapping[target] = rule
     setInput({ ...input, mapping })
   }
+  /** Adds proposals as fields with their mapping, one state update so the two files change together. */
+  const addSuggestions = (chosen: FieldSuggestion[]): void => {
+    const fields = { ...output.fields }
+    const mapping = { ...input.mapping }
+    for (const suggestion of chosen) {
+      fields[suggestion.name] = suggestion.field
+      mapping[suggestion.name] = suggestion.rule
+    }
+    setOutput({ ...output, fields })
+    setInput({ ...input, mapping })
+  }
   const addField = (): void => {
     let name = 'field'
     let n = 1
@@ -155,6 +170,16 @@ export function RecordEditor ({ inputRecipe, outputRecipe, records, rejected, on
         </HStack>
       </HStack>
 
+      <FieldSuggestions
+        suggestions={suggestions}
+        onAdd={suggestion => { addSuggestions([suggestion]) }}
+        onAddAll={() => { addSuggestions(suggestions) }}
+      />
+
+      {Object.keys(output.fields).length === 0 && suggestions.length === 0 && (
+        <Text fontSize='xs' color='fg.muted'>No output fields yet. Pick values in the content pane (Steps tab) and they show up here as proposals, or add a field by hand.</Text>
+      )}
+
       <Stack gap={2}>
         {Object.entries(output.fields).map(([name, field]) => (
           <FieldTree
@@ -326,6 +351,7 @@ function FieldTree ({ name, field, target, input, record, rejected, scopeIds, de
       rule={fromRule}
       scopeIds={scopeIds}
       trace={trace}
+      value={value}
       missing={value === null}
       // eslint-disable-next-line unicorn/prefer-at -- `.at()` needs an ES2022+ `lib`; `tsconfig.app.json` (the browser build) only has `dom` (see this file's own TS2550 history)
       rejected={rejectedForField.length === 0 ? undefined : { count: rejectedForField.length, reason: rejectedForField[rejectedForField.length - 1].reason }}

apps/studio-ui/src/record-editor/suggest-fields.algorithm.test.ts

@@ -0,0 +1,85 @@
+import { suggestFields } from './suggest-fields.algorithm'
+
+const books = {
+  steps: [
+    { type: 'goto', url: '{{start.url}}' },
+    { type: 'extract', id: 'books', kind: 'css', selector: 'article.product_pod', many: true },
+    {
+      type:  'forEach',
+      over:  'books',
+      as:    'book',
+      steps: [
+        { type: 'extract', id: 'title', from: 'book', kind: 'css', selector: 'h3 a', take: 'attr:title' },
+        { type: 'extract', id: 'price', from: 'book', kind: 'css', selector: '.price_color' },
+        { type: 'extract', id: 'availability', from: 'book', kind: 'css', selector: '.availability' },
+        { type: 'extract', id: 'detailLink', from: 'book', kind: 'css', selector: 'h3 a', take: 'attr:href' },
+        { type: 'emit' },
+      ],
+    },
+  ],
+}
+
+describe('suggestFields', () => {
+  it('proposes a field for each value read in a loop, never for the list it loops over', () => {
+    const suggestions = suggestFields(books, {}, {})
+    expect(suggestions.map(s => [s.source, s.name, s.field.type])).toEqual([
+      ['title', 'title', 'string'],
+      ['price', 'price', 'number'],
+      ['availability', 'availability', 'boolean'],
+      ['detailLink', 'detailLink', 'url'],
+    ])
+  })
+
+  it('makes a relative link absolute, since a url field refuses a relative one', () => {
+    const link = suggestFields(books, {}, {}).find(s => s.source === 'detailLink')
+    expect(link?.rule).toEqual({ from: 'detailLink', transform: [{ op: 'absoluteUrl' }] })
+  })
+
+  it('leaves out what a field already reads, a list id an each rule uses, and what the mapping names in a template path', () => {
+    const mapping = { title: { from: 'title' }, offers: { each: 'books', fields: { cost: { from: 'price' } } }, link: { from: ['detailLink.href'] } }
+    expect(suggestFields(books, { title: { type: 'string' } }, mapping).map(s => s.source)).toEqual(['availability'])
+  })
+
+  it('finds values nested in paginate and if bodies and in else', () => {
+    const recipe = {
+      steps: [
+        { type: 'paginate', steps: [{ type: 'extract', id: 'headline', kind: 'css', selector: 'h1' }] },
+        { type: 'if', steps: [], else: [{ type: 'set', id: 'note', value: 'x' }] },
+      ],
+    }
+    expect(suggestFields(recipe, {}, {}).map(s => s.source)).toEqual(['headline', 'note'])
+  })
+
+  it('proposes a list of text for an extract of every match that nothing loops over', () => {
+    const recipe = { steps: [{ type: 'extract', id: 'genres', kind: 'css', selector: '.genre', many: true }] }
+    const [suggestion] = suggestFields(recipe, {}, {})
+    expect(suggestion.field).toEqual({ type: 'array', items: { type: 'string' } })
+  })
+
+  it('skips tables, requests and navigation: they are not values', () => {
+    const recipe = {
+      steps: [
+        { type: 'request', id: 'doc', url: '{{start.url}}', as: 'pdf' },
+        { type: 'extract', id: 'sheet', kind: 'table', selector: '^Model', many: true },
+        { type: 'extract', id: 'validUntil', kind: 'anchor', selector: 'right of ^Valid until' },
+      ],
+    }
+    expect(suggestFields(recipe, {}, {}).map(s => [s.source, s.field.type])).toEqual([['validUntil', 'date']])
+  })
+
+  it('names a field after its id, legal and unused', () => {
+    const recipe = {
+      steps: [
+        { type: 'extract', id: 'title', kind: 'css', selector: 'h1' },
+        { type: 'extract', id: 'a.b', kind: 'css', selector: 'h2' },
+        { type: 'extract', id: '9lives', kind: 'css', selector: 'h3' },
+      ],
+    }
+    expect(suggestFields(recipe, { title: { type: 'string' } }, {}).map(s => s.name)).toEqual(['title2', 'a_b', 'field_9lives'])
+  })
+
+  it('reads an empty or half-written recipe without throwing', () => {
+    expect(suggestFields(undefined, {}, {})).toEqual([])
+    expect(suggestFields({ steps: [null, 4, { type: 'extract' }] }, {}, {})).toEqual([])
+  })
+})