diff --git a/apps/studio-ui/src/record-editor/field-row.component.tsx b/apps/studio-ui/src/record-editor/field-row.component.tsx index c046671e..0473b750 100644 --- a/apps/studio-ui/src/record-editor/field-row.component.tsx +++ b/apps/studio-ui/src/record-editor/field-row.component.tsx @@ -1,7 +1,8 @@ import { useState } from 'react' import { Badge, Box, Button, Checkbox, HStack, IconButton, Input, NativeSelect, Stack, Text } from '@chakra-ui/react' import type { FieldTraceView } from '@opencraw/studio' -import { PILL_DRAG_MIME } from '../steps-outline' +import { PILL_DRAG_MIME, Pill } from '../steps-outline' +import { typeColor } from './field-type-color.mapper' import { FIELD_NAME_PATTERN, FIELD_TYPES, MISSING_POLICIES } from './field-spec.model' import type { FieldSpecJson, FromRuleJson } from './field-spec.model' import { TransformChain } from './transform-chain.component' @@ -16,6 +17,8 @@ export interface FieldRowProps { scopeIds: string[] /** The selected sample record's trace for this field, when it was mapped and a sample has run. */ trace?: FieldTraceView + /** The selected sample record's value for this field, shown beside its source so a mapping reads as "pill → value" without running anything else; `undefined` before a sample has run. */ + value?: unknown /** `true` when the selected record's value for this field is `null` (the missing-value policy applied) — the row's Why? affordance. */ missing?: boolean /** How many of the last sample's records this field rejected, and why (its most recent reason); `undefined` when none did. */ @@ -41,7 +44,7 @@ export interface FieldRowProps { * Steps tab), the transform chain with real trace values, key/required, and * an expanded row for `nullable`/`default`/`onMissing`/validation options. */ -export function FieldRow ({ name, field, rule, scopeIds, trace, missing, rejected, depth = 0, stepId, highlighted = false, onHoverStepId, onFieldChange, onRuleChange, onRename, onRemove, onExplainMissing, onExplainRejected }: FieldRowProps) { +export function FieldRow ({ name, field, rule, scopeIds, trace, value, missing, rejected, depth = 0, stepId, highlighted = false, onHoverStepId, onFieldChange, onRuleChange, onRename, onRemove, onExplainMissing, onExplainRejected }: FieldRowProps) { const [expanded, setExpanded] = useState(false) const [nameText, setNameText] = useState(name) const [nameError, setNameError] = useState() @@ -72,6 +75,16 @@ export function FieldRow ({ name, field, rule, scopeIds, trace, missing, rejecte onMouseEnter={stepId === undefined || onHoverStepId === undefined ? undefined : () => { onHoverStepId(stepId) }} onMouseLeave={stepId === undefined || onHoverStepId === undefined ? undefined : () => { onHoverStepId(undefined) }} > + + {field.type} + {generated + ? generated + : (stepId === undefined + ? {rule === undefined ? 'not mapped' : 'from a template'} + : <>←)} + {value !== undefined && = {previewOf(value)}} + + { setNameText(event.target.value) }} onBlur={commitName} /> @@ -226,6 +239,13 @@ export function FieldRow ({ name, field, rule, scopeIds, trace, missing, rejecte ) } +/** A value as one short line: text as it is, anything else as JSON, cut at 80 characters. */ +function previewOf (value: unknown): string { + const text = typeof value === 'string' ? value : JSON.stringify(value) + + return text.length > 80 ? `${text.slice(0, 79)}…` : text +} + /** A small read-only badge for a `key`/`required` field, used by rows that only summarise a nested field rather than editing it fully. Exported for `record-editor.component.tsx`'s each-item header. */ export function FieldBadges ({ field }: { field: FieldSpecJson }) { return ( diff --git a/apps/studio-ui/src/record-editor/field-suggestions.component.tsx b/apps/studio-ui/src/record-editor/field-suggestions.component.tsx new file mode 100644 index 00000000..0be5f3da --- /dev/null +++ b/apps/studio-ui/src/record-editor/field-suggestions.component.tsx @@ -0,0 +1,52 @@ +import { Badge, Box, Button, HStack, Stack, Text } from '@chakra-ui/react' +import { Pill } from '../steps-outline' +import { typeColor } from './field-type-color.mapper' +import type { FieldSuggestion } from './suggest-fields.algorithm' + +export interface FieldSuggestionsProps { + suggestions: FieldSuggestion[] + /** Adds one proposal: its field and the rule that reads its pill. */ + onAdd: (suggestion: FieldSuggestion) => void + onAddAll: () => void +} + +/** + * What the steps read that no output field uses yet, one chip each: the pill, + * the field it would become and its guessed type. A click adds the field and + * its mapping in one go, so building the output is picking, not typing. + * Renders nothing when everything read is already used. + */ +export function FieldSuggestions ({ suggestions, onAdd, onAddAll }: FieldSuggestionsProps) { + if (suggestions.length === 0) return null + + return ( + + + Read by the steps, not in the output yet + + + + {suggestions.map(suggestion => ( + + + → + {suggestion.name} + {suggestion.field.type} + + + ))} + + + ) +} diff --git a/apps/studio-ui/src/record-editor/field-type-color.mapper.ts b/apps/studio-ui/src/record-editor/field-type-color.mapper.ts new file mode 100644 index 00000000..a2241ff5 --- /dev/null +++ b/apps/studio-ui/src/record-editor/field-type-color.mapper.ts @@ -0,0 +1,25 @@ +const TYPE_COLORS: Record = { + string: 'gray', + number: 'blue', + integer: 'blue', + boolean: 'green', + date: 'purple', + datetime: 'purple', + currency: 'orange', + url: 'cyan', + enum: 'pink', + array: 'teal', + object: 'teal', + json: 'yellow', +} + +/** + * The colour palette a field type's badge is drawn with, so a record's shape + * reads at a glance: text grey, numbers blue, dates purple, links cyan. + * + * @param type - A field type. + * @returns A Chakra colour palette name; grey for an unknown type. + */ +export function typeColor (type: string): string { + return TYPE_COLORS[type] ?? 'gray' +} diff --git a/apps/studio-ui/src/record-editor/index.ts b/apps/studio-ui/src/record-editor/index.ts index 549ceb23..3e99963d 100644 --- a/apps/studio-ui/src/record-editor/index.ts +++ b/apps/studio-ui/src/record-editor/index.ts @@ -7,3 +7,9 @@ export type { TransformChainProps, TransformJson } from './transform-chain.compo export { scopeIdsOf } from './scope-ids.algorithm' export { FIELD_NAME_PATTERN, FIELD_TYPES, GENERATED_VALUES, MISSING_POLICIES, fieldToStepIdOf, isEachRule } from './field-spec.model' export type { EachRuleJson, FieldSpecJson, FromRuleJson, MappingRuleJson } from './field-spec.model' + +export { FieldSuggestions } from './field-suggestions.component' +export type { FieldSuggestionsProps } from './field-suggestions.component' +export { suggestFields } from './suggest-fields.algorithm' +export type { FieldSuggestion } from './suggest-fields.algorithm' +export { typeColor } from './field-type-color.mapper' diff --git a/apps/studio-ui/src/record-editor/record-editor.component.tsx b/apps/studio-ui/src/record-editor/record-editor.component.tsx index 4a555b32..8e995b4f 100644 --- a/apps/studio-ui/src/record-editor/record-editor.component.tsx +++ b/apps/studio-ui/src/record-editor/record-editor.component.tsx @@ -5,9 +5,12 @@ import type { PreviewRecord } from '../preview' import type { RejectedRecord } from '../studio-store' import { useStudioUiStore } from '../studio-store' import { FieldRow } from './field-row.component' +import { FieldSuggestions } from './field-suggestions.component' import type { FieldSpecJson, MappingRuleJson } from './field-spec.model' import { isEachRule, MISSING_POLICIES } from './field-spec.model' import { scopeIdsOf } from './scope-ids.algorithm' +import { suggestFields } from './suggest-fields.algorithm' +import type { FieldSuggestion } from './suggest-fields.algorithm' export interface RecordEditorProps { /** The selected input recipe (its `mapping`), or `undefined` before one is selected. */ @@ -75,6 +78,7 @@ export function RecordEditor ({ inputRecipe, outputRecipe, records, rejected, on } const scopeIds = scopeIdsOf(input) + const suggestions = suggestFields(input, output.fields, input.mapping) const record = records[recordIndex] as PreviewRecord | undefined const dirty = JSON.stringify(output) !== savedOutputJson || JSON.stringify(input) !== savedInputJson @@ -102,6 +106,17 @@ export function RecordEditor ({ inputRecipe, outputRecipe, records, rejected, on else mapping[target] = rule setInput({ ...input, mapping }) } + /** Adds proposals as fields with their mapping, one state update so the two files change together. */ + const addSuggestions = (chosen: FieldSuggestion[]): void => { + const fields = { ...output.fields } + const mapping = { ...input.mapping } + for (const suggestion of chosen) { + fields[suggestion.name] = suggestion.field + mapping[suggestion.name] = suggestion.rule + } + setOutput({ ...output, fields }) + setInput({ ...input, mapping }) + } const addField = (): void => { let name = 'field' let n = 1 @@ -155,6 +170,16 @@ export function RecordEditor ({ inputRecipe, outputRecipe, records, rejected, on + { addSuggestions([suggestion]) }} + onAddAll={() => { addSuggestions(suggestions) }} + /> + + {Object.keys(output.fields).length === 0 && suggestions.length === 0 && ( + No output fields yet. Pick values in the content pane (Steps tab) and they show up here as proposals, or add a field by hand. + )} + {Object.entries(output.fields).map(([name, field]) => ( { + it('proposes a field for each value read in a loop, never for the list it loops over', () => { + const suggestions = suggestFields(books, {}, {}) + expect(suggestions.map(s => [s.source, s.name, s.field.type])).toEqual([ + ['title', 'title', 'string'], + ['price', 'price', 'number'], + ['availability', 'availability', 'boolean'], + ['detailLink', 'detailLink', 'url'], + ]) + }) + + it('makes a relative link absolute, since a url field refuses a relative one', () => { + const link = suggestFields(books, {}, {}).find(s => s.source === 'detailLink') + expect(link?.rule).toEqual({ from: 'detailLink', transform: [{ op: 'absoluteUrl' }] }) + }) + + it('leaves out what a field already reads, a list id an each rule uses, and what the mapping names in a template path', () => { + const mapping = { title: { from: 'title' }, offers: { each: 'books', fields: { cost: { from: 'price' } } }, link: { from: ['detailLink.href'] } } + expect(suggestFields(books, { title: { type: 'string' } }, mapping).map(s => s.source)).toEqual(['availability']) + }) + + it('finds values nested in paginate and if bodies and in else', () => { + const recipe = { + steps: [ + { type: 'paginate', steps: [{ type: 'extract', id: 'headline', kind: 'css', selector: 'h1' }] }, + { type: 'if', steps: [], else: [{ type: 'set', id: 'note', value: 'x' }] }, + ], + } + expect(suggestFields(recipe, {}, {}).map(s => s.source)).toEqual(['headline', 'note']) + }) + + it('proposes a list of text for an extract of every match that nothing loops over', () => { + const recipe = { steps: [{ type: 'extract', id: 'genres', kind: 'css', selector: '.genre', many: true }] } + const [suggestion] = suggestFields(recipe, {}, {}) + expect(suggestion.field).toEqual({ type: 'array', items: { type: 'string' } }) + }) + + it('skips tables, requests and navigation: they are not values', () => { + const recipe = { + steps: [ + { type: 'request', id: 'doc', url: '{{start.url}}', as: 'pdf' }, + { type: 'extract', id: 'sheet', kind: 'table', selector: '^Model', many: true }, + { type: 'extract', id: 'validUntil', kind: 'anchor', selector: 'right of ^Valid until' }, + ], + } + expect(suggestFields(recipe, {}, {}).map(s => [s.source, s.field.type])).toEqual([['validUntil', 'date']]) + }) + + it('names a field after its id, legal and unused', () => { + const recipe = { + steps: [ + { type: 'extract', id: 'title', kind: 'css', selector: 'h1' }, + { type: 'extract', id: 'a.b', kind: 'css', selector: 'h2' }, + { type: 'extract', id: '9lives', kind: 'css', selector: 'h3' }, + ], + } + expect(suggestFields(recipe, { title: { type: 'string' } }, {}).map(s => s.name)).toEqual(['title2', 'a_b', 'field_9lives']) + }) + + it('reads an empty or half-written recipe without throwing', () => { + expect(suggestFields(undefined, {}, {})).toEqual([]) + expect(suggestFields({ steps: [null, 4, { type: 'extract' }] }, {}, {})).toEqual([]) + }) +}) diff --git a/apps/studio-ui/src/record-editor/suggest-fields.algorithm.ts b/apps/studio-ui/src/record-editor/suggest-fields.algorithm.ts new file mode 100644 index 00000000..5afa06c8 --- /dev/null +++ b/apps/studio-ui/src/record-editor/suggest-fields.algorithm.ts @@ -0,0 +1,151 @@ +import { FIELD_NAME_PATTERN } from './field-spec.model' +import type { FieldSpecJson, FromRuleJson, MappingRuleJson } from './field-spec.model' + +/** One output field the Record tab proposes for a value the steps read and no field uses yet. */ +export interface FieldSuggestion { + /** The step id the field would read (its pill). */ + source: string + /** The proposed field name: the id made a legal, unused field name. */ + name: string + field: FieldSpecJson + rule: FromRuleJson + /** Why this type, in a few words (a tooltip). */ + reason: string +} + +interface Typed { + type: string + reason: string + items?: FieldSpecJson + transform?: { op: string }[] +} + +type StepJson = Record + +/** The attributes a link or an image source is taken from. */ +const LINK_TAKES: ReadonlySet = new Set(['attr:href', 'attr:src', 'attr:data-src']) + +/** Words in a step id that say what its value is. First match wins, so a more specific kind comes first. */ +const WORD_TYPES: readonly (readonly [readonly string[], Typed])[] = [ + [['url', 'link', 'href', 'website', 'homepage', 'image', 'img', 'photo', 'thumbnail', 'src'], { type: 'url', reason: 'its name says a link' }], + [['datetime', 'timestamp'], { type: 'datetime', reason: 'its name says a moment in time' }], + [['date', 'day', 'published', 'created', 'updated', 'released', 'expires', 'expiry', 'until', 'since'], { type: 'date', reason: 'its name says a date' }], + [['available', 'availability', 'active', 'enabled', 'instock', 'is', 'has'], { type: 'boolean', reason: 'its name says yes or no' }], + [['qty', 'quantity', 'count', 'pages', 'year', 'age', 'rank', 'position', 'votes', 'reviews'], { type: 'integer', reason: 'its name says a whole number' }], + [['price', 'cost', 'amount', 'total', 'fee', 'discount', 'salary', 'rate', 'rating', 'score', 'weight', 'height', 'width', 'length', 'percent', 'sconto'], { type: 'number', reason: 'its name says an amount' }], +] + +/** + * The output fields to propose for what a recipe's steps read: every `extract` + * (and `set`) with an id that no mapping rule uses yet and that no `forEach` + * loops over. Each gets a legal, unused name, a type guessed from how the value + * was taken and from the words in its id, and the rule that reads it. + * + * @param input - The input recipe's parsed JSON (possibly mid-edit). + * @param fields - The output recipe's fields so far. + * @param mapping - The input recipe's mapping so far. + * @returns The proposals, in the order the steps read them. + */ +export function suggestFields (input: unknown, fields: Record, mapping: Record): FieldSuggestion[] { + const steps = flatten(stepsOf(input)) + const looped = new Set(steps.flatMap(step => (typeof step.over === 'string' ? [step.over] : []))) + const used = mappedIds(mapping) + const taken = new Set(Object.keys(fields)) + const suggestions: FieldSuggestion[] = [] + const seen = new Set() + for (const step of steps) { + const id = step.id + if (typeof id !== 'string' || id === '' || !isReadStep(step) || looped.has(id) || used.has(id) || seen.has(id)) continue + seen.add(id) + const typed = inferType(id, step) + const name = freeName(id, taken) + taken.add(name) + suggestions.push({ + source: id, + name, + field: { type: typed.type, ...(typed.items !== undefined && { items: typed.items }) }, + rule: { from: id, ...(typed.transform !== undefined && { transform: typed.transform }) }, + reason: typed.reason, + }) + } + + return suggestions +} + +/** The type a value's id and way of being read suggest. Never `currency` (it fails without a code); `number` reads a price with a symbol in it. */ +function inferType (id: string, step: StepJson): Typed { + if (step.many === true) return { type: 'array', items: { type: 'string' }, reason: 'it reads every match, a list' } + const take = typeof step.take === 'string' ? step.take : undefined + if (take !== undefined && LINK_TAKES.has(take)) return { type: 'url', transform: [{ op: 'absoluteUrl' }], reason: `it takes ${take}, a link (made absolute)` } + const words = wordsOf(id) + for (const [keys, typed] of WORD_TYPES) { + if (words.some(word => keys.includes(word))) return typed + } + + return { type: 'string', reason: 'text, the safe default' } +} + +function isReadStep (step: StepJson): boolean { + if (step.type === 'set') return true + + // A table or a whole document is a collection the person loops over, not a value. + return step.type === 'extract' && step.kind !== 'table' +} + +/** Every step id a mapping rule reads: a plain id, a list of ids, an `each` list's id and its members' ids. */ +function mappedIds (mapping: Record): Set { + const ids = new Set() + const add = (rule: FromRuleJson): void => { + const sources = Array.isArray(rule.from) ? rule.from : [rule.from] + for (const from of sources) ids.add(from.split('.', 1)[0]) + } + for (const rule of Object.values(mapping)) { + if ('each' in rule) { + ids.add(rule.each) + for (const member of Object.values(rule.fields)) add(member) + } else { + add(rule) + } + } + + return ids +} + +/** An id as lowercase words: `bookTitle`, `book_title` and `book-title` are `book`, `title`. */ +function wordsOf (id: string): string[] { + // eslint-disable-next-line unicorn/prefer-string-replace-all -- apps/studio-ui's tsconfig lib is ["dom"] only (no ES2021 String#replaceAll). + return id.replace(/([a-z\d])([A-Z])/g, '$1 $2').toLowerCase().split(/[^a-z\d]+/).filter(word => word !== '') +} + +/** The id as a field name no other field has: illegal characters become `_`, a clash gets `2`, `3`... */ +function freeName (id: string, taken: ReadonlySet): string { + // eslint-disable-next-line unicorn/prefer-string-replace-all -- apps/studio-ui's tsconfig lib is ["dom"] only (no ES2021 String#replaceAll). + let base = id.replace(/[^\w-]/g, '_') + if (!FIELD_NAME_PATTERN.test(base)) base = `field_${base}` + let name = base + let n = 2 + while (taken.has(name)) { + name = `${base}${String(n)}` + n += 1 + } + + return name +} + +function stepsOf (value: unknown): unknown[] { + if (typeof value !== 'object' || value === null) return [] + const steps = (value as { steps?: unknown }).steps + + return Array.isArray(steps) ? steps : [] +} + +/** Every step, nested ones (`forEach`, `paginate`, `if` bodies and `else`) included, in reading order. */ +function flatten (steps: unknown[]): StepJson[] { + return steps.flatMap((raw) => { + if (typeof raw !== 'object' || raw === null) return [] + const step = raw as StepJson + const elseSteps = Array.isArray(step.else) ? step.else : [] + + return [step, ...flatten(stepsOf(step)), ...flatten(elseSteps)] + }) +} diff --git a/docs/studio/manual.md b/docs/studio/manual.md index 7fc32ab1..fee29742 100644 --- a/docs/studio/manual.md +++ b/docs/studio/manual.md @@ -149,8 +149,16 @@ Switch to the **Record** tab. This is the paired **output** recipe: the fields a with a type, and the input recipe's **mapping** — which pill each field reads from, and what happens to the value on the way in. -Add fields for `title` (`string`), `price` (`currency`), `available` (`boolean`); drag the **Steps** tab's -`title`/`price`/`availability` pills onto their matching rows to bind each field's source (or type the pill +A strip at the top of the tab lists every value the steps read that no field uses yet, one chip each: +the pill, the field it would become and a guessed type (`price` becomes a `number`, `availability` a +`boolean`, a link taken from `href` a `url` made absolute; hover a chip for why). Click a chip's **＋** to add +the field **and** its mapping at once, or **Add all**. Nothing is guessed silently: change a type or a name +afterwards like any field. + +Each field row opens with a one-line summary: a coloured **type** badge, the **pill** it reads from (or +*not mapped*, *generated*) and, once a sample has run, the **value** the selected record got. + +You can still add a field by hand with **+ field** and drag a **Steps** pill onto its row (or type the pill name directly). Mark `title` as the record's **key**, so a duplicate never gets emitted twice. Each field's **transform chain** is a row of small blocks — `trim`, `currency`, `boolean`, and so on