feat(core,ocr-tesseract): read scanned PDFs by OCR (extension 0.8.0)

apps/vscode-extension/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opencraw/vscode",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "private": true,
   "description": "OpenCraw in the editor: open folders, recipes and documents in Studio, run your recipes from a view and save the result in the format you choose, and learn how it all works.",
   "type": "module",
@@ -348,6 +348,7 @@
           "build",
           "@opencraw/cli:build",
           "@opencraw/mcp:build",
+          "@opencraw/ocr-tesseract:build",
           "@opencraw/agent:build",
           "@opencraw/studio:build",
           "@opencraw/studio-ui:build"
@@ -364,6 +365,7 @@
           "build",
           "@opencraw/cli:build",
           "@opencraw/mcp:build",
+          "@opencraw/ocr-tesseract:build",
           "@opencraw/agent:build",
           "@opencraw/studio:build",
           "@opencraw/studio-ui:build"

apps/vscode-extension/packaging/package-vsix.mjs

@@ -11,7 +11,7 @@ const appDir = resolve(dirname(fileURLToPath(import.meta.url)), '..')
 const repoRoot = resolve(appDir, '..', '..')
 const stage = join(appDir, 'dist-vsix')
 const tarballs = join(stage, 'tarballs')
-const PACKAGES = ['core', 'office-reader', 'probe', 'studio', 'cli', 'mcp', 'agent']
+const PACKAGES = ['core', 'office-reader', 'ocr-tesseract', 'probe', 'studio', 'cli', 'mcp', 'agent']
 
 // Run npm's own CLI with this Node: no shell, so no quoting to get wrong and no .cmd shim on Windows.
 const npmCli = process.env.npm_execpath ?? [

docs/how-it-works/06-documents.md

@@ -204,8 +204,9 @@ stellantis-it: 147 emitted, 0 rejected, 0 duplicates, 1 pages
    The font name, the direction (`dir`), the rotation part of the matrix, colours and every drawing are
    **not** kept: the reader works from text positions alone, and never sees table ruling.
 5. The page's runs go through `assembleRows`. If **every** page ends with no row, the read fails with
-   `no page has a text layer (a scanned PDF? OCR is not supported)` (`:40`). A PDF where only some pages are
-   scans reads, and those pages have no rows.
+   `no page has a text layer: it is a scan. Read it with OCR…`. A PDF where only some pages are scans reads, and
+   those pages have no rows, unless the request has `ocr` (see below): then those pages are rendered, read by the OCR engine and their words
+   become runs like a text layer's.
 
 The result, `{ kind: 'pdf', pages: [{ number, width, height, rows: [{ top, bottom, cells, text }] }] }`, is
 what `jsonpath` queries. A cell is `{ x, y, width, height, text }`; a row's `top` is its highest `y + height`,
@@ -676,7 +677,7 @@ as on workbooks ([`extract-from-document.use-case.ts:128`](../../packages/core/s
 
 | Case | What happens | Why |
 |---|---|---|
-| A scanned page | No rows on it. If every page is a scan, the read fails. | pdf.js reads the text layer; there is no OCR. |
+| A scanned page | No rows on it. If every page is a scan, the read fails. With `ocr` the page is read from its picture. | pdf.js reads the text layer; OCR is opt-in. |
 | Password-protected PDF | The read fails at load. | No password is passed to `getDocument`. |
 | Rotated or vertical text | Rows and columns come out wrong. | Only the origin and `width`/`height` are kept; every stage assumes horizontal, left-to-right baselines. A page with `/Rotate` reports its rotated size, but the item coordinates are not transformed. |
 | Two columns closer than `JOIN_GAP` | They merge into one cell (3.5 pt at 10 pt). | Nothing but the gap separates two runs on a line. |

docs/recipes/authoring.md

@@ -265,7 +265,7 @@ call their JSON endpoints: a report whose table the page reads from an API is re
 
 | Step | Fields | Notes |
 |---|---|---|
-| `request` | `url` (template), `method?`, `query?`, `headers?`, `body?` or `form?`, `as?` (`json`, `jsonl`, `html`, `text`, `pdf`, `csv`, `xlsx`, `pptx`, `yaml`, `markdown`, `xml`, `docx`, `youtube`: §4.14), `encoding?`, `delimiter?`, `scalars?`, `youtube?` | The response becomes the current document and, if `id` is set, the id holds the parsed JSON, the read PDF, workbook or deck, the markup or the text. Relative URLs resolve against the current page; one with no page to resolve against fails with a message saying so. Without `as`, the content type decides (`application/pdf` is a PDF, `text/csv` a CSV, a spreadsheet type a workbook, a presentation type a deck, `application/yaml` YAML, `application/x-ndjson` JSON Lines, `text/markdown` Markdown, an XML type XML: §4.12, a Word type a document read as HTML: §4.13); a type that says nothing (below) is sniffed. A `file:` URL reads a local file, its kind from `as`, else the extension (`.pdf`, `.csv`, `.tsv`, `.xlsx`, `.xlsm`, `.pptx`, `.yaml`, `.yml`, `.json`, `.jsonl`, `.ndjson`, `.md`, `.html`, `.xml`, `.rss`, `.atom`, `.xml.gz`, `.docx`), else its bytes; a relative one (`file:data/listino.csv`) is read from the recipe file's folder (below). 4xx/5xx fail the step. |
+| `request` | `url` (template), `method?`, `query?`, `headers?`, `body?` or `form?`, `as?` (`json`, `jsonl`, `html`, `text`, `pdf`, `csv`, `xlsx`, `pptx`, `yaml`, `markdown`, `xml`, `docx`, `youtube`: §4.14), `encoding?`, `delimiter?`, `scalars?`, `youtube?`, `ocr?` (a scanned PDF: see **A scanned PDF** in the PDF section) | The response becomes the current document and, if `id` is set, the id holds the parsed JSON, the read PDF, workbook or deck, the markup or the text. Relative URLs resolve against the current page; one with no page to resolve against fails with a message saying so. Without `as`, the content type decides (`application/pdf` is a PDF, `text/csv` a CSV, a spreadsheet type a workbook, a presentation type a deck, `application/yaml` YAML, `application/x-ndjson` JSON Lines, `text/markdown` Markdown, an XML type XML: §4.12, a Word type a document read as HTML: §4.13); a type that says nothing (below) is sniffed. A `file:` URL reads a local file, its kind from `as`, else the extension (`.pdf`, `.csv`, `.tsv`, `.xlsx`, `.xlsm`, `.pptx`, `.yaml`, `.yml`, `.json`, `.jsonl`, `.ndjson`, `.md`, `.html`, `.xml`, `.rss`, `.atom`, `.xml.gz`, `.docx`), else its bytes; a relative one (`file:data/listino.csv`) is read from the recipe file's folder (below). 4xx/5xx fail the step. |
 
 **A body served as anything** (`application/octet-stream`, `binary/octet-stream`, `text/plain`, `application/zip`
 and the like, or no content type: a file behind a bucket or a CDN, a download link, a raw file) is read by what it
@@ -630,7 +630,7 @@ repeating element (`table.credit_group tr`, not `tr`) and check the first record
 `request` with `as: "pdf"` (or a response served as `application/pdf`, or a local `file:…pdf`) reads the
 PDF's text layer with pdf.js into pages of **rows**: text that sits side by side becomes a cell, cells whose
 vertical extents overlap become a row. A scanned page has no text layer and gives no rows; a PDF where no page
-has one fails the step (no OCR). Five extract kinds read it:
+has one fails the step, unless the request says `"ocr": true` (below). Five extract kinds read it:
 
 | `kind` | Reads | Use for |
 |---|---|---|
@@ -640,6 +640,21 @@ has one fails the step (no OCR). Five extract kinds read it:
 | `regex` | the text: one line per row, cells separated by a tab, pages by a blank line | a value next to a label: `Valid until\t(\S+)` |
 | `jsonpath` | the structure: `{ kind: "pdf", pages: [{ number, width, height, rows: [{ text, top, bottom, cells: [{ x, y, width, height, text }] }] }] }` | positions, a given page |
 
+**A scanned PDF** has pictures where the text should be. Say `"ocr": true` on the `request` and OpenCraw renders each page
+that has no text, reads it with Tesseract, and hands the words on as the same positioned cells a text layer gives, so every kind
+above (`table`, `region`, `section`) works on a scan as it does on a digital PDF:
+
+```json
+{ "type": "request", "id": "doc", "url": "{{start.url}}", "as": "pdf", "ocr": { "languages": ["por", "eng"] } }
+```
+
+`ocr` is `true` (English, only the pages that have no text) or an object: `languages` (Tesseract codes; `eng` comes with the engine, any
+other is `npm install @tesseract.js-data/<code>`), `always` (read pages that have text too, for a PDF with a wrong text layer from bad OCR;
+the text layer is then dropped) and `dpi` (default: the scan's own resolution, between 150 and 300; rendering a soft scan larger only blurs
+it). The engine is the `@opencraw/ocr-tesseract` package, installed next to `@opencraw/core` (the request says how if it is not). It
+straightens a crooked page, flattens a shaded one and tries a second, stronger cleaning when the first reads poorly. OCR reads
+what is printed; it can still confuse a character (`8,0%` read as `8,09`), so check a number that matters against a second source.
+
 **Prefer structure to position.** A `region` is a place on the page: it breaks when a description is longer, a
 table has one more row or a column is added. Two sheets of the same layout and different content are read with
 `section` (the text between two headings) and `regex` (the value after a label), which find their text wherever it

docs/requirements.md

@@ -175,7 +175,7 @@ steps; section 5).
 | `evaluate` | web | value | `script`, JavaScript run in the page; `args?`. Trusted recipes only. |
 | `screenshot` | web | – | `path` |
 | `captcha` | web | – | `solver?`, `selector?`, `verify?`, `attempts?`, `timeoutMs?`; for form captchas `image`, `refresh?`, `field`, `submit`; defaults from `session.captcha` |
-| `request` | both | document | in web mode through the page's session, with `form?` (`{ selector, omit?, set? }`) instead of `body`; `method?`, `url` (`http(s):` or a local `file:`; relative to the current page, else a clear error), `query?`, `headers?`, `body?` (templated at every depth), `as: 'json' \| 'jsonl' \| 'html' \| 'text' \| 'pdf' \| 'csv' \| 'xlsx' \| 'pptx' \| 'yaml' \| 'markdown' \| 'xml' \| 'docx'`, `encoding?`, `delimiter?` (CSV), `scalars?` (YAML) |
+| `request` | both | document | in web mode through the page's session, with `form?` (`{ selector, omit?, set? }`) instead of `body`; `method?`, `url` (`http(s):` or a local `file:`; relative to the current page, else a clear error), `query?`, `headers?`, `body?` (templated at every depth), `as: 'json' \| 'jsonl' \| 'html' \| 'text' \| 'pdf' \| 'csv' \| 'xlsx' \| 'pptx' \| 'yaml' \| 'markdown' \| 'xml' \| 'docx'`, `encoding?`, `delimiter?` (CSV), `scalars?` (YAML), `ocr?` (a scanned PDF: `true`, or `{ languages?, always?, dpi? }`) |
 | `extract` | both | value or list | `selector` (a template), `kind: 'css' \| 'xpath' \| 'jsonpath' \| 'regex' \| 'table' \| 'region'`, `take`, `many?`, `from?`; `table` also `columns?`, `until?`, `align?` (PDF), `fillDown?`, `sheet?`, `headerRows?`, `includeHidden?` (workbook), `slide?`, `shapes?` (deck) |
 | `set` | both | value | `value` (template or literal) |
 | `collect` | both | – | `into` (a list id bound in an enclosing scope), `value` (template or literal); appends, so values outlive the `forEach` iteration or `paginate` page that found them |
@@ -197,7 +197,7 @@ the folder of the recipe file that names it when the recipe is read from a file
 `vars`, `matrix` values, every step `url`; a template's placeholders are kept), else against the process working
 directory. `allowedHosts` refuses `file:` URLs, relative or not.
 
-A PDF (`as: 'pdf'`) is read into pages of rows of positioned cells (pdf.js, text layer only; a scan fails).
+A PDF (`as: 'pdf'`) is read into pages of rows of positioned cells (pdf.js on the text layer; a scan fails unless the request has `ocr`, which reads its pages by OCR with `@opencraw/ocr-tesseract` and gives the words as the same positioned cells, so `table` and `region` work on a scan too).
 `table` finds tables by their header row and returns `{ page, title, header, rows }`, rows keyed by column:
 columns come from where the body's cells start, and lines of a wrapped cell are regrouped into their row.
 `regex` reads a PDF as text (one line per row, cells tab-separated), `jsonpath` reads its structure. `region`

package.json

@@ -33,6 +33,8 @@
     "captcha-tesseract:qa": "nx run captcha-tesseract:lint && nx run captcha-tesseract:test",
     "captcha-tesseract:build": "nx run captcha-tesseract:build",
     "captcha-tesseract:e2e": "nx run captcha-tesseract:e2e",
+    "ocr-tesseract:qa": "nx run ocr-tesseract:lint && nx run ocr-tesseract:test",
+    "ocr-tesseract:build": "nx run ocr-tesseract:build",
     "azure-durable:qa": "nx run azure-durable:lint && nx run azure-durable:test",
     "azure-durable:build": "nx run azure-durable:build",
     "azure-durable:e2e": "nx run azure-durable:e2e",

packages/core/package.json

@@ -20,6 +20,7 @@
     "!**/*.js.map"
   ],
   "dependencies": {
+    "@napi-rs/canvas": "^1.0.0",
     "@opencraw/office-reader": "^0.0.1",
     "@xmldom/xmldom": "^0.9.12",
     "cheerio": "^1.2.0",

packages/core/schemas/input-recipe.schema.json

@@ -1617,6 +1617,35 @@
                 "text"
               ]
             },
+            "ocr": {
+              "anyOf": [
+                {
+                  "type": "boolean"
+                },
+                {
+                  "type": "object",
+                  "properties": {
+                    "languages": {
+                      "minItems": 1,
+                      "type": "array",
+                      "items": {
+                        "type": "string",
+                        "pattern": "^[a-z]{3}(?:_[a-z]+)?$"
+                      }
+                    },
+                    "always": {
+                      "type": "boolean"
+                    },
+                    "dpi": {
+                      "type": "integer",
+                      "minimum": 72,
+                      "maximum": 600
+                    }
+                  },
+                  "additionalProperties": false
+                }
+              ]
+            },
             "youtube": {
               "type": "object",
               "properties": {

packages/core/src/api-steps/send-request.use-case.ts

@@ -43,6 +43,7 @@ export async function sendRequest (step: RequestStep, scope: ExtractionScope, cl
     encoding:  step.encoding,
     delimiter: step.delimiter,
     scalars:   step.scalars,
+    ocr:       step.ocr === false ? undefined : step.ocr,
     // The video's address is templated like the rest; the languages may be too (`{{start.lang}}`).
     youtube:   step.youtube === undefined ? undefined : { ...step.youtube, ...(step.youtube.language !== undefined && { language: renderText(step.youtube.language, lookup) }), ...(step.youtube.translateTo !== undefined && { translateTo: renderText(step.youtube.translateTo, lookup) }) },
     timeoutMs: recipe.limits?.timeoutMs,

packages/core/src/http-session/http-response.contract.ts

@@ -1,5 +1,6 @@
 import type { DeckDocument } from '../deck-document'
 import type { PdfDocument } from '../pdf-document'
+import type { OcrSetting } from '../pdf-ocr'
 import type { WorkbookDocument } from '../workbook-document'
 import type { XmlDocument } from '../xml-document'
 import type { BodyKind, HttpMethod } from '../recipe-schema'
@@ -20,6 +21,8 @@ export interface HttpRequest {
   delimiter?: string
   /** A YAML body's scalars: `typed` (default) or `text`. */
   scalars?:   'typed' | 'text'
+  /** Read a PDF that is a scan by OCR (needs `@opencraw/ocr-tesseract`): `true`, or the languages, `always` and `dpi`. */
+  ocr?:       true | OcrSetting
   /** With `as: 'youtube'`: the transcript's language, a translation, or no transcript. */
   youtube?:   YoutubeOptions
   timeoutMs?: number

packages/core/src/http-session/http.client.ts

@@ -9,6 +9,8 @@ import { readDocxHtml } from '../docx-document'
 import { readMarkdown } from '../markdown-document'
 import { openDocumentFormat } from '../opendocument-sniff'
 import { readPdf } from '../pdf-document'
+import { withOcr } from '../pdf-ocr'
+import type { OcrSetting } from '../pdf-ocr'
 import type { BodyKind } from '../recipe-schema'
 import { csvWorkbook, readXlsxWorkbook, sheetNameOf } from '../workbook-document'
 import { parseJsonLike, parseJsonLines } from '../selection'
@@ -182,7 +184,7 @@ export class HttpClient implements HttpSender {
  * @param reading - `as`, `encoding`, `delimiter`, `scalars`.
  * @returns The document, what reading noticed, and the format.
  */
-export async function readFileBody (bytes: Uint8Array, name: string, reading: Pick<HttpRequest, 'as' | 'encoding' | 'delimiter' | 'scalars'> = {}): Promise<ReadBody> {
+export async function readFileBody (bytes: Uint8Array, name: string, reading: Pick<HttpRequest, 'as' | 'encoding' | 'delimiter' | 'scalars' | 'ocr'> = {}): Promise<ReadBody> {
   return parseBody(reading.as ?? formatOfFile(name, bytes), bytes, name, reading)
 }
 
@@ -214,7 +216,7 @@ async function readLocalFile (httpRequest: HttpRequest): Promise<HttpResponse> {
   return { status: 200, url: httpRequest.url, headers: {}, body, format, ...(warnings.length > 0 && { warnings }) }
 }
 
-async function parseBody (format: BodyKind, bytes: Uint8Array, url: string, reading: { encoding?: string, delimiter?: string, scalars?: 'typed' | 'text', charset?: string }): Promise<ReadBody> {
+async function parseBody (format: BodyKind, bytes: Uint8Array, url: string, reading: { encoding?: string, delimiter?: string, scalars?: 'typed' | 'text', charset?: string, ocr?: true | OcrSetting }): Promise<ReadBody> {
   if (format === 'youtube') throw new Error(`${url}: "youtube" reads a video address, not a file or a response: use it as the "as" of a request step to a YouTube address`)
   if (format === 'yaml') {
     const { text } = decodeText(bytes, reading)
@@ -244,8 +246,8 @@ async function parseBody (format: BodyKind, bytes: Uint8Array, url: string, read
   return { body: await parseFormat(format, bytes, url, reading), warnings: [], format }
 }
 
-async function parseFormat (format: BodyKind, bytes: Uint8Array, url: string, reading: { encoding?: string, delimiter?: string, charset?: string }): Promise<HttpBody> {
-  if (format === 'pdf') return readPdf(bytes, url)
+async function parseFormat (format: BodyKind, bytes: Uint8Array, url: string, reading: { encoding?: string, delimiter?: string, charset?: string, ocr?: true | OcrSetting }): Promise<HttpBody> {
+  if (format === 'pdf') return reading.ocr === undefined ? readPdf(bytes, url) : withOcr(reading.ocr, async ocr => readPdf(bytes, url, { ocr }))
   if (format === 'xlsx') return readXlsxWorkbook(bytes, url)
   if (format === 'pptx') return readPptxDeck(bytes, url)
   const { text, encoding } = decodeText(bytes, reading)