diff --git a/apps/vscode-extension/package.json b/apps/vscode-extension/package.json index 470f723e..b9205ad2 100644 --- a/apps/vscode-extension/package.json +++ b/apps/vscode-extension/package.json @@ -1,6 +1,6 @@ { "name": "@opencraw/vscode", - "version": "0.7.0", + "version": "0.8.0", "private": true, "description": "OpenCraw in the editor: open folders, recipes and documents in Studio, run your recipes from a view and save the result in the format you choose, and learn how it all works.", "type": "module", @@ -348,6 +348,7 @@ "build", "@opencraw/cli:build", "@opencraw/mcp:build", + "@opencraw/ocr-tesseract:build", "@opencraw/agent:build", "@opencraw/studio:build", "@opencraw/studio-ui:build" @@ -364,6 +365,7 @@ "build", "@opencraw/cli:build", "@opencraw/mcp:build", + "@opencraw/ocr-tesseract:build", "@opencraw/agent:build", "@opencraw/studio:build", "@opencraw/studio-ui:build" diff --git a/apps/vscode-extension/packaging/package-vsix.mjs b/apps/vscode-extension/packaging/package-vsix.mjs index 6e7f58bb..681be3dc 100644 --- a/apps/vscode-extension/packaging/package-vsix.mjs +++ b/apps/vscode-extension/packaging/package-vsix.mjs @@ -11,7 +11,7 @@ const appDir = resolve(dirname(fileURLToPath(import.meta.url)), '..') const repoRoot = resolve(appDir, '..', '..') const stage = join(appDir, 'dist-vsix') const tarballs = join(stage, 'tarballs') -const PACKAGES = ['core', 'office-reader', 'probe', 'studio', 'cli', 'mcp', 'agent'] +const PACKAGES = ['core', 'office-reader', 'ocr-tesseract', 'probe', 'studio', 'cli', 'mcp', 'agent'] // Run npm's own CLI with this Node: no shell, so no quoting to get wrong and no .cmd shim on Windows. const npmCli = process.env.npm_execpath ?? [ diff --git a/docs/how-it-works/06-documents.md b/docs/how-it-works/06-documents.md index 6e7bcf98..e90a6239 100644 --- a/docs/how-it-works/06-documents.md +++ b/docs/how-it-works/06-documents.md @@ -204,8 +204,9 @@ stellantis-it: 147 emitted, 0 rejected, 0 duplicates, 1 pages The font name, the direction (`dir`), the rotation part of the matrix, colours and every drawing are **not** kept: the reader works from text positions alone, and never sees table ruling. 5. The page's runs go through `assembleRows`. If **every** page ends with no row, the read fails with - `no page has a text layer (a scanned PDF? OCR is not supported)` (`:40`). A PDF where only some pages are - scans reads, and those pages have no rows. + `no page has a text layer: it is a scan. Read it with OCR…`. A PDF where only some pages are scans reads, and + those pages have no rows, unless the request has `ocr` (see below): then those pages are rendered, read by the OCR engine and their words + become runs like a text layer's. The result, `{ kind: 'pdf', pages: [{ number, width, height, rows: [{ top, bottom, cells, text }] }] }`, is what `jsonpath` queries. A cell is `{ x, y, width, height, text }`; a row's `top` is its highest `y + height`, @@ -676,7 +677,7 @@ as on workbooks ([`extract-from-document.use-case.ts:128`](../../packages/core/s | Case | What happens | Why | |---|---|---| -| A scanned page | No rows on it. If every page is a scan, the read fails. | pdf.js reads the text layer; there is no OCR. | +| A scanned page | No rows on it. If every page is a scan, the read fails. With `ocr` the page is read from its picture. | pdf.js reads the text layer; OCR is opt-in. | | Password-protected PDF | The read fails at load. | No password is passed to `getDocument`. | | Rotated or vertical text | Rows and columns come out wrong. | Only the origin and `width`/`height` are kept; every stage assumes horizontal, left-to-right baselines. A page with `/Rotate` reports its rotated size, but the item coordinates are not transformed. | | Two columns closer than `JOIN_GAP` | They merge into one cell (3.5 pt at 10 pt). | Nothing but the gap separates two runs on a line. | diff --git a/docs/recipes/authoring.md b/docs/recipes/authoring.md index fb8b9df6..954bb223 100644 --- a/docs/recipes/authoring.md +++ b/docs/recipes/authoring.md @@ -265,7 +265,7 @@ call their JSON endpoints: a report whose table the page reads from an API is re | Step | Fields | Notes | |---|---|---| -| `request` | `url` (template), `method?`, `query?`, `headers?`, `body?` or `form?`, `as?` (`json`, `jsonl`, `html`, `text`, `pdf`, `csv`, `xlsx`, `pptx`, `yaml`, `markdown`, `xml`, `docx`, `youtube`: §4.14), `encoding?`, `delimiter?`, `scalars?`, `youtube?` | The response becomes the current document and, if `id` is set, the id holds the parsed JSON, the read PDF, workbook or deck, the markup or the text. Relative URLs resolve against the current page; one with no page to resolve against fails with a message saying so. Without `as`, the content type decides (`application/pdf` is a PDF, `text/csv` a CSV, a spreadsheet type a workbook, a presentation type a deck, `application/yaml` YAML, `application/x-ndjson` JSON Lines, `text/markdown` Markdown, an XML type XML: §4.12, a Word type a document read as HTML: §4.13); a type that says nothing (below) is sniffed. A `file:` URL reads a local file, its kind from `as`, else the extension (`.pdf`, `.csv`, `.tsv`, `.xlsx`, `.xlsm`, `.pptx`, `.yaml`, `.yml`, `.json`, `.jsonl`, `.ndjson`, `.md`, `.html`, `.xml`, `.rss`, `.atom`, `.xml.gz`, `.docx`), else its bytes; a relative one (`file:data/listino.csv`) is read from the recipe file's folder (below). 4xx/5xx fail the step. | +| `request` | `url` (template), `method?`, `query?`, `headers?`, `body?` or `form?`, `as?` (`json`, `jsonl`, `html`, `text`, `pdf`, `csv`, `xlsx`, `pptx`, `yaml`, `markdown`, `xml`, `docx`, `youtube`: §4.14), `encoding?`, `delimiter?`, `scalars?`, `youtube?`, `ocr?` (a scanned PDF: see **A scanned PDF** in the PDF section) | The response becomes the current document and, if `id` is set, the id holds the parsed JSON, the read PDF, workbook or deck, the markup or the text. Relative URLs resolve against the current page; one with no page to resolve against fails with a message saying so. Without `as`, the content type decides (`application/pdf` is a PDF, `text/csv` a CSV, a spreadsheet type a workbook, a presentation type a deck, `application/yaml` YAML, `application/x-ndjson` JSON Lines, `text/markdown` Markdown, an XML type XML: §4.12, a Word type a document read as HTML: §4.13); a type that says nothing (below) is sniffed. A `file:` URL reads a local file, its kind from `as`, else the extension (`.pdf`, `.csv`, `.tsv`, `.xlsx`, `.xlsm`, `.pptx`, `.yaml`, `.yml`, `.json`, `.jsonl`, `.ndjson`, `.md`, `.html`, `.xml`, `.rss`, `.atom`, `.xml.gz`, `.docx`), else its bytes; a relative one (`file:data/listino.csv`) is read from the recipe file's folder (below). 4xx/5xx fail the step. | **A body served as anything** (`application/octet-stream`, `binary/octet-stream`, `text/plain`, `application/zip` and the like, or no content type: a file behind a bucket or a CDN, a download link, a raw file) is read by what it @@ -630,7 +630,7 @@ repeating element (`table.credit_group tr`, not `tr`) and check the first record `request` with `as: "pdf"` (or a response served as `application/pdf`, or a local `file:…pdf`) reads the PDF's text layer with pdf.js into pages of **rows**: text that sits side by side becomes a cell, cells whose vertical extents overlap become a row. A scanned page has no text layer and gives no rows; a PDF where no page -has one fails the step (no OCR). Five extract kinds read it: +has one fails the step, unless the request says `"ocr": true` (below). Five extract kinds read it: | `kind` | Reads | Use for | |---|---|---| @@ -640,6 +640,21 @@ has one fails the step (no OCR). Five extract kinds read it: | `regex` | the text: one line per row, cells separated by a tab, pages by a blank line | a value next to a label: `Valid until\t(\S+)` | | `jsonpath` | the structure: `{ kind: "pdf", pages: [{ number, width, height, rows: [{ text, top, bottom, cells: [{ x, y, width, height, text }] }] }] }` | positions, a given page | +**A scanned PDF** has pictures where the text should be. Say `"ocr": true` on the `request` and OpenCraw renders each page +that has no text, reads it with Tesseract, and hands the words on as the same positioned cells a text layer gives, so every kind +above (`table`, `region`, `section`) works on a scan as it does on a digital PDF: + +```json +{ "type": "request", "id": "doc", "url": "{{start.url}}", "as": "pdf", "ocr": { "languages": ["por", "eng"] } } +``` + +`ocr` is `true` (English, only the pages that have no text) or an object: `languages` (Tesseract codes; `eng` comes with the engine, any +other is `npm install @tesseract.js-data/`), `always` (read pages that have text too, for a PDF with a wrong text layer from bad OCR; +the text layer is then dropped) and `dpi` (default: the scan's own resolution, between 150 and 300; rendering a soft scan larger only blurs +it). The engine is the `@opencraw/ocr-tesseract` package, installed next to `@opencraw/core` (the request says how if it is not). It +straightens a crooked page, flattens a shaded one and tries a second, stronger cleaning when the first reads poorly. OCR reads +what is printed; it can still confuse a character (`8,0%` read as `8,09`), so check a number that matters against a second source. + **Prefer structure to position.** A `region` is a place on the page: it breaks when a description is longer, a table has one more row or a column is added. Two sheets of the same layout and different content are read with `section` (the text between two headings) and `regex` (the value after a label), which find their text wherever it diff --git a/docs/requirements.md b/docs/requirements.md index 926447fb..881f97f0 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -175,7 +175,7 @@ steps; section 5). | `evaluate` | web | value | `script`, JavaScript run in the page; `args?`. Trusted recipes only. | | `screenshot` | web | – | `path` | | `captcha` | web | – | `solver?`, `selector?`, `verify?`, `attempts?`, `timeoutMs?`; for form captchas `image`, `refresh?`, `field`, `submit`; defaults from `session.captcha` | -| `request` | both | document | in web mode through the page's session, with `form?` (`{ selector, omit?, set? }`) instead of `body`; `method?`, `url` (`http(s):` or a local `file:`; relative to the current page, else a clear error), `query?`, `headers?`, `body?` (templated at every depth), `as: 'json' \| 'jsonl' \| 'html' \| 'text' \| 'pdf' \| 'csv' \| 'xlsx' \| 'pptx' \| 'yaml' \| 'markdown' \| 'xml' \| 'docx'`, `encoding?`, `delimiter?` (CSV), `scalars?` (YAML) | +| `request` | both | document | in web mode through the page's session, with `form?` (`{ selector, omit?, set? }`) instead of `body`; `method?`, `url` (`http(s):` or a local `file:`; relative to the current page, else a clear error), `query?`, `headers?`, `body?` (templated at every depth), `as: 'json' \| 'jsonl' \| 'html' \| 'text' \| 'pdf' \| 'csv' \| 'xlsx' \| 'pptx' \| 'yaml' \| 'markdown' \| 'xml' \| 'docx'`, `encoding?`, `delimiter?` (CSV), `scalars?` (YAML), `ocr?` (a scanned PDF: `true`, or `{ languages?, always?, dpi? }`) | | `extract` | both | value or list | `selector` (a template), `kind: 'css' \| 'xpath' \| 'jsonpath' \| 'regex' \| 'table' \| 'region'`, `take`, `many?`, `from?`; `table` also `columns?`, `until?`, `align?` (PDF), `fillDown?`, `sheet?`, `headerRows?`, `includeHidden?` (workbook), `slide?`, `shapes?` (deck) | | `set` | both | value | `value` (template or literal) | | `collect` | both | – | `into` (a list id bound in an enclosing scope), `value` (template or literal); appends, so values outlive the `forEach` iteration or `paginate` page that found them | @@ -197,7 +197,7 @@ the folder of the recipe file that names it when the recipe is read from a file `vars`, `matrix` values, every step `url`; a template's placeholders are kept), else against the process working directory. `allowedHosts` refuses `file:` URLs, relative or not. -A PDF (`as: 'pdf'`) is read into pages of rows of positioned cells (pdf.js, text layer only; a scan fails). +A PDF (`as: 'pdf'`) is read into pages of rows of positioned cells (pdf.js on the text layer; a scan fails unless the request has `ocr`, which reads its pages by OCR with `@opencraw/ocr-tesseract` and gives the words as the same positioned cells, so `table` and `region` work on a scan too). `table` finds tables by their header row and returns `{ page, title, header, rows }`, rows keyed by column: columns come from where the body's cells start, and lines of a wrapped cell are regrouped into their row. `regex` reads a PDF as text (one line per row, cells tab-separated), `jsonpath` reads its structure. `region` diff --git a/package-lock.json b/package-lock.json index 79a849bd..65dd4d10 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9470,6 +9470,10 @@ "resolved": "packages/office-reader", "link": true }, + "node_modules/@opencraw/ocr-tesseract": { + "resolved": "packages/ocr-tesseract", + "link": true + }, "node_modules/@opencraw/probe": { "resolved": "packages/probe", "link": true @@ -38583,6 +38587,7 @@ "version": "0.0.1", "license": "MIT", "dependencies": { + "@napi-rs/canvas": "^1.0.0", "@opencraw/office-reader": "^0.0.1", "@xmldom/xmldom": "^0.9.12", "cheerio": "^1.2.0", @@ -38618,6 +38623,22 @@ "node": ">=22" } }, + "packages/ocr-tesseract": { + "name": "@opencraw/ocr-tesseract", + "version": "0.0.1", + "license": "MIT", + "dependencies": { + "@tesseract.js-data/eng": "^1.0.0", + "pngjs": "^7.0.0", + "tesseract.js": "^7.0.0" + }, + "peerDependencies": { + "@opencraw/core": "^0.0.1" + }, + "engines": { + "node": ">=22" + } + }, "packages/office-reader": { "name": "@opencraw/office-reader", "version": "0.0.1", diff --git a/package.json b/package.json index 13364e89..be78e4eb 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,8 @@ "captcha-tesseract:qa": "nx run captcha-tesseract:lint && nx run captcha-tesseract:test", "captcha-tesseract:build": "nx run captcha-tesseract:build", "captcha-tesseract:e2e": "nx run captcha-tesseract:e2e", + "ocr-tesseract:qa": "nx run ocr-tesseract:lint && nx run ocr-tesseract:test", + "ocr-tesseract:build": "nx run ocr-tesseract:build", "azure-durable:qa": "nx run azure-durable:lint && nx run azure-durable:test", "azure-durable:build": "nx run azure-durable:build", "azure-durable:e2e": "nx run azure-durable:e2e", diff --git a/packages/core/package.json b/packages/core/package.json index 4121cc6b..726c73aa 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -20,6 +20,7 @@ "!**/*.js.map" ], "dependencies": { + "@napi-rs/canvas": "^1.0.0", "@opencraw/office-reader": "^0.0.1", "@xmldom/xmldom": "^0.9.12", "cheerio": "^1.2.0", diff --git a/packages/core/schemas/input-recipe.schema.json b/packages/core/schemas/input-recipe.schema.json index f268a346..4724e918 100644 --- a/packages/core/schemas/input-recipe.schema.json +++ b/packages/core/schemas/input-recipe.schema.json @@ -1617,6 +1617,35 @@ "text" ] }, + "ocr": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "object", + "properties": { + "languages": { + "minItems": 1, + "type": "array", + "items": { + "type": "string", + "pattern": "^[a-z]{3}(?:_[a-z]+)?$" + } + }, + "always": { + "type": "boolean" + }, + "dpi": { + "type": "integer", + "minimum": 72, + "maximum": 600 + } + }, + "additionalProperties": false + } + ] + }, "youtube": { "type": "object", "properties": { diff --git a/packages/core/src/api-steps/send-request.use-case.ts b/packages/core/src/api-steps/send-request.use-case.ts index ccd9bd0d..2dd3d402 100644 --- a/packages/core/src/api-steps/send-request.use-case.ts +++ b/packages/core/src/api-steps/send-request.use-case.ts @@ -43,6 +43,7 @@ export async function sendRequest (step: RequestStep, scope: ExtractionScope, cl encoding: step.encoding, delimiter: step.delimiter, scalars: step.scalars, + ocr: step.ocr === false ? undefined : step.ocr, // The video's address is templated like the rest; the languages may be too (`{{start.lang}}`). youtube: step.youtube === undefined ? undefined : { ...step.youtube, ...(step.youtube.language !== undefined && { language: renderText(step.youtube.language, lookup) }), ...(step.youtube.translateTo !== undefined && { translateTo: renderText(step.youtube.translateTo, lookup) }) }, timeoutMs: recipe.limits?.timeoutMs, diff --git a/packages/core/src/http-session/http-response.contract.ts b/packages/core/src/http-session/http-response.contract.ts index 6f89eb49..22dd9ff0 100644 --- a/packages/core/src/http-session/http-response.contract.ts +++ b/packages/core/src/http-session/http-response.contract.ts @@ -1,5 +1,6 @@ import type { DeckDocument } from '../deck-document' import type { PdfDocument } from '../pdf-document' +import type { OcrSetting } from '../pdf-ocr' import type { WorkbookDocument } from '../workbook-document' import type { XmlDocument } from '../xml-document' import type { BodyKind, HttpMethod } from '../recipe-schema' @@ -20,6 +21,8 @@ export interface HttpRequest { delimiter?: string /** A YAML body's scalars: `typed` (default) or `text`. */ scalars?: 'typed' | 'text' + /** Read a PDF that is a scan by OCR (needs `@opencraw/ocr-tesseract`): `true`, or the languages, `always` and `dpi`. */ + ocr?: true | OcrSetting /** With `as: 'youtube'`: the transcript's language, a translation, or no transcript. */ youtube?: YoutubeOptions timeoutMs?: number diff --git a/packages/core/src/http-session/http.client.ts b/packages/core/src/http-session/http.client.ts index 68a0feb3..73c53d36 100644 --- a/packages/core/src/http-session/http.client.ts +++ b/packages/core/src/http-session/http.client.ts @@ -9,6 +9,8 @@ import { readDocxHtml } from '../docx-document' import { readMarkdown } from '../markdown-document' import { openDocumentFormat } from '../opendocument-sniff' import { readPdf } from '../pdf-document' +import { withOcr } from '../pdf-ocr' +import type { OcrSetting } from '../pdf-ocr' import type { BodyKind } from '../recipe-schema' import { csvWorkbook, readXlsxWorkbook, sheetNameOf } from '../workbook-document' import { parseJsonLike, parseJsonLines } from '../selection' @@ -182,7 +184,7 @@ export class HttpClient implements HttpSender { * @param reading - `as`, `encoding`, `delimiter`, `scalars`. * @returns The document, what reading noticed, and the format. */ -export async function readFileBody (bytes: Uint8Array, name: string, reading: Pick = {}): Promise { +export async function readFileBody (bytes: Uint8Array, name: string, reading: Pick = {}): Promise { return parseBody(reading.as ?? formatOfFile(name, bytes), bytes, name, reading) } @@ -214,7 +216,7 @@ async function readLocalFile (httpRequest: HttpRequest): Promise { return { status: 200, url: httpRequest.url, headers: {}, body, format, ...(warnings.length > 0 && { warnings }) } } -async function parseBody (format: BodyKind, bytes: Uint8Array, url: string, reading: { encoding?: string, delimiter?: string, scalars?: 'typed' | 'text', charset?: string }): Promise { +async function parseBody (format: BodyKind, bytes: Uint8Array, url: string, reading: { encoding?: string, delimiter?: string, scalars?: 'typed' | 'text', charset?: string, ocr?: true | OcrSetting }): Promise { if (format === 'youtube') throw new Error(`${url}: "youtube" reads a video address, not a file or a response: use it as the "as" of a request step to a YouTube address`) if (format === 'yaml') { const { text } = decodeText(bytes, reading) @@ -244,8 +246,8 @@ async function parseBody (format: BodyKind, bytes: Uint8Array, url: string, read return { body: await parseFormat(format, bytes, url, reading), warnings: [], format } } -async function parseFormat (format: BodyKind, bytes: Uint8Array, url: string, reading: { encoding?: string, delimiter?: string, charset?: string }): Promise { - if (format === 'pdf') return readPdf(bytes, url) +async function parseFormat (format: BodyKind, bytes: Uint8Array, url: string, reading: { encoding?: string, delimiter?: string, charset?: string, ocr?: true | OcrSetting }): Promise { + if (format === 'pdf') return reading.ocr === undefined ? readPdf(bytes, url) : withOcr(reading.ocr, async ocr => readPdf(bytes, url, { ocr })) if (format === 'xlsx') return readXlsxWorkbook(bytes, url) if (format === 'pptx') return readPptxDeck(bytes, url) const { text, encoding } = decodeText(bytes, reading) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 228ee589..5696bb7b 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -33,6 +33,9 @@ export { tryParseJson, selectHtml, countMatches, matchesOf, NODE_ID_ATTRIBUTE } export type { HtmlMatch } from './selection' export { readPdf, PdfReadError, analyzeTables, findTables, pdfText, isPdfDocument, rowsOfCells, findSections, marginRows, parseRegion, regionSelector, regionText, regionCells, isInsideRegion } from './pdf-document' export type { DocumentRegion, PdfRegionMatch, PositionedBox, PdfSectionMatch, PdfSectionQuery } from './pdf-document' +export type { PdfPageHandle, PdfPageOcr, ReadPdfOptions } from './pdf-document' +export { pdfPageOcr, ocrRuns, withOcr } from './pdf-ocr' +export type { OcrEngine, OcrLine, OcrOptions, OcrPageImage, OcrWord, PdfOcr } from './pdf-ocr' export type { PdfDocument, PdfPage, PdfRow, PdfCell, PositionedText, PdfTable, TableQuery, TableAlign, TableDiagnostics, TableBandDiagnostics } from './pdf-document' export { parseCsv, detectDelimiter, csvWorkbook, readXlsxWorkbook, findGridTables, fillDown, workbookText, isWorkbookDocument, htmlTableSheets } from './workbook-document' export { readMarkdown } from './markdown-document' diff --git a/packages/core/src/pdf-document/index.ts b/packages/core/src/pdf-document/index.ts index 02e4abbf..08bf3f15 100644 --- a/packages/core/src/pdf-document/index.ts +++ b/packages/core/src/pdf-document/index.ts @@ -1,4 +1,5 @@ export { readPdf, PdfReadError } from './read-pdf.client' +export type { PdfPageHandle, PdfPageOcr, ReadPdfOptions } from './read-pdf.client' export { pdfText, isPdfDocument } from './pdf-document.model' export type { PdfDocument, PdfPage, PdfRow, PdfCell, PositionedText } from './pdf-document.model' export { assembleRows, rowsOfCells } from './row-assembly.algorithm' diff --git a/packages/core/src/pdf-document/read-pdf.client.ts b/packages/core/src/pdf-document/read-pdf.client.ts index e374e625..35313f77 100644 --- a/packages/core/src/pdf-document/read-pdf.client.ts +++ b/packages/core/src/pdf-document/read-pdf.client.ts @@ -1,3 +1,4 @@ +import type { PDFDocumentProxy } from 'pdfjs-dist/legacy/build/pdf.mjs' import type { PdfDocument, PdfPage, PositionedText } from './pdf-document.model' import { assembleRows } from './row-assembly.algorithm' @@ -6,18 +7,38 @@ export class PdfReadError extends Error { override readonly name = 'PdfReadError' } +/** A page as pdf.js gives it. */ +export type PdfPageHandle = Awaited> + +/** How to read a page that has no text layer: render it and read its picture. */ +export interface PdfPageOcr { + /** The text runs of the page, read from its picture; `height` is the page's height in points. */ + readPage: (page: PdfPageHandle, height: number) => Promise + /** Read every page this way and drop the text layer, for a PDF whose text layer is wrong. Default: only the pages with no text. */ + always?: boolean +} + +/** How to read a PDF beyond its text layer. */ +export interface ReadPdfOptions { + /** Read the pages that have no text layer (a scan) by OCR, or every page with `always`. Without it a PDF with no text at all is refused. */ + ocr?: PdfPageOcr +} + /** * Reads a PDF's text layer into rows of positioned cells, with pdf.js. pdf.js * is imported on first use, so recipes that never read a PDF never load it. - * Only text is read: no page is rendered, no script runs, no font is loaded. + * Only text is read: no page is rendered, no script runs, no font is loaded, unless a page is read by + * OCR (`options.ocr`): then that page, a scan with no text layer, is rendered and its picture read, and its + * words come back as the runs a text layer would have given. * * @param bytes - The file. * @param source - Where it came from, for messages. + * @param options - OCR, for a scanned PDF. * @returns The document. - * @throws PdfReadError when the bytes are not a readable PDF, or no page has a - * text layer (a scan: OCR is not supported). + * @throws PdfReadError when the bytes are not a readable PDF, or no page has any text: no text layer + * and no OCR asked for (a scan), or an OCR that found nothing. */ -export async function readPdf (bytes: Uint8Array, source = 'PDF'): Promise { +export async function readPdf (bytes: Uint8Array, source = 'PDF', options: ReadPdfOptions = {}): Promise { const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs') // pdf.js takes ownership of the buffer it is given, and refuses a Node // Buffer: hand it a plain copy. @@ -34,10 +55,16 @@ export async function readPdf (bytes: Uint8Array, source = 'PDF'): Promise ('str' in item ? [runOf(item)] : [])) + let runs = content.items.flatMap(item => ('str' in item ? [runOf(item)] : [])) + const { ocr } = options + if (ocr !== undefined && (ocr.always === true || runs.every(run => run.text.trim() === ''))) runs = await ocr.readPage(page, height) pages.push({ number, width, height, rows: assembleRows(runs) }) } - if (pages.every(page => page.rows.length === 0)) throw new PdfReadError(`${source}: no page has a text layer (a scanned PDF? OCR is not supported)`) + if (pages.every(page => page.rows.length === 0)) { + throw new PdfReadError(options.ocr === undefined + ? `${source}: no page has a text layer: it is a scan. Read it with OCR ("ocr": true on the request step; it needs @opencraw/ocr-tesseract)` + : `${source}: no page has a text layer, and OCR found no text on any page`) + } return { kind: 'pdf', pages } } finally { diff --git a/packages/core/src/pdf-ocr/fixtures/scan-clean.pdf b/packages/core/src/pdf-ocr/fixtures/scan-clean.pdf new file mode 100644 index 00000000..d089371b Binary files /dev/null and b/packages/core/src/pdf-ocr/fixtures/scan-clean.pdf differ diff --git a/packages/core/src/pdf-ocr/fixtures/scan-crooked-shaded.pdf b/packages/core/src/pdf-ocr/fixtures/scan-crooked-shaded.pdf new file mode 100644 index 00000000..2e743d81 Binary files /dev/null and b/packages/core/src/pdf-ocr/fixtures/scan-crooked-shaded.pdf differ diff --git a/packages/core/src/pdf-ocr/fixtures/scan-poor.pdf b/packages/core/src/pdf-ocr/fixtures/scan-poor.pdf new file mode 100644 index 00000000..b2453e29 Binary files /dev/null and b/packages/core/src/pdf-ocr/fixtures/scan-poor.pdf differ diff --git a/packages/core/src/pdf-ocr/index.ts b/packages/core/src/pdf-ocr/index.ts new file mode 100644 index 00000000..82abdae9 --- /dev/null +++ b/packages/core/src/pdf-ocr/index.ts @@ -0,0 +1,6 @@ +export { ocrPage, pdfPageOcr } from './ocr-page.use-case' +export { ocrRuns } from './ocr-runs.mapper' +export { withOcr } from './with-ocr.use-case' +export { renderPagePng, scaleOfDpi, DEFAULT_DPI } from './render-page.client' +export { ocrDpi } from './native-dpi.client' +export type { OcrEngine, OcrLine, OcrOptions, OcrPageImage, OcrSetting, OcrWord, PdfOcr } from './ocr-engine.contract' diff --git a/packages/core/src/pdf-ocr/native-dpi.client.test.ts b/packages/core/src/pdf-ocr/native-dpi.client.test.ts new file mode 100644 index 00000000..265766f0 --- /dev/null +++ b/packages/core/src/pdf-ocr/native-dpi.client.test.ts @@ -0,0 +1,36 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { ocrDpi } from './native-dpi.client' +import type { PdfPageHandle } from '../pdf-document' + +async function firstPage (path: string[]): Promise<{ page: PdfPageHandle, destroy: () => Promise }> { + const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs') + const bytes = readFileSync(join(__dirname, ...path)) + const task = pdfjs.getDocument({ data: new Uint8Array(bytes), verbosity: 0 }) + const document = await task.promise + + return { page: await document.getPage(1), destroy: async () => { await task.destroy() } } +} + +describe('ocrDpi', () => { + it('is the scan\'s own resolution: the pixels of its picture across the width of the page', async () => { + const clean = await firstPage(['fixtures', 'scan-clean.pdf']) + const poor = await firstPage(['fixtures', 'scan-poor.pdf']) + + expect(await ocrDpi(clean.page)).toBe(180) + // 840 pixels across 420 points is 144 dpi: held at the 150 floor. + expect(await ocrDpi(poor.page)).toBe(150) + await clean.destroy() + await poor.destroy() + }) + + it('is 300 for a page with no picture, and what was asked for when something was', async () => { + const text = await firstPage(['..', 'pdf-document', 'fixtures', 'discounts.pdf']) + const clean = await firstPage(['fixtures', 'scan-clean.pdf']) + + expect(await ocrDpi(text.page)).toBe(300) + expect(await ocrDpi(clean.page, 220)).toBe(220) + await text.destroy() + await clean.destroy() + }) +}) diff --git a/packages/core/src/pdf-ocr/native-dpi.client.ts b/packages/core/src/pdf-ocr/native-dpi.client.ts new file mode 100644 index 00000000..9b1461ad --- /dev/null +++ b/packages/core/src/pdf-ocr/native-dpi.client.ts @@ -0,0 +1,33 @@ +import type { PdfPageHandle } from '../pdf-document' +import { DEFAULT_DPI } from './render-page.client' + +/** Below this the scan is too coarse for any reading to gain from being rendered smaller: pages are read at least this sharply. */ +const MIN_DPI = 150 + +/** The names of the operators that paint a picture. */ +const PICTURE_OPERATORS = new Set(['paintImageXObject', 'paintInlineImageXObject', 'paintImageMaskXObject', 'paintJpegXObject']) + +/** + * The resolution to render a page for OCR at. A scan is read best at its own resolution: rendered larger, a soft picture only gets blurrier + * (Tesseract reads a 144 dpi scan well at 150 and not at all at 300), rendered smaller, it loses detail. So the page's largest picture says + * (its pixels across the page's width), held between 150 and 300 dpi; a page with no picture (text drawn as outlines) is read at 300. + * + * @param page - The pdf.js page. + * @param requested - A resolution the recipe asked for, used as it is. + * @returns The dots per inch to render at. + */ +export async function ocrDpi (page: PdfPageHandle, requested?: number): Promise { + if (requested !== undefined) return requested + const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs') + const operators = await page.getOperatorList() + const names = new Map(Object.entries(pdfjs.OPS).map(([name, code]) => [code, name])) + const { width, height } = page.getViewport({ scale: 1 }) + let widest = 0 + for (const [index, code] of operators.fnArray.entries()) { + if (!PICTURE_OPERATORS.has(names.get(code) ?? '')) continue + const [, across, down] = (operators.argsArray[index] ?? []) as [unknown, unknown, unknown] + if (typeof across === 'number' && typeof down === 'number' && across > 0 && down > 0) widest = Math.max(widest, Math.max((across / width), (down / height)) * 72) + } + + return widest === 0 ? DEFAULT_DPI : Math.round(Math.min(DEFAULT_DPI, Math.max(MIN_DPI, widest))) +} diff --git a/packages/core/src/pdf-ocr/ocr-engine.contract.ts b/packages/core/src/pdf-ocr/ocr-engine.contract.ts new file mode 100644 index 00000000..eedb26c3 --- /dev/null +++ b/packages/core/src/pdf-ocr/ocr-engine.contract.ts @@ -0,0 +1,59 @@ +/** A PDF page as an OCR engine reads it: the page rendered to a PNG, `scale` pixels to each point of the page. */ +export interface OcrPageImage { + png: Uint8Array + width: number + height: number + scale: number +} + +/** One word an engine found, its box in the image's pixels, y growing downwards from the top edge. */ +export interface OcrWord { + text: string + x0: number + y0: number + x1: number + y1: number + /** How sure the engine is, 0 to 100. */ + confidence: number +} + +/** One line of text an engine found: its words left to right, and the line the baseline runs along, when the engine measures it. */ +export interface OcrLine { + words: OcrWord[] + /** The baseline under the line's letters, from its left end to its right end, in the image's pixels. It follows a scan that was fed in crooked. */ + baseline?: { x0: number, y0: number, x1: number, y1: number } +} + +/** What to read a page for. */ +export interface OcrOptions { + /** The languages the page is written in, as Tesseract names them (`eng`, `ita`, `por`, `chi_sim`); the engine's own default when empty. */ + languages: readonly string[] +} + +/** + * Reads text from the picture of a page. `@opencraw/ocr-tesseract` is the engine OpenCraw ships; anything that answers this can stand in for it + * (a cloud service, another local engine), handed to the crawler as `ocr`. + */ +export interface OcrEngine { + readonly name: string + recognize: (image: OcrPageImage, options: OcrOptions) => Promise + /** Stops what the engine started (worker threads, a loaded model); called when the client that loaded it is disposed. */ + dispose?: () => Promise +} + +/** What a recipe's `ocr` says: the languages (Tesseract codes, default English), whether pages that have text are read again, the resolution to render at. */ +export interface OcrSetting { + languages?: readonly string[] + always?: boolean + dpi?: number +} + +/** The OCR a PDF is read with: which engine (loaded when a page first needs it), in which languages, and whether pages that have text are read again. */ +export interface PdfOcr { + engine: () => Promise + languages?: readonly string[] + /** Read every page by OCR and drop the text layer, for a PDF whose text layer is wrong (a scan somebody ran bad OCR over). Default: only pages with no text. */ + always?: boolean + /** The resolution a page is rendered at for the engine, in dots per inch. Default: the scan's own resolution, between 150 and 300; more only blurs a soft scan. */ + dpi?: number +} diff --git a/packages/core/src/pdf-ocr/ocr-page.use-case.test.ts b/packages/core/src/pdf-ocr/ocr-page.use-case.test.ts new file mode 100644 index 00000000..e9952d94 --- /dev/null +++ b/packages/core/src/pdf-ocr/ocr-page.use-case.test.ts @@ -0,0 +1,88 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { readPdf } from '../pdf-document' +import { pdfPageOcr } from './ocr-page.use-case' +import type { OcrEngine, OcrLine, OcrOptions, OcrPageImage } from './ocr-engine.contract' + +const PNG_SIGNATURE = [0x89, 0x50, 0x4E, 0x47] + +/** An engine that reads what it is told to, and remembers the pictures and options it was given. */ +function fakeEngine (lines: OcrLine[]): OcrEngine & { seen: { image: OcrPageImage, options: OcrOptions }[] } { + const seen: { image: OcrPageImage, options: OcrOptions }[] = [] + + return { + name: 'fake', + seen, + recognize: (image, options) => { + seen.push({ image, options }) + + return Promise.resolve(lines) + }, + } +} + +const word = (text: string, x0: number, y0: number, x1: number, y1: number): OcrLine['words'][number] => ({ text, x0, y0, x1, y1, confidence: 95 }) + +describe('reading a scanned PDF by OCR', () => { + const scan = new Uint8Array(readFileSync(join(__dirname, 'fixtures', 'scan-clean.pdf'))) + + it('is refused without OCR, saying it is a scan and what to do', async () => { + await expect(readPdf(scan, 'scan.pdf')).rejects.toThrow(/scan\.pdf: no page has a text layer: it is a scan\. Read it with OCR/) + }) + + it('renders the page, gives the engine its picture, and reads the words as rows and cells in points', async () => { + const engine = fakeEngine([ + { words: [word('Model', 150, 200, 280, 240), word('Price', 700, 200, 820, 240)] }, + { words: [word('Pandina', 150, 300, 330, 340), word('15950', 700, 300, 820, 340)] }, + ]) + + const document = await readPdf(scan, 'scan.pdf', { ocr: pdfPageOcr({ engine: () => Promise.resolve(engine), languages: ['eng', 'ita'] }) }) + + const [page] = document.pages + expect([page.width, page.height]).toEqual([420, 595]) + expect(page.rows.map(row => row.cells.map(cell => cell.text))).toEqual([['Model', 'Price'], ['Pandina', '15950']]) + const [first] = page.rows + expect(first.cells[0].x).toBeCloseTo(60, 5) + expect(first.cells[1].x).toBeCloseTo(280, 5) + expect(first.cells[0].y).toBeCloseTo(595 - 96, 5) + expect(engine.seen).toHaveLength(1) + expect([...engine.seen[0].image.png.subarray(0, 4)]).toEqual(PNG_SIGNATURE) + // The scan is 1050 pixels across 420 points: 180 dpi, and that is what it is read at. + expect(engine.seen[0].image.scale).toBeCloseTo(2.5, 5) + expect([engine.seen[0].image.width, engine.seen[0].image.height]).toEqual([1050, 1488]) + expect(engine.seen[0].options.languages).toEqual(['eng', 'ita']) + }) + + it('loads the engine only when a page needs it', async () => { + let loaded = 0 + const ocr = pdfPageOcr({ + engine: () => { + loaded += 1 + + return Promise.resolve(fakeEngine([{ words: [word('x', 0, 0, 10, 20)] }])) + }, + }) + const withText = readFileSync(join(__dirname, '..', 'pdf-document', 'fixtures', 'discounts.pdf')) + + await readPdf(new Uint8Array(withText), 'discounts.pdf', { ocr }) + expect(loaded).toBe(0) + await readPdf(scan, 'scan.pdf', { ocr }) + expect(loaded).toBe(1) + }) + + it('leaves the pages that have text as they are, unless every page is to be read by OCR', async () => { + const withText = new Uint8Array(readFileSync(join(__dirname, '..', 'pdf-document', 'fixtures', 'discounts.pdf'))) + const plain = await readPdf(withText, 'discounts.pdf') + const engine = fakeEngine([{ words: [word('replaced', 100, 100, 300, 140)] }]) + + const untouched = await readPdf(withText, 'discounts.pdf', { ocr: pdfPageOcr({ engine: () => Promise.resolve(engine) }) }) + const forced = await readPdf(withText, 'discounts.pdf', { ocr: pdfPageOcr({ engine: () => Promise.resolve(engine), always: true }) }) + + expect(untouched).toEqual(plain) + expect(forced.pages[0].rows.map(row => row.text)).toEqual(['replaced']) + }) + + it('says so when OCR finds nothing on any page', async () => { + await expect(readPdf(scan, 'scan.pdf', { ocr: pdfPageOcr({ engine: () => Promise.resolve(fakeEngine([])) }) })).rejects.toThrow('scan.pdf: no page has a text layer, and OCR found no text on any page') + }) +}) diff --git a/packages/core/src/pdf-ocr/ocr-page.use-case.ts b/packages/core/src/pdf-ocr/ocr-page.use-case.ts new file mode 100644 index 00000000..02694ebd --- /dev/null +++ b/packages/core/src/pdf-ocr/ocr-page.use-case.ts @@ -0,0 +1,34 @@ +import type { PdfPageHandle, PdfPageOcr, PositionedText } from '../pdf-document' +import type { OcrEngine, PdfOcr } from './ocr-engine.contract' +import { ocrDpi } from './native-dpi.client' +import { ocrRuns } from './ocr-runs.mapper' +import { renderPagePng } from './render-page.client' + +/** + * Reads a PDF page that has no text layer by OCR: renders it, has the engine read the picture, and gives the words as the text runs a text layer + * would have. + * + * @param page - The pdf.js page. + * @param pageHeight - The page's height in points. + * @param engine - The OCR engine. + * @param languages - The languages to read for. + * @param dpi - The resolution to render the page at; default: the scan's own, between 150 and 300. + * @returns The page's text runs. + */ +export async function ocrPage (page: PdfPageHandle, pageHeight: number, engine: OcrEngine, languages: readonly string[], dpi?: number): Promise { + const image = await renderPagePng(page, await ocrDpi(page, dpi)) + const lines = await engine.recognize(image, { languages }) + + return ocrRuns(lines, image.scale, pageHeight) +} + +/** + * The way `readPdf` reads a page that has no text layer, from an OCR setting: the engine is loaded when a page first needs it, and every page read + * goes through it. + * + * @param ocr - The engine (as a loader), the languages, and whether pages with text are read again. + * @returns What `readPdf` takes as `ocr`. + */ +export function pdfPageOcr (ocr: PdfOcr): PdfPageOcr { + return { readPage: async (page, height) => await ocrPage(page, height, await ocr.engine(), ocr.languages ?? [], ocr.dpi), ...(ocr.always !== undefined && { always: ocr.always }) } +} diff --git a/packages/core/src/pdf-ocr/ocr-runs.mapper.test.ts b/packages/core/src/pdf-ocr/ocr-runs.mapper.test.ts new file mode 100644 index 00000000..3f0718f6 --- /dev/null +++ b/packages/core/src/pdf-ocr/ocr-runs.mapper.test.ts @@ -0,0 +1,51 @@ +import { ocrRuns } from './ocr-runs.mapper' +import type { OcrLine, OcrWord } from './ocr-engine.contract' + +const word = (text: string, x0: number, y0: number, x1: number, y1: number, confidence = 90): OcrWord => ({ text, x0, y0, x1, y1, confidence }) + +describe('ocrRuns', () => { + it('turns words in pixels into runs in points, y growing upwards from the bottom of the page', () => { + const lines: OcrLine[] = [{ words: [word('Price', 200, 100, 300, 140), word('list', 320, 100, 380, 140)] }] + + const runs = ocrRuns(lines, 2, 800) + + expect(runs.map(run => [run.x, run.y, run.width, run.text])).toEqual([[100, 730, 50, 'Price'], [160, 730, 30, 'list']]) + expect(runs[0].height).toBeCloseTo(26.67, 1) + }) + + it('puts every word of a line on the line\'s baseline, whatever hangs below it (a g, a y)', () => { + // "Typing" has two tails: its box ends 12 px below the baseline the other words end on. + const lines: OcrLine[] = [{ words: [word('Fast', 0, 100, 80, 140), word('Typing', 100, 100, 220, 152), word('works', 240, 100, 340, 140)] }] + + const ys = ocrRuns(lines, 1, 500).map(run => run.y) + + expect(new Set(ys).size).toBe(1) + expect(ys[0]).toBe(500 - 140) + }) + + it('follows the slanted baseline of a line that was scanned crooked', () => { + const baseline = { x0: 0, y0: 200, x1: 400, y1: 220 } + const lines: OcrLine[] = [{ baseline, words: [word('left', 0, 170, 50, 200), word('right', 350, 190, 400, 220)] }] + + const [left, right] = ocrRuns(lines, 1, 1000) + + // The baseline is at 201.25 under the middle of "left" and at 218.75 under the middle of "right". + expect(left.y).toBeCloseTo(1000 - 201.25, 5) + expect(right.y).toBeCloseTo(1000 - 218.75, 5) + expect(right.y).toBeLessThan(left.y) + }) + + it('leaves out words the engine is not sure of, and empty ones, and gives nothing for a line with none left', () => { + const lines: OcrLine[] = [ + { words: [word('real', 0, 0, 40, 20, 88), word('speck', 50, 0, 90, 20, 8), word(' '.repeat(3), 100, 0, 120, 20, 90)] }, + { words: [word('smudge', 0, 40, 60, 60, 3)] }, + ] + + expect(ocrRuns(lines, 1, 100).map(run => run.text)).toEqual(['real']) + }) + + it('trims the text of a word, and handles no lines', () => { + expect(ocrRuns([{ words: [word(' 15.950 ', 0, 0, 80, 20)] }], 1, 100)[0].text).toBe('15.950') + expect(ocrRuns([], 1, 100)).toEqual([]) + }) +}) diff --git a/packages/core/src/pdf-ocr/ocr-runs.mapper.ts b/packages/core/src/pdf-ocr/ocr-runs.mapper.ts new file mode 100644 index 00000000..05c3481e --- /dev/null +++ b/packages/core/src/pdf-ocr/ocr-runs.mapper.ts @@ -0,0 +1,55 @@ +import type { PositionedText } from '../pdf-document' +import type { OcrLine, OcrWord } from './ocr-engine.contract' + +/** Below this confidence (0 to 100) a word is the engine guessing at a speck or a stain, not reading: it is left out. */ +const MIN_CONFIDENCE = 20 +/** The share of an em a word's box rises above the baseline: capitals and ascenders reach about this far. */ +const ASCENDER_SHARE = 0.75 + +function median (values: readonly number[]): number { + const sorted = [...values].sort((a, b) => a - b) + const middle = Math.floor(sorted.length / 2) + + return sorted.length % 2 === 0 ? ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2 : (sorted[middle] ?? 0) +} + +/** Where the baseline is under a word, in pixels: on the line the engine measured, else where most of the line's words end (a word with a tail, g or y, ends lower). */ +function baselineAt (line: OcrLine, word: OcrWord, fallback: number): number { + const { baseline } = line + if (baseline === undefined) return fallback + const run = baseline.x1 - baseline.x0 + if (run <= 0) return baseline.y0 + const centre = (word.x0 + word.x1) / 2 + const at = Math.min(1, Math.max(0, (centre - baseline.x0) / run)) + + return baseline.y0 + ((baseline.y1 - baseline.y0) * at) +} + +/** + * The words of a page as the text runs a PDF's text layer gives (`PositionedText`: points, y growing upwards, the baseline), so rows, cells, + * tables and regions are found on a scan exactly as they are on a PDF with text. A word on a crooked scan sits on its line's slanted baseline; + * the font size is what the capitals and ascenders of the line imply. + * + * @param lines - What the engine read, in the page image's pixels. + * @param scale - Pixels to each point of the page. + * @param pageHeight - The page's height in points. + * @returns One run per word the engine is sure enough of. + */ +export function ocrRuns (lines: readonly OcrLine[], scale: number, pageHeight: number): PositionedText[] { + return lines.flatMap((line) => { + const words = line.words.filter(word => word.text.trim() !== '' && word.confidence >= MIN_CONFIDENCE) + if (words.length === 0) return [] + const fallback = median(words.map(word => word.y1)) + const rise = median(words.map(word => fallback - word.y0)) + const boxes = median(words.map(word => word.y1 - word.y0)) + const size = (rise > 0 ? rise / ASCENDER_SHARE : boxes) / scale + + return words.map(word => ({ + x: word.x0 / scale, + y: pageHeight - (baselineAt(line, word, fallback) / scale), + width: (word.x1 - word.x0) / scale, + height: size, + text: word.text.trim(), + })) + }) +} diff --git a/packages/core/src/pdf-ocr/render-page.client.ts b/packages/core/src/pdf-ocr/render-page.client.ts new file mode 100644 index 00000000..9a2b2db0 --- /dev/null +++ b/packages/core/src/pdf-ocr/render-page.client.ts @@ -0,0 +1,34 @@ +import type { PdfPageHandle } from '../pdf-document' +import type { OcrPageImage } from './ocr-engine.contract' + +/** A PDF point is 1/72 inch: the pixels to each point of a page rendered at this many dots per inch. */ +export function scaleOfDpi (dpi: number): number { + return dpi / 72 +} + +/** 300 dpi, what Tesseract is trained for: text a few points tall still has glyphs it can read, and most scans are at least this sharp. */ +export const DEFAULT_DPI = 300 + +/** The widest side a rendered page may have, in pixels: a poster or a very large MediaBox is scaled down to this. */ +const MAX_SIDE = 6000 + +/** + * Renders a PDF page to a PNG with pdf.js and `@napi-rs/canvas` (the canvas pdf.js itself uses in Node), the way a scanner would give it: only + * the page's pictures and drawing matter here, so a scan comes out as the scan. + * + * @param page - The pdf.js page. + * @param dpi - Dots per inch to render at; default {@link DEFAULT_DPI}, lowered for a page so big its longer side would pass 6000 pixels. + * @returns The page's image. + * @throws Error when `@napi-rs/canvas` cannot be loaded (its native part is missing for this system). + */ +export async function renderPagePng (page: PdfPageHandle, dpi = DEFAULT_DPI): Promise { + const natural = page.getViewport({ scale: 1 }) + const effective = Math.min(scaleOfDpi(dpi), MAX_SIDE / Math.max(natural.width, natural.height)) + const viewport = page.getViewport({ scale: effective }) + const { createCanvas } = await import('@napi-rs/canvas') + const canvas = createCanvas(Math.ceil(viewport.width), Math.ceil(viewport.height)) + // The canvas is @napi-rs/canvas's: the one pdf.js itself uses in Node, whose types are not the DOM's. + await page.render({ canvasContext: canvas.getContext('2d') as never, canvas: canvas as never, viewport }).promise + + return { png: canvas.toBuffer('image/png'), width: canvas.width, height: canvas.height, scale: effective } +} diff --git a/packages/core/src/pdf-ocr/with-ocr.use-case.ts b/packages/core/src/pdf-ocr/with-ocr.use-case.ts new file mode 100644 index 00000000..54154e1e --- /dev/null +++ b/packages/core/src/pdf-ocr/with-ocr.use-case.ts @@ -0,0 +1,44 @@ +import type { PdfPageOcr } from '../pdf-document' +import type { OcrEngine, OcrSetting } from './ocr-engine.contract' +import { pdfPageOcr } from './ocr-page.use-case' + +/** The package that reads pictures; it is not a dependency of core (it brings a WebAssembly engine and a language model), so it is loaded when a recipe asks for OCR. */ +const ENGINE_PACKAGE = '@opencraw/ocr-tesseract' + +/** The engine package, loaded by name so no bundler or compiler tries to resolve it ahead of time. */ +async function loadEngine (): Promise { + let loaded: { createOcrEngine?: () => OcrEngine } + try { + loaded = await import(/* webpackIgnore: true */ ENGINE_PACKAGE) as typeof loaded + } catch (error) { + throw new Error(`OCR needs the ${ENGINE_PACKAGE} package: install it (npm install ${ENGINE_PACKAGE}) next to @opencraw/core`, { cause: error }) + } + if (loaded.createOcrEngine === undefined) throw new Error(`${ENGINE_PACKAGE} does not export createOcrEngine: it is too old for this version of @opencraw/core`) + + return loaded.createOcrEngine() +} + +/** + * Runs a PDF read with OCR from a recipe's `ocr` setting. The engine starts when a page first needs it and its worker threads are stopped when the + * read ends, whether it succeeds or not. + * + * @param setting - `true` for the defaults (English, pages with no text), or the languages, `always` and `dpi`. + * @param run - The read, given the OCR to pass to `readPdf`. + * @returns What the read returns. + */ +export async function withOcr (setting: true | OcrSetting, run: (ocr: PdfPageOcr) => Promise): Promise { + const options = setting === true ? {} : setting + let engine: OcrEngine | undefined + try { + return await run(pdfPageOcr({ + ...options, + engine: async () => { + engine ??= await loadEngine() + + return engine + }, + })) + } finally { + await engine?.dispose?.() + } +} diff --git a/packages/core/src/recipe-schema/step.contract.ts b/packages/core/src/recipe-schema/step.contract.ts index c6a0d068..057f1507 100644 --- a/packages/core/src/recipe-schema/step.contract.ts +++ b/packages/core/src/recipe-schema/step.contract.ts @@ -145,6 +145,8 @@ export interface RequestStep extends StepBaseFields { delimiter?: string /** A YAML body's scalars: `typed` (default, YAML 1.2) or `text`, every scalar as written (`0123` stays `"0123"`). */ scalars?: 'typed' | 'text' + /** Read a PDF that is a scan by OCR (needs `@opencraw/ocr-tesseract`): `true` reads the pages that have no text, in English; or give the Tesseract `languages`, `always` (read pages that have text too, for a wrong text layer) and `dpi`. */ + ocr?: boolean | { languages?: string[], always?: boolean, dpi?: number } /** With `as: "youtube"`: the transcript's language, a translation of it, or `transcript: false` to read only what is about the video. */ youtube?: { language?: string, translateTo?: string, transcript?: boolean } } @@ -314,6 +316,9 @@ const waitStep = z.strictObject({ ...base, type: z.literal('wait'), selector: pl const evaluateStep = z.strictObject({ ...base, type: z.literal('evaluate'), script: z.string().min(1), args: z.record(z.string(), z.unknown()).optional() }) const screenshotStep = z.strictObject({ ...base, type: z.literal('screenshot'), path: z.string().min(1) }) const formFields = z.array(z.string().min(1)) +const ocrLanguage = z.string().regex(/^[a-z]{3}(?:_[a-z]+)?$/, 'a Tesseract language code: eng, por, ita, chi_sim') +const ocrOptions = z.strictObject({ languages: z.array(ocrLanguage).min(1).optional(), always: z.boolean().optional(), dpi: z.int().min(72).max(600).optional() }) +const ocrSetting = z.union([z.boolean(), ocrOptions]) const youtubeOptions = z.strictObject({ language: z.string().min(1).optional(), translateTo: z.string().min(1).optional(), transcript: z.boolean().optional() }) const requestForm = z.strictObject({ selector: plainSelector, omit: formFields.optional(), set: stringMap.optional() }) const requestStep = z.strictObject({ @@ -329,11 +334,13 @@ const requestStep = z.strictObject({ encoding: z.string().min(1).optional(), delimiter: z.string().length(1).optional(), scalars: z.enum(YAML_SCALARS).optional(), + ocr: ocrSetting.optional(), youtube: youtubeOptions.optional(), }) .refine(step => step.body === undefined || step.form === undefined, { message: 'give body or form, not both', path: ['form'] }) .refine(step => step.delimiter === undefined || step.as === undefined || step.as === 'csv', { message: '"delimiter" reads CSV only: drop it or set "as": "csv"', path: ['delimiter'] }) .refine(step => step.scalars === undefined || step.as === undefined || step.as === 'yaml', { message: '"scalars" reads YAML only: drop it or set "as": "yaml"', path: ['scalars'] }) + .refine(step => step.ocr === undefined || step.ocr === false || step.as === undefined || step.as === 'pdf', { message: '"ocr" reads PDF scans only: drop it or set "as": "pdf"', path: ['ocr'] }) .refine(step => step.youtube === undefined || step.as === 'youtube', { message: '"youtube" is for a video: set "as": "youtube"', path: ['youtube'] }) const tableOnly = ['columns', 'until', 'align', 'sheet', 'headerRows', 'fillDown', 'includeHidden', 'slide', 'shapes'] as const const extractStep = z.strictObject({ diff --git a/packages/core/tools/make-scanned-fixtures.mjs b/packages/core/tools/make-scanned-fixtures.mjs new file mode 100644 index 00000000..50bbb7fc --- /dev/null +++ b/packages/core/tools/make-scanned-fixtures.mjs @@ -0,0 +1,95 @@ +// Writes the scanned-PDF fixtures the OCR tests read into src/pdf-ocr/fixtures/: one-page PDFs whose only content is a JPEG of text, what a scanner makes. They have no +// text layer. The text is drawn with the system's sans-serif font, so run this on a machine that has one, then commit the files: the tests read them, they do not make them. +// Run: node tools/make-scanned-fixtures.mjs +import { mkdirSync, writeFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { createCanvas } from '@napi-rs/canvas' + +const here = dirname(fileURLToPath(import.meta.url)) +const target = join(here, '..', 'src', 'pdf-ocr', 'fixtures') +mkdirSync(target, { recursive: true }) + +/** A one-page PDF with one JPEG filling it. */ +function pdfWithJpeg (jpeg, width, height, pageWidth, pageHeight) { + const parts = [] + const offsets = [] + let length = 0 + const push = (chunk) => { + const buffer = typeof chunk === 'string' ? Buffer.from(chunk, 'latin1') : chunk + parts.push(buffer) + length += buffer.length + } + const object = (number, body) => { + offsets[number] = length + push(`${number} 0 obj\n${body}\nendobj\n`) + } + push('%PDF-1.4\n') + object(1, '<< /Type /Catalog /Pages 2 0 R >>') + object(2, '<< /Type /Pages /Kids [3 0 R] /Count 1 >>') + object(3, `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 ${pageWidth} ${pageHeight}] /Resources << /XObject << /Im0 4 0 R >> >> /Contents 5 0 R >>`) + offsets[4] = length + push(`4 0 obj\n<< /Type /XObject /Subtype /Image /Width ${width} /Height ${height} /ColorSpace /DeviceRGB /BitsPerComponent 8 /Filter /DCTDecode /Length ${jpeg.length} >>\nstream\n`) + push(jpeg) + push('\nendstream\nendobj\n') + const content = `q ${pageWidth} 0 0 ${pageHeight} 0 0 cm /Im0 Do Q` + object(5, `<< /Length ${content.length} >>\nstream\n${content}\nendstream`) + const xref = length + push(`xref\n0 6\n0000000000 65535 f \n${[1, 2, 3, 4, 5].map(number => `${String(offsets[number]).padStart(10, '0')} 00000 n \n`).join('')}`) + push(`trailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`) + + return Buffer.concat(parts) +} + +/** Draws lines of text on a sheet and scans it: `tiltDegrees` is how crooked the paper went in, `noise` (0 to 1) the scanner's grain. */ +function scan ({ lines, pageWidth = 420, pageHeight = 595, pixelsPerPoint = 2.5, fontPoints = 14, tiltDegrees = 0, noise = 0, shade = 0, quality = 70 }) { + const width = Math.round(pageWidth * pixelsPerPoint) + const height = Math.round(pageHeight * pixelsPerPoint) + const canvas = createCanvas(width, height) + const context = canvas.getContext('2d') + // Paper that is darker at one side, as a scan near a book's spine is: `shade` is how many levels darker. + const paper = context.createLinearGradient(0, 0, width, 0) + paper.addColorStop(0, `rgb(${246 - shade}, ${243 - shade}, ${236 - shade})`) + paper.addColorStop(1, '#f6f3ec') + context.fillStyle = paper + context.fillRect(0, 0, width, height) + context.save() + context.translate(width / 2, height / 2) + context.rotate((tiltDegrees * Math.PI) / 180) + context.translate(-width / 2, -height / 2) + context.fillStyle = '#1c1c1c' + const size = fontPoints * pixelsPerPoint + context.font = `${size}px sans-serif` + for (const [index, line] of lines.entries()) context.fillText(line, 40 * pixelsPerPoint, (70 * pixelsPerPoint) + (index * size * 1.9)) + context.restore() + if (noise > 0) { + const image = context.getImageData(0, 0, width, height) + // A fixed sequence, so the fixture is the same file every time it is made. + let seed = 12_345 + const random = () => { + seed = (Math.imul(seed, 1_103_515_245) + 12_345) >>> 0 + + return seed / 4_294_967_296 + } + for (let at = 0; at < image.data.length; at += 4) { + const shift = Math.round((random() - 0.5) * 110 * noise) + for (let channel = 0; channel < 3; channel += 1) image.data[at + channel] = Math.min(255, Math.max(0, image.data[at + channel] + shift)) + } + context.putImageData(image, 0, 0) + } + + return pdfWithJpeg(canvas.toBuffer('image/jpeg', quality), width, height, pageWidth, pageHeight) +} + +const TABLE = ['Incentivi giugno 2026', 'Marca Modello Prezzo Sconto', 'Fiat Pandina 15950 12,5%', 'Jeep Avenger 24950 8,0%', 'Totale ordini: 1.250'] + +const fixtures = { + 'scan-clean.pdf': scan({ lines: TABLE }), + 'scan-poor.pdf': scan({ lines: TABLE, pixelsPerPoint: 2, tiltDegrees: 2.2, noise: 0.5 }), + // 200 dpi, fed in 4.5 degrees crooked, with shaded paper: read as it is, Tesseract finds nothing; cleaned, every character. + 'scan-crooked-shaded.pdf': scan({ lines: TABLE, pageHeight: 300, pixelsPerPoint: 2.8, fontPoints: 10, tiltDegrees: 4.5, noise: 0.5, shade: 60, quality: 60 }), +} +for (const [name, bytes] of Object.entries(fixtures)) { + writeFileSync(join(target, name), bytes) + console.log(`wrote ${name} (${bytes.length} bytes)`) +} diff --git a/packages/ocr-tesseract/.spec.swcrc b/packages/ocr-tesseract/.spec.swcrc new file mode 100644 index 00000000..3b52a537 --- /dev/null +++ b/packages/ocr-tesseract/.spec.swcrc @@ -0,0 +1,22 @@ +{ + "jsc": { + "target": "es2017", + "parser": { + "syntax": "typescript", + "decorators": true, + "dynamicImport": true + }, + "transform": { + "decoratorMetadata": true, + "legacyDecorator": true + }, + "keepClassNames": true, + "externalHelpers": true, + "loose": true + }, + "module": { + "type": "es6" + }, + "sourceMaps": true, + "exclude": [] +} diff --git a/packages/ocr-tesseract/.swcrc b/packages/ocr-tesseract/.swcrc new file mode 100644 index 00000000..3ced5f17 --- /dev/null +++ b/packages/ocr-tesseract/.swcrc @@ -0,0 +1,22 @@ +{ + "jsc": { + "target": "es2017", + "parser": { + "syntax": "typescript", + "decorators": true, + "dynamicImport": true + }, + "transform": { + "decoratorMetadata": true, + "legacyDecorator": true + }, + "keepClassNames": true, + "externalHelpers": true, + "loose": true + }, + "module": { + "type": "es6" + }, + "sourceMaps": true, + "exclude": ["jest.config.[ct]s",".*\\.spec.tsx?$",".*\\.test.tsx?$","./src/jest-setup.ts$","./**/jest-setup.ts$",".*.js$"] +} diff --git a/packages/ocr-tesseract/README.md b/packages/ocr-tesseract/README.md new file mode 100644 index 00000000..1763a9a0 --- /dev/null +++ b/packages/ocr-tesseract/README.md @@ -0,0 +1,65 @@ +

+ OpenCraw +

+ +# @opencraw/ocr-tesseract + +OCR for scanned PDFs, for [OpenCraw](https://github.com/Open-Craw/OpenCraw/blob/main/README.md). A scanned PDF is a picture of a page with no text in it; +this package reads the picture with Tesseract and gives the words back where they sit on the page, so a recipe's `table`, `region` and `section` +work on a scan as they do on a digital PDF. + +```sh +npm install @opencraw/ocr-tesseract +``` + +Tesseract runs in WebAssembly in worker threads, and the English model comes with the package, so nothing is downloaded at run time. Another +language is one install away: `npm install @tesseract.js-data/por`. + +## Use in a recipe + +Install the package next to `@opencraw/core` and ask for OCR on the `request`: + +```json +{ "type": "request", "id": "doc", "url": "{{start.url}}", "as": "pdf", "ocr": true } +{ "type": "request", "id": "doc", "url": "{{start.url}}", "as": "pdf", "ocr": { "languages": ["por", "eng"], "dpi": 200 } } +``` + +`ocr` is `true` (English, and only the pages that have no text layer) or an object: + +| Option | Meaning | +|---|---| +| `languages` | Tesseract codes (`eng`, `por`, `ita`, `chi_sim`…). Each needs its `@tesseract.js-data/` package, except `eng`. | +| `always` | Read every page by OCR and drop the text layer, for a PDF somebody ran bad OCR over. | +| `dpi` | The resolution a page is rendered at. Default: the scan's own, between 150 and 300. More only blurs a soft scan. | + +Without the package, a request with `ocr` fails and says to install it. A scan read without `ocr` fails and says to ask for it. + +## Use from code + +```ts +import { createOcrEngine } from '@opencraw/ocr-tesseract' +import { pdfPageOcr, readPdf } from '@opencraw/core' + +const engine = createOcrEngine() +const pdf = await readPdf(bytes, 'scan.pdf', { ocr: pdfPageOcr({ engine: () => Promise.resolve(engine), languages: ['eng'] }) }) +await engine.dispose?.() +``` + +`createOcrEngine(options)` takes `languages` (used when a recipe names none), `langPath` (a folder of your own `.traineddata.gz` models), +`segmentation` (Tesseract's page segmentation mode, default automatic) and `cleanup` (one way of cleaning a page, a list of ways tried in turn, +or `false` for the picture as it is). Anything that answers the `OcrEngine` contract of `@opencraw/core` can stand in for it: a cloud +service, another local engine. + +## What it does to a page before reading it + +Scans are rarely clean, and Tesseract reads clean pages. Each page is cleaned first, and every word comes back mapped to the original page's +coordinates, not the cleaned one's: + +- **Background flattening** divides out yellowed paper, a dark spine and uneven lighting. +- **Deskew** measures the tilt from the rows of text and straightens the page (up to 5 degrees either way). +- **Adaptive binarisation and a median filter** turn a grainy, low-contrast scan into black ink on white. + +No single cleaning is best for every scan, so the engine reads with a gentle one first, keeps it when the read is confident, and otherwise +tries a stronger one and keeps whichever read better. + +OCR reads what is printed; it can still confuse a character (`8,0%` read as `8,09`). Check a number that matters against a second source. diff --git a/packages/ocr-tesseract/jest-pdfjs.cjs b/packages/ocr-tesseract/jest-pdfjs.cjs new file mode 100644 index 00000000..8dd5830e --- /dev/null +++ b/packages/ocr-tesseract/jest-pdfjs.cjs @@ -0,0 +1,8 @@ +// Jest runs tests as CommonJS in its own module registry, which cannot load +// pdf.js (an ES module that reads import.meta), and it hands tests its own +// `module` builtin. Node's real one, from process.getBuiltinModule, gives a +// native require, which loads ES modules. Tests only: the published build +// imports pdf.js natively. +const { createRequire } = process.getBuiltinModule('node:module') + +module.exports = createRequire(__filename)('pdfjs-dist/legacy/build/pdf.mjs') diff --git a/packages/ocr-tesseract/jest.config.cts b/packages/ocr-tesseract/jest.config.cts new file mode 100644 index 00000000..9eff3f14 --- /dev/null +++ b/packages/ocr-tesseract/jest.config.cts @@ -0,0 +1,23 @@ +/* eslint-disable */ +const { readFileSync } = require('fs') + +// Reading the SWC compilation config for the spec files +const swcJestConfig = JSON.parse(readFileSync(`${__dirname}/.spec.swcrc`, 'utf-8')) + +// Disable .swcrc look-up by SWC core because we're passing in swcJestConfig ourselves +swcJestConfig.swcrc = false + +module.exports = { + displayName: '@opencraw/ocr-tesseract', + preset: '../../jest.preset.js', + testEnvironment: 'node', + transform: { + '^.+\\.[tj]s$': ['@swc/jest', swcJestConfig], + }, + moduleFileExtensions: ['ts', 'js', 'html'], + // pdf.js is an ES module Jest cannot load itself: see jest-pdfjs.cjs. + moduleNameMapper: { '^pdfjs-dist/legacy/build/pdf\\.mjs$': '/jest-pdfjs.cjs', '^@opencraw/ocr-tesseract$': '/src/index.ts' }, + // marked ships as an ES module only: transform it like the sources (@opencraw/core is read from its sources). + transformIgnorePatterns: ['/node_modules/(?!marked/)'], + coverageDirectory: 'test-output/jest/coverage', +} diff --git a/packages/ocr-tesseract/package.json b/packages/ocr-tesseract/package.json new file mode 100644 index 00000000..3b87c785 --- /dev/null +++ b/packages/ocr-tesseract/package.json @@ -0,0 +1,52 @@ +{ + "name": "@opencraw/ocr-tesseract", + "version": "0.0.1", + "type": "module", + "main": "./dist/index.esm.js", + "module": "./dist/index.esm.js", + "types": "./dist/src/index.d.ts", + "exports": { + "./package.json": "./package.json", + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/index.esm.js", + "default": "./dist/index.esm.js" + } + }, + "files": [ + "dist", + "!**/*.tsbuildinfo", + "!**/*.d.ts.map", + "!**/*.js.map" + ], + "dependencies": { + "@tesseract.js-data/eng": "^1.0.0", + "pngjs": "^7.0.0", + "tesseract.js": "^7.0.0" + }, + "peerDependencies": { + "@opencraw/core": "^0.0.1" + }, + "publishConfig": { + "access": "public" + }, + "description": "OCR for scanned PDFs, for OpenCraw: Tesseract (WebAssembly, bundled English model, other languages from @tesseract.js-data packages) behind the engine contract @opencraw/core reads a scan with. Each page is cleaned before it is read (background flattened, straightened, binarized), and every word comes back where it was on the original page.", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/Open-Craw/OpenCraw.git", + "directory": "packages/ocr-tesseract" + }, + "keywords": [ + "ocr", + "tesseract", + "pdf", + "scan", + "crawler", + "opencraw" + ], + "engines": { + "node": ">=22" + }, + "sideEffects": false +} diff --git a/packages/ocr-tesseract/rollup.config.cjs b/packages/ocr-tesseract/rollup.config.cjs new file mode 100644 index 00000000..3596d961 --- /dev/null +++ b/packages/ocr-tesseract/rollup.config.cjs @@ -0,0 +1,116 @@ +const { withNx } = require('@nx/rollup/with-nx') + +module.exports = withNx( + { + main: './src/index.ts', + outputPath: './dist', + tsConfig: './tsconfig.lib.json', + // Swapped from swc by MoNecromanCI. @nx/rollup runs swc without + // sourceMaps, so it returns no map and the bundle's map comes out empty - + // valid-looking, and useless for debugging. See ROADMAP. + compiler: 'babel', + format: ['esm'], + // Added by MoNecromanCI: without this rollup emits no .js.map at all, so + // a breakpoint in a .ts file can never bind. Not published - see `files`. + sourceMap: true, + }, + { + // Added by MoNecromanCI. rollup hands sourcemapPathTransform an OS-NATIVE + // path with one parent segment too many, so `sources` resolve to nothing + // and no breakpoint can bind. Separators are normalised too: a sources + // entry is URL-style, so a backslash is wrong on every platform. + output: { + sourcemapPathTransform: relativeSourcePath => + relativeSourcePath + .replaceAll(String.fromCodePoint(92), '/') + .replace(/^(\.\.\/)+/, '../'), + }, + // Added by MoNecromanCI. @nx/rollup's dts-bundle plugin writes dist/index.d.ts + // as a stub re-exporting the real declarations, and builds that specifier with + // path.relative() - an OS-NATIVE path. On Windows it emits + // export * from "./src\\index"; + // which is not a valid module specifier on ANY platform: a specifier is + // URL-style, so / is correct everywhere and a backslash nowhere. It resolves on + // Windows only because the resolver normalises separators there, leaving the + // package untyped on Linux and macOS. + // + // mnci also points `types` past this stub, so nothing depends on it being + // correct; this makes the emitted file correct too. Remove once Nx fixes the + // plugin - its own devkit already exports normalizePath for exactly this. + // + // The second half fixes a separate defect in the REAL declarations `types` + // points at: tsconfig.lib.json declares under moduleResolution "bundler", + // where a bare relative specifier ("./lib/align") is valid, so every emitted + // .d.ts keeps the source's own extensionless imports verbatim. A consumer on + // "moduleResolution": "nodenext" - the workspace root default - requires an + // explicit extension on every relative specifier and gets TS2834 (or, + // combined with skipLibCheck, a silent zero-export module) instead. Declared + // extensions are left untouched; only a bare relative specifier gets .js + // appended, matching what tsc itself emits under node16/nodenext. + plugins: [ + { + name: 'mnci-normalise-declaration-specifiers', + writeBundle (outputOptions) { + const { existsSync, readdirSync, readFileSync, writeFileSync } = require('node:fs') + const { join } = require('node:path') + const dir = outputOptions.dir ?? './dist' + const stub = join(dir, 'index.d.ts') + let source + try { + source = readFileSync(stub, 'utf8') + } catch { + return + } + // The stub carries a TWO-character escape (JSON.stringify escaped one + // backslash), so this must not match a single one - that would turn + // "./src\index" into "./src//index". Built from char codes so there is + // no escaping in this file to get wrong. + const separator = String.fromCodePoint(92, 92) + const normalised = source.replaceAll(separator, '/') + if (normalised !== source) writeFileSync(stub, normalised) + + const bareRelativeSpecifier = /from(\s+)(['"])(\.[^'"]+)\2/g + const hasExtension = /\.(?:mjs|cjs|jsx?|json)$/ + let entries + try { + entries = readdirSync(dir, { recursive: true, withFileTypes: true }) + } catch { + return + } + for (const entry of entries) { + if (!entry.name.endsWith('.d.ts')) continue + const from = entry.parentPath ?? entry.path + const filePath = join(from, entry.name) + let declaration + try { + declaration = readFileSync(filePath, 'utf8') + } catch { + continue + } + // A bare specifier may name a FILE or a DIRECTORY BARREL - resolved + // against what rollup actually emitted next to this file, never + // guessed. A directory needs /index.js, not .js (a file that was + // never written); anything neither form matches is left alone rather + // than rewritten to a specifier that cannot resolve. + const resolveSpecifierSuffix = (specifier) => { + if (hasExtension.test(specifier)) return null + if (existsSync(join(from, specifier + '.d.ts'))) return specifier + '.js' + if (existsSync(join(from, specifier, 'index.d.ts'))) return specifier + '/index.js' + + return null + } + const withExtensions = declaration.replaceAll( + bareRelativeSpecifier, + (match, space, quote, specifier) => { + const resolved = resolveSpecifierSuffix(specifier) + + return resolved === null ? match : `from${space}${quote}${resolved}${quote}` + }, + ) + if (withExtensions !== declaration) writeFileSync(filePath, withExtensions) + } + }, + }, + ], + }, +) diff --git a/packages/ocr-tesseract/src/index.ts b/packages/ocr-tesseract/src/index.ts new file mode 100644 index 00000000..02d7dd10 --- /dev/null +++ b/packages/ocr-tesseract/src/index.ts @@ -0,0 +1,5 @@ +export { createOcrEngine } from './recognition' +export type { OcrEngineOptions } from './recognition' +export { OcrLanguageError } from './recognition' +export { cleanPage } from './page-cleanup' +export type { CleanPage, CleanupOptions } from './page-cleanup' diff --git a/packages/ocr-tesseract/src/page-cleanup/binarize.algorithm.ts b/packages/ocr-tesseract/src/page-cleanup/binarize.algorithm.ts new file mode 100644 index 00000000..a5543677 --- /dev/null +++ b/packages/ocr-tesseract/src/page-cleanup/binarize.algorithm.ts @@ -0,0 +1,96 @@ +import { grayImage } from './gray-image.model' +import type { GrayImage } from './gray-image.model' +import { higherOf, lowerOf, roundedUp, wholePartOf } from './integer-math.algorithm' + +const BLACK = 0 +const WHITE = 255 + +/** + * The threshold that best splits the values in two groups (Otsu's method): the one maximising the variance between ink and ground. + * + * @param values - One 0 to 255 value per pixel. + * @returns The threshold: values at or under it are ink. + */ +export function otsuThreshold (values: Uint8Array): number { + const histogram: number[] = Array.from({ length: 256 }, () => 0) + for (const value of values) histogram[value] = (histogram[value] ?? 0) + 1 + const total = values.length + let weighted = 0 + for (let level = 0; level < 256; level += 1) weighted += level * (histogram[level] ?? 0) + let below = 0 + let belowWeighted = 0 + let best = 0 + let bestVariance = -1 + for (let level = 0; level < 256; level += 1) { + below += histogram[level] ?? 0 + if (below === 0) continue + const above = total - below + if (above === 0) break + belowWeighted += level * (histogram[level] ?? 0) + const spread = (belowWeighted / below) - ((weighted - belowWeighted) / above) + const variance = below * above * spread * spread + if (variance > bestVariance) { + bestVariance = variance + best = level + } + } + + return best +} + +/** Black ink on white from one global threshold. */ +export function globalBinarize (image: GrayImage): GrayImage { + const threshold = otsuThreshold(image.pixels) + const out = grayImage(image.width, image.height) + for (let at = 0; at < image.pixels.length; at += 1) out.pixels[at] = (image.pixels[at] ?? WHITE) <= threshold ? BLACK : WHITE + + return out +} + +/** + * Black ink on white from a threshold that follows the page (Sauvola's method): each pixel is compared with the mean of the window around it, + * lowered where the window is flat, so shading and stains do not turn paper into ink and faint strokes are kept. Integral images make the cost + * independent of the window's size. + * + * @param image - The scan, in gray. + * @param window - The window's side in pixels, about the height of a line of text; default a sixteenth of the shorter side, at least 15. + * @param sensitivity - How far below the local mean ink is taken to start; 0.2 for ordinary print. + * @returns The picture, only black and white. + */ +export function adaptiveBinarize (image: GrayImage, window = higherOf(15, roundedUp(lowerOf(image.width, image.height) / 16)), sensitivity = 0.2): GrayImage { + const { width, height, pixels } = image + const columns = width + 1 + const sums = new Float64Array(columns * (height + 1)) + const squares = new Float64Array(columns * (height + 1)) + for (let y = 0; y < height; y += 1) { + let rowSum = 0 + let rowSquare = 0 + for (let x = 0; x < width; x += 1) { + const value = pixels[(y * width) + x] ?? WHITE + rowSum += value + rowSquare += value * value + sums[((y + 1) * columns) + x + 1] = (sums[(y * columns) + x + 1] ?? 0) + rowSum + squares[((y + 1) * columns) + x + 1] = (squares[(y * columns) + x + 1] ?? 0) + rowSquare + } + } + const half = wholePartOf(window / 2) + const out = grayImage(width, height) + for (let y = 0; y < height; y += 1) { + const top = higherOf(0, y - half) + const bottom = lowerOf(height - 1, y + half) + 1 + for (let x = 0; x < width; x += 1) { + const left = higherOf(0, x - half) + const right = lowerOf(width - 1, x + half) + 1 + const area = (right - left) * (bottom - top) + const sum = (sums[(bottom * columns) + right] ?? 0) - (sums[(top * columns) + right] ?? 0) - (sums[(bottom * columns) + left] ?? 0) + (sums[(top * columns) + left] ?? 0) + const square = (squares[(bottom * columns) + right] ?? 0) - (squares[(top * columns) + right] ?? 0) - (squares[(bottom * columns) + left] ?? 0) + (squares[(top * columns) + left] ?? 0) + const mean = sum / area + const variance = (square / area) - (mean * mean) + const deviation = variance > 0 ? Math.sqrt(variance) : 0 + const threshold = mean * (1 + (sensitivity * ((deviation / 128) - 1))) + out.pixels[(y * width) + x] = (pixels[(y * width) + x] ?? WHITE) <= threshold ? BLACK : WHITE + } + } + + return out +} diff --git a/packages/ocr-tesseract/src/page-cleanup/clean-page.use-case.ts b/packages/ocr-tesseract/src/page-cleanup/clean-page.use-case.ts new file mode 100644 index 00000000..7538356d --- /dev/null +++ b/packages/ocr-tesseract/src/page-cleanup/clean-page.use-case.ts @@ -0,0 +1,73 @@ +import { adaptiveBinarize, globalBinarize } from './binarize.algorithm' +import { estimateSkew, rotate, unrotate } from './deskew.algorithm' +import { flattenBackground } from './flatten-background.algorithm' +import { medianFilter } from './median-filter.algorithm' +import { grayImage } from './gray-image.model' +import type { GrayImage } from './gray-image.model' +import { grayOfPng, pngOfGray } from './png-gray.mapper' + +/** + * How a scanned page is cleaned before Tesseract reads it. Flattening the background and levelling the lines are on by default: measured on scans with shaded + * paper or a crooked feed, they are the difference between reading nothing and reading every character. Binarizing is off by default: on a flattened page it gains + * nothing over Tesseract's own, and at low resolution it thins strokes. + */ +export interface CleanupOptions { + /** Take out shading and a tinted paper. Default true. */ + flatten?: boolean + /** Level the lines of a page that was fed in crooked: `true` finds the angle, a number is the angle to turn by (clockwise, degrees). Default true. */ + deskew?: boolean | number + /** Ink as pure black on white: `adaptive` follows the page, `global` is one threshold for all of it, `none` keeps the grays. Default none. */ + binarize?: 'adaptive' | 'global' | 'none' + /** Take the grain out (a 3 by 3 median) before the ink is made black on white. Default false. */ + denoise?: boolean + /** White margin around the page, in pixels: Tesseract misses glyphs that touch the edge. Default 12. */ + padding?: number +} + +/** A cleaned page: its picture, and where its pixels came from on the picture it was made from. */ +export interface CleanPage { + png: Uint8Array + width: number + height: number + /** How far the page was turned to level its lines, clockwise degrees; 0 when it was level. */ + skewDegrees: number + /** A position on the cleaned picture as a position on the original, undoing the margin and the turn. */ + toSource: (point: { x: number, y: number }) => { x: number, y: number } +} + +/** A skew smaller than this is the measurement's noise, not paper: the page is left as it is. */ +const MIN_SKEW = 0.3 + +/** + * Cleans a page picture for OCR: grayscale, the background flattened, the lines levelled, optionally the ink made black on white by a threshold that + * follows the page, and a white margin. A scan is read better when it looks like a clean copy of what it was a picture of. + * + * @param png - The page, as PNG bytes. + * @param options - What to do. + * @returns The cleaned picture and how to map positions on it back to the original. + */ +export function cleanPage (png: Uint8Array, options: CleanupOptions = {}): CleanPage { + let image: GrayImage = grayOfPng(png) + const original = { width: image.width, height: image.height } + if (options.flatten !== false) image = flattenBackground(image) + let skew = 0 + if (typeof options.deskew === 'number') skew = options.deskew + else if (options.deskew !== false) skew = estimateSkew(image) + if (Math.abs(skew) < MIN_SKEW) skew = 0 + image = rotate(image, skew) + if (options.denoise === true) image = medianFilter(image) + const binarize = options.binarize ?? 'none' + if (binarize === 'adaptive') image = adaptiveBinarize(image) + else if (binarize === 'global') image = globalBinarize(image) + const padding = Math.max(0, Math.round(options.padding ?? 12)) + const padded = grayImage(image.width + (2 * padding), image.height + (2 * padding)) + for (let y = 0; y < image.height; y += 1) padded.pixels.set(image.pixels.subarray(y * image.width, (y + 1) * image.width), ((y + padding) * padded.width) + padding) + + return { + png: pngOfGray(padded), + width: padded.width, + height: padded.height, + skewDegrees: skew, + toSource: point => unrotate({ x: point.x - padding, y: point.y - padding }, skew, original), + } +} diff --git a/packages/ocr-tesseract/src/page-cleanup/deskew.algorithm.ts b/packages/ocr-tesseract/src/page-cleanup/deskew.algorithm.ts new file mode 100644 index 00000000..7e0a7bb2 --- /dev/null +++ b/packages/ocr-tesseract/src/page-cleanup/deskew.algorithm.ts @@ -0,0 +1,159 @@ +import { otsuThreshold } from './binarize.algorithm' +import { grayImage } from './gray-image.model' +import type { GrayImage } from './gray-image.model' +import { higherOf, roundedUp, wholePartOf } from './integer-math.algorithm' + +/** The long side the skew is measured on: a scan's skew shows at a fraction of its size, and the measure is quadratic in pixels. */ +const MEASURE_SIDE = 700 +/** The most points of ink looked at: the skew shows in a sample as well as in all of them. */ +const MAX_POINTS = 60_000 + +interface Point { + x: number + y: number +} + +/** The picture shrunk by whole-pixel averaging so its long side is about `side`, and how many times smaller it is. */ +function shrunk (image: GrayImage, side: number): { image: GrayImage, factor: number } { + const factor = higherOf(1, wholePartOf(higherOf(image.width, image.height) / side)) + if (factor === 1) return { image, factor } + const width = wholePartOf(image.width / factor) + const height = wholePartOf(image.height / factor) + const out = grayImage(width, height) + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + let sum = 0 + for (let dy = 0; dy < factor; dy += 1) for (let dx = 0; dx < factor; dx += 1) sum += image.pixels[(((y * factor) + dy) * image.width) + (x * factor) + dx] ?? 255 + out.pixels[(y * width) + x] = roundedUp(sum / (factor * factor)) + } + } + + return { image: out, factor } +} + +/** The ink's coordinates around the picture's centre, thinned to at most {@link MAX_POINTS}. */ +function inkPoints (image: GrayImage): Point[] { + const threshold = otsuThreshold(image.pixels) + const points: Point[] = [] + for (let y = 0; y < image.height; y += 1) { + for (let x = 0; x < image.width; x += 1) if ((image.pixels[(y * image.width) + x] ?? 255) <= threshold) points.push({ x: x - (image.width / 2), y: y - (image.height / 2) }) + } + const every = higherOf(1, Math.ceil(points.length / MAX_POINTS)) + + return every === 1 ? points : points.filter((_, index) => index % every === 0) +} + +/** How sharply the ink falls into rows when the page is turned by `degrees`: rows of text and the gaps between them make a spiky histogram only when the lines are level. */ +function sharpness (points: readonly Point[], degrees: number, rows: number): number { + const radians = (degrees * Math.PI) / 180 + const sine = Math.sin(radians) + const cosine = Math.cos(radians) + const histogram = new Float64Array(rows) + for (const point of points) { + const position = (point.x * sine) + (point.y * cosine) + (rows / 2) + const row = position < 0 ? -1 : roundedUp(position) + if (row >= 0 && row < rows) histogram[row] = (histogram[row] ?? 0) + 1 + } + let sum = 0 + for (const count of histogram) sum += count * count + + return sum +} + +/** + * How much a scan was fed in crooked, in degrees, from the sharpness of its rows of text: turning the page by the right amount makes the lines + * level, and the ink then falls into few, full rows. Searched coarsely over `maxDegrees` either way, then finely around the best. + * + * @param image - The scan, in gray. + * @param maxDegrees - The most skew looked for either way; default 5. + * @returns The angle to turn the picture by (clockwise, degrees) to level the text; 0 when there is no text to measure. + */ +export function estimateSkew (image: GrayImage, maxDegrees = 5): number { + const { image: small } = shrunk(image, MEASURE_SIDE) + const points = inkPoints(small) + if (points.length < 200) return 0 + const rows = Math.ceil(Math.hypot(small.width, small.height)) + 2 + let best = 0 + let bestScore = sharpness(points, 0, rows) + for (let degrees = -maxDegrees; degrees <= maxDegrees; degrees += 0.5) { + const score = sharpness(points, degrees, rows) + if (score > bestScore) { + best = degrees + bestScore = score + } + } + const around = best + for (let degrees = around - 0.5; degrees <= around + 0.5; degrees += 0.05) { + const score = sharpness(points, degrees, rows) + if (score > bestScore) { + best = degrees + bestScore = score + } + } + + return Math.round(best * 100) / 100 +} + +/** The picture's value at a position between pixels (bilinear), white where the position is off the picture. */ +function sampled (image: GrayImage, x: number, y: number): number { + const { width, height, pixels } = image + if (x < 0 || y < 0 || x > width - 1 || y > height - 1) return 255 + const left = wholePartOf(x) + const top = wholePartOf(y) + const right = left < width - 1 ? left + 1 : left + const below = top < height - 1 ? top + 1 : top + const fx = x - left + const fy = y - top + const value = ((pixels[(top * width) + left] ?? 255) * (1 - fx) * (1 - fy)) + ((pixels[(top * width) + right] ?? 255) * fx * (1 - fy)) + ((pixels[(below * width) + left] ?? 255) * (1 - fx) * fy) + ((pixels[(below * width) + right] ?? 255) * fx * fy) + + return roundedUp(value) +} + +/** + * Turns a picture clockwise about its centre, the corners it uncovers white and the edges smoothed (bilinear). The picture keeps its size. + * + * @param image - The picture. + * @param degrees - How far to turn it, clockwise. + * @returns The turned picture. + */ +export function rotate (image: GrayImage, degrees: number): GrayImage { + if (degrees === 0) return image + const radians = (degrees * Math.PI) / 180 + const sine = Math.sin(radians) + const cosine = Math.cos(radians) + const centreX = image.width / 2 + const centreY = image.height / 2 + const out = grayImage(image.width, image.height) + for (let y = 0; y < image.height; y += 1) { + for (let x = 0; x < image.width; x += 1) { + // Where this pixel came from: the output turned back by the same angle. + const dx = x - centreX + const dy = y - centreY + const sourceX = (dx * cosine) + (dy * sine) + centreX + const sourceY = (-dx * sine) + (dy * cosine) + centreY + out.pixels[(y * image.width) + x] = sampled(image, sourceX, sourceY) + } + } + + return out +} + +/** + * Where a point of the turned picture was in the picture before it was turned: the inverse of {@link rotate}, to put what was read from a straightened + * page back where it sits on the original. + * + * @param point - A position in the turned picture. + * @param degrees - The angle the picture was turned by. + * @param size - The picture's size. + * @returns The position in the original. + */ +export function unrotate (point: { x: number, y: number }, degrees: number, size: { width: number, height: number }): { x: number, y: number } { + const radians = (degrees * Math.PI) / 180 + const sine = Math.sin(radians) + const cosine = Math.cos(radians) + const dx = point.x - (size.width / 2) + const dy = point.y - (size.height / 2) + + // The same mapping `rotate` uses to find where each output pixel came from. + return { x: (dx * cosine) + (dy * sine) + (size.width / 2), y: (-dx * sine) + (dy * cosine) + (size.height / 2) } +} diff --git a/packages/ocr-tesseract/src/page-cleanup/flatten-background.algorithm.ts b/packages/ocr-tesseract/src/page-cleanup/flatten-background.algorithm.ts new file mode 100644 index 00000000..94734ec8 --- /dev/null +++ b/packages/ocr-tesseract/src/page-cleanup/flatten-background.algorithm.ts @@ -0,0 +1,86 @@ +import { grayImage } from './gray-image.model' +import type { GrayImage } from './gray-image.model' +import { higherOf, lowerOf, roundedUp } from './integer-math.algorithm' + +/** The ink is thinner than this many pixels at the page's usual scale, so the background estimate (the lightest pixel around) steps over it. */ +const INK_RADIUS = 6 + +/** Running maximum along one axis, window `2 * radius + 1`: the lightest value around each pixel. */ +function lightest (values: Uint8Array, width: number, height: number, radius: number, vertical: boolean): Uint8Array { + const out = new Uint8Array(values.length) + const length = vertical ? height : width + const lines = vertical ? width : height + const stride = vertical ? width : 1 + const step = vertical ? 1 : width + for (let line = 0; line < lines; line += 1) { + const start = line * step + for (let at = 0; at < length; at += 1) { + const from = higherOf(0, at - radius) + const to = lowerOf(length - 1, at + radius) + let best = 0 + for (let near = from; near <= to; near += 1) { + const value = values[start + (near * stride)] ?? 0 + if (value > best) best = value + } + out[start + (at * stride)] = best + } + } + + return out +} + +/** Box blur along one axis, window `2 * radius + 1`, by running sums: the cost does not grow with the radius. */ +function averaged (values: Uint8Array, width: number, height: number, radius: number, vertical: boolean): Uint8Array { + const out = new Uint8Array(values.length) + const length = vertical ? height : width + const lines = vertical ? width : height + const stride = vertical ? width : 1 + const step = vertical ? 1 : width + for (let line = 0; line < lines; line += 1) { + const start = line * step + let sum = 0 + let count = 0 + for (let near = 0; near <= lowerOf(length - 1, radius); near += 1) { + sum += values[start + (near * stride)] ?? 0 + count += 1 + } + for (let at = 0; at < length; at += 1) { + out[start + (at * stride)] = roundedUp(sum / count) + const entering = at + radius + 1 + const leaving = at - radius + if (entering < length) { + sum += values[start + (entering * stride)] ?? 0 + count += 1 + } + if (leaving >= 0) { + sum -= values[start + (leaving * stride)] ?? 0 + count -= 1 + } + } + } + + return out +} + +/** + * Takes the shading out of a scan: yellowed paper, a darker edge near the spine, the lamp's fall-off. The page's background is estimated as the + * lightest pixel around each one (which steps over ink), smoothed, and every pixel is divided by it, so paper comes out white wherever it was + * and ink stays dark. Like photocopying with the lid raised. + * + * @param image - The scan, in gray. + * @returns The flattened picture. + */ +export function flattenBackground (image: GrayImage): GrayImage { + const { width, height, pixels } = image + const smooth = higherOf(12, roundedUp(lowerOf(width, height) / 25)) + let background = lightest(lightest(pixels, width, height, INK_RADIUS, false), width, height, INK_RADIUS, true) + background = averaged(averaged(background, width, height, smooth, false), width, height, smooth, true) + const out = grayImage(width, height) + for (const [at, pixel] of pixels.entries()) { + const paper = background[at] ?? 255 + const value = roundedUp(((pixel ?? 255) * 255) / (Math.max(paper, 1))) + out.pixels[at] = Math.min(value, 255) + } + + return out +} diff --git a/packages/ocr-tesseract/src/page-cleanup/gray-image.model.ts b/packages/ocr-tesseract/src/page-cleanup/gray-image.model.ts new file mode 100644 index 00000000..d0ea9652 --- /dev/null +++ b/packages/ocr-tesseract/src/page-cleanup/gray-image.model.ts @@ -0,0 +1,18 @@ +/** A picture in shades of gray: one 0 (black) to 255 (white) value per pixel, rows top to bottom. */ +export interface GrayImage { + width: number + height: number + pixels: Uint8Array +} + +/** + * A blank picture. + * + * @param width - Pixels across. + * @param height - Pixels down. + * @param fill - The gray every pixel starts as; default white. + * @returns The picture. + */ +export function grayImage (width: number, height: number, fill = 255): GrayImage { + return { width, height, pixels: new Uint8Array(width * height).fill(fill) } +} diff --git a/packages/ocr-tesseract/src/page-cleanup/index.ts b/packages/ocr-tesseract/src/page-cleanup/index.ts new file mode 100644 index 00000000..1df56b0a --- /dev/null +++ b/packages/ocr-tesseract/src/page-cleanup/index.ts @@ -0,0 +1,9 @@ +export { cleanPage } from './clean-page.use-case' +export type { CleanPage, CleanupOptions } from './clean-page.use-case' +export { adaptiveBinarize, globalBinarize, otsuThreshold } from './binarize.algorithm' +export { estimateSkew, rotate, unrotate } from './deskew.algorithm' +export { flattenBackground } from './flatten-background.algorithm' +export { grayImage } from './gray-image.model' +export { medianFilter } from './median-filter.algorithm' +export type { GrayImage } from './gray-image.model' +export { grayOfPng, pngOfGray } from './png-gray.mapper' diff --git a/packages/ocr-tesseract/src/page-cleanup/integer-math.algorithm.ts b/packages/ocr-tesseract/src/page-cleanup/integer-math.algorithm.ts new file mode 100644 index 00000000..4e4acf50 --- /dev/null +++ b/packages/ocr-tesseract/src/page-cleanup/integer-math.algorithm.ts @@ -0,0 +1,35 @@ +/* eslint-disable unicorn/prefer-math-min-max, unicorn/prefer-math-trunc -- see below: these helpers exist to not call Math.* */ +/** + * The few arithmetic helpers the per-pixel loops use. They are plain comparisons and bit operations on purpose: a call to `Math.min`, `Math.max`, + * `Math.floor` or `Math.round` is about a hundred times slower inside a sandboxed test runner than in Node (measured: 150 ns against 1 ns), and these + * loops make tens of millions of them a page. With these the cleanup of a page takes a third of a second under the test runner, not fifteen seconds. + * Written as `Math.*` the rules above would be right; do not "simplify" them. + */ + +/** The smaller of two numbers. */ +export function lowerOf (a: number, b: number): number { + return a < b ? a : b +} + +/** The larger of two numbers. */ +export function higherOf (a: number, b: number): number { + return a > b ? a : b +} + +/** A number held between two limits. */ +export function clampedTo (value: number, low: number, high: number): number { + if (value < low) return low + if (value > high) return high + + return value +} + +/** A non-negative number rounded to the nearest integer (halves up), as `Math.round` does for it. */ +export function roundedUp (value: number): number { + return (value + 0.5) | 0 +} + +/** A non-negative number with its fraction dropped, as `Math.floor` does for it. */ +export function wholePartOf (value: number): number { + return value | 0 +} diff --git a/packages/ocr-tesseract/src/page-cleanup/median-filter.algorithm.ts b/packages/ocr-tesseract/src/page-cleanup/median-filter.algorithm.ts new file mode 100644 index 00000000..5356398d --- /dev/null +++ b/packages/ocr-tesseract/src/page-cleanup/median-filter.algorithm.ts @@ -0,0 +1,51 @@ +import { grayImage } from './gray-image.model' +import type { GrayImage } from './gray-image.model' + +/** The middle value of nine, by insertion sort on a small buffer: no allocation and no generic sort per pixel. */ +function medianOfNine (values: Uint8Array): number { + for (let at = 1; at < 9; at += 1) { + const value = values[at] ?? 0 + let before = at - 1 + while (before >= 0 && (values[before] ?? 0) > value) { + values[before + 1] = values[before] ?? 0 + before -= 1 + } + values[before + 1] = value + } + + return values[4] ?? 255 +} + +/** + * Replaces each pixel by the median of the 3 by 3 around it: the speckle of a grainy scan or a photocopy goes, and strokes keep their edges (a mean + * would smear them). The picture's edge repeats outward. + * + * @param image - The picture. + * @returns The picture without the grain. + */ +export function medianFilter (image: GrayImage): GrayImage { + const { width, height, pixels } = image + const out = grayImage(width, height) + const around = new Uint8Array(9) + for (let y = 0; y < height; y += 1) { + const above = (y > 0 ? y - 1 : 0) * width + const here = y * width + const below = (y < height - 1 ? y + 1 : y) * width + for (let x = 0; x < width; x += 1) { + const left = x > 0 ? x - 1 : 0 + const right = x < width - 1 ? x + 1 : x + around[0] = pixels[above + left] ?? 255 + around[1] = pixels[above + x] ?? 255 + around[2] = pixels[above + right] ?? 255 + around[3] = pixels[here + left] ?? 255 + around[4] = pixels[here + x] ?? 255 + around[5] = pixels[here + right] ?? 255 + around[6] = pixels[below + left] ?? 255 + around[7] = pixels[below + x] ?? 255 + around[8] = pixels[below + right] ?? 255 + out.pixels[here + x] = medianOfNine(around) + } + } + + return out +} diff --git a/packages/ocr-tesseract/src/page-cleanup/page-cleanup.test.ts b/packages/ocr-tesseract/src/page-cleanup/page-cleanup.test.ts new file mode 100644 index 00000000..dfe0d721 --- /dev/null +++ b/packages/ocr-tesseract/src/page-cleanup/page-cleanup.test.ts @@ -0,0 +1,166 @@ +import { adaptiveBinarize, globalBinarize, otsuThreshold } from './binarize.algorithm' +import { cleanPage } from './clean-page.use-case' +import { estimateSkew, rotate, unrotate } from './deskew.algorithm' +import { flattenBackground } from './flatten-background.algorithm' +import { grayImage } from './gray-image.model' +import type { GrayImage } from './gray-image.model' +import { medianFilter } from './median-filter.algorithm' +import { grayOfPng, pngOfGray } from './png-gray.mapper' + +/** A page of text-like lines: rows of black dashes (words) on white, level. */ +function textLikePage (width = 800, height = 600): GrayImage { + const image = grayImage(width, height) + for (let line = 0; line < 12; line += 1) { + const top = 60 + (line * 40) + let x = 60 + while (x < width - 80) { + const word = 30 + ((x * 7 + line * 13) % 60) + for (let y = top; y < top + 14; y += 1) for (let dx = 0; dx < word; dx += 1) image.pixels[(y * width) + x + dx] = 0 + x += word + 14 + } + } + + return image +} + +describe('png-gray', () => { + it('goes from a gray picture to a PNG and back, and counts a transparent pixel as paper', () => { + const image = grayImage(4, 2) + image.pixels.set([0, 50, 100, 150, 200, 255, 10, 20]) + + const back = grayOfPng(pngOfGray(image)) + + expect([back.width, back.height, [...back.pixels]]).toEqual([4, 2, [0, 50, 100, 150, 200, 255, 10, 20]]) + }) +}) + +describe('flattenBackground', () => { + it('brings paper that is shaded across the page to white and leaves the ink dark', () => { + const image = grayImage(400, 300) + // The paper goes from 120 on the left to 230 on the right; a bar of ink sits in the dark part and one in the light part. + for (let y = 0; y < 300; y += 1) for (let x = 0; x < 400; x += 1) image.pixels[(y * 400) + x] = Math.round(120 + ((x / 399) * 110)) + for (let y = 100; y < 112; y += 1) for (let x = 40; x < 140; x += 1) image.pixels[(y * 400) + x] = 20 + for (let y = 100; y < 112; y += 1) for (let x = 260; x < 360; x += 1) image.pixels[(y * 400) + x] = 90 + + const flat = flattenBackground(image) + + const paper = [10, 200, 390].map(x => flat.pixels[(250 * 400) + x] ?? 0) + expect(Math.min(...paper)).toBeGreaterThanOrEqual(245) + expect(flat.pixels[(105 * 400) + 90]).toBeLessThan(40) + expect(flat.pixels[(105 * 400) + 310]).toBeLessThan(140) + }) +}) + +describe('skew', () => { + it('finds how far a page was turned, from its lines of text', () => { + const level = textLikePage() + const crooked = rotate(level, 2.5) + + expect(Math.abs(estimateSkew(level))).toBeLessThan(0.3) + expect(estimateSkew(crooked)).toBeCloseTo(-2.5, 0) + expect(Math.abs(estimateSkew(crooked) + 2.5)).toBeLessThan(0.3) + }) + + it('has no skew to find in a page with nothing on it', () => { + expect(estimateSkew(grayImage(300, 300))).toBe(0) + }) + + it('turns a picture and puts a position back where it was', () => { + const image = grayImage(300, 200) + for (let y = 58; y < 62; y += 1) for (let x = 218; x < 222; x += 1) image.pixels[(y * 300) + x] = 0 + + const turned = rotate(image, 12) + // eslint-disable-next-line unicorn/prefer-iterator-to-array -- this package's lib has no Iterator#toArray + const ink = [...turned.pixels.entries()].filter(([, value]) => value < 128).map(([at]) => ({ x: at % 300, y: Math.floor(at / 300) })) + const count = ink.length + const found = { x: ink.reduce((sum, point) => sum + point.x, 0) / count, y: ink.reduce((sum, point) => sum + point.y, 0) / count } + + expect(count).toBeGreaterThan(8) + expect(Math.hypot(found.x - 220, found.y - 60)).toBeGreaterThan(10) + const back = unrotate(found, 12, { width: 300, height: 200 }) + // Within a pixel: the turn smooths the dot, which moves its centre a little. + expect(Math.hypot(back.x - 220, back.y - 60)).toBeLessThan(1.5) + }) + + it('leaves a picture alone for no turn', () => { + const image = textLikePage(100, 100) + + expect(rotate(image, 0)).toBe(image) + }) +}) + +describe('binarize', () => { + it('splits a page of two grays at the threshold between them', () => { + const values = new Uint8Array(1000) + values.fill(220, 0, 700) + values.fill(40, 700) + + const threshold = otsuThreshold(values) + + expect(threshold).toBeGreaterThanOrEqual(40) + expect(threshold).toBeLessThan(220) + }) + + it('makes ink black on white, one global threshold or one that follows the page', () => { + const image = grayImage(300, 200) + for (let y = 0; y < 200; y += 1) for (let x = 0; x < 300; x += 1) image.pixels[(y * 300) + x] = 150 + Math.round((x / 299) * 90) + for (let y = 90; y < 100; y += 1) for (let x = 20; x < 120; x += 1) image.pixels[(y * 300) + x] = 80 + for (let y = 90; y < 100; y += 1) for (let x = 190; x < 290; x += 1) image.pixels[(y * 300) + x] = 160 + + // A window wider than the strokes: the default scales with a real page, which this small picture is not. + const adaptive = adaptiveBinarize(image, 41) + const global = globalBinarize(image) + + expect(new Set(adaptive.pixels)).toEqual(new Set([0, 255])) + // The ink in the dark part and the faint ink in the light part are both found, the shaded paper is not turned to ink. + expect(adaptive.pixels[(95 * 300) + 70]).toBe(0) + expect(adaptive.pixels[(95 * 300) + 240]).toBe(0) + expect(adaptive.pixels[(30 * 300) + 20]).toBe(255) + expect(adaptive.pixels[(30 * 300) + 280]).toBe(255) + expect(new Set(global.pixels)).toEqual(new Set([0, 255])) + }) +}) + +describe('cleanPage', () => { + it('levels a crooked page, adds a margin, and says where each position on the clean picture is on the original', () => { + const level = textLikePage() + const crooked = rotate(level, 2.2) + + const clean = cleanPage(pngOfGray(crooked), { padding: 10, binarize: 'adaptive' }) + + expect(Math.abs(clean.skewDegrees + 2.2)).toBeLessThan(0.3) + expect([clean.width, clean.height]).toEqual([820, 620]) + // The page's centre is the centre of the original, whatever the margin and the turn. + const centre = clean.toSource({ x: 410, y: 310 }) + expect(centre.x).toBeCloseTo(400, 0) + expect(centre.y).toBeCloseTo(300, 0) + // A position at the top left of the clean text maps back near the top of the tilted page. + const decoded = grayOfPng(clean.png) + expect(new Set(decoded.pixels)).toEqual(new Set([0, 255])) + }) + + it('leaves a level page unturned, and does only what it is asked to', () => { + const level = textLikePage() + const clean = cleanPage(pngOfGray(level), { padding: 0, binarize: 'none', flatten: false, deskew: false }) + + expect(clean.skewDegrees).toBe(0) + expect([...grayOfPng(clean.png).pixels]).toEqual([...level.pixels]) + expect(cleanPage(pngOfGray(level), { padding: 0, deskew: 4 }).skewDegrees).toBe(4) + }) +}) + +describe('medianFilter', () => { + it('takes isolated specks out and keeps a stroke\'s edge where it is', () => { + const image = grayImage(40, 20) + for (let y = 0; y < 20; y += 1) for (let x = 20; x < 40; x += 1) image.pixels[(y * 40) + x] = 0 + image.pixels[(10 * 40) + 5] = 0 + image.pixels[(5 * 40) + 30] = 255 + + const filtered = medianFilter(image) + + expect(filtered.pixels[(10 * 40) + 5]).toBe(255) + expect(filtered.pixels[(5 * 40) + 30]).toBe(0) + expect(filtered.pixels[(10 * 40) + 19]).toBe(255) + expect(filtered.pixels[(10 * 40) + 20]).toBe(0) + }) +}) diff --git a/packages/ocr-tesseract/src/page-cleanup/png-gray.mapper.ts b/packages/ocr-tesseract/src/page-cleanup/png-gray.mapper.ts new file mode 100644 index 00000000..dabbf239 --- /dev/null +++ b/packages/ocr-tesseract/src/page-cleanup/png-gray.mapper.ts @@ -0,0 +1,46 @@ +import { PNG } from 'pngjs' +import { grayImage } from './gray-image.model' +import type { GrayImage } from './gray-image.model' +import { roundedUp } from './integer-math.algorithm' + +/** PNG output for Tesseract, not for storage: no row filter search and light compression, many times faster to write. */ +const FAST_PNG = { filterType: 0, deflateLevel: 1 } + +/** + * A PNG as a gray picture: luminance by the usual weights, a transparent pixel counted as white paper. + * + * @param png - The PNG's bytes. + * @returns The picture. + */ +export function grayOfPng (png: Uint8Array): GrayImage { + const decoded = PNG.sync.read(Buffer.from(png)) + const image = grayImage(decoded.width, decoded.height) + for (let pixel = 0; pixel < decoded.width * decoded.height; pixel += 1) { + const at = pixel * 4 + const alpha = (decoded.data[at + 3] ?? 255) / 255 + const luminance = (0.299 * (decoded.data[at] ?? 0)) + (0.587 * (decoded.data[at + 1] ?? 0)) + (0.114 * (decoded.data[at + 2] ?? 0)) + image.pixels[pixel] = roundedUp((luminance * alpha) + (255 * (1 - alpha))) + } + + return image +} + +/** + * A gray picture as a PNG. + * + * @param image - The picture. + * @returns The PNG's bytes. + */ +export function pngOfGray (image: GrayImage): Buffer { + const png = new PNG({ width: image.width, height: image.height }) + for (let pixel = 0; pixel < image.width * image.height; pixel += 1) { + const value = image.pixels[pixel] ?? 255 + const at = pixel * 4 + png.data[at] = value + png.data[at + 1] = value + png.data[at + 2] = value + png.data[at + 3] = 255 + } + + return PNG.sync.write(png, FAST_PNG) +} diff --git a/packages/ocr-tesseract/src/recognition/create-ocr-engine.use-case.test.ts b/packages/ocr-tesseract/src/recognition/create-ocr-engine.use-case.test.ts new file mode 100644 index 00000000..e642d529 --- /dev/null +++ b/packages/ocr-tesseract/src/recognition/create-ocr-engine.use-case.test.ts @@ -0,0 +1,85 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { pdfPageOcr, readPdf } from '@opencraw/core' +import type { OcrEngine } from '@opencraw/core' +import { createOcrEngine } from './create-ocr-engine.use-case' + +const fixtures = join(__dirname, '..', '..', '..', 'core', 'src', 'pdf-ocr', 'fixtures') +const scan = (name: string): Uint8Array => new Uint8Array(readFileSync(join(fixtures, name))) + +/** The text that was drawn on every fixture, one line each (spaces between the cells collapse). */ +const TABLE = ['Incentivi giugno 2026', 'Marca Modello Prezzo Sconto', 'Fiat Pandina 15950 12,5%', 'Jeep Avenger 24950 8,0%', 'Totale ordini: 1.250'] + +async function rowsOf (name: string, engine: OcrEngine): Promise { + const document = await readPdf(scan(name), name, { ocr: pdfPageOcr({ engine: () => Promise.resolve(engine) }) }) + + return document.pages[0].rows.map(row => row.cells.map(cell => cell.text).join(' ')) +} + +describe('the Tesseract engine', () => { + let engine: OcrEngine + beforeAll(() => { engine = createOcrEngine() }) + afterAll(async () => { await engine.dispose?.() }) + + it('reads a scanned PDF into the rows that were drawn on it', async () => { + expect(await rowsOf('scan-clean.pdf', engine)).toEqual(TABLE) + }, 60_000) + + it('reads cells where they sit on the page, in points from the bottom left', async () => { + const document = await readPdf(scan('scan-clean.pdf'), 'scan-clean.pdf', { ocr: pdfPageOcr({ engine: () => Promise.resolve(engine) }) }) + + const [page] = document.pages + const brand = page.rows[2].cells.find(cell => cell.text === 'Fiat') + // Drawn at x = 40 pt on a 420 x 595 pt page, the third line down: the engine finds it within a few points. + expect(brand?.x).toBeGreaterThan(36) + expect(brand?.x).toBeLessThan(48) + expect(brand?.y).toBeGreaterThan(595 - 140) + expect(brand?.y).toBeLessThan(595 - 90) + // The cells of a row come left to right, as they are on the page. + for (const row of page.rows) expect(row.cells.map(cell => cell.x)).toEqual(row.cells.map(cell => cell.x).sort((a, b) => a - b)) + }, 60_000) + + it('reads a scan that is tilted and shaded: the plain cleaning reads nothing of it, the second try, with the grain out and the ink black, reads it', async () => { + const rows = await rowsOf('scan-crooked-shaded.pdf', engine) + + expect(rows).toHaveLength(5) + for (const token of ['Incentivi', 'Pandina', '15950', 'Avenger', '24950', 'Totale']) expect(rows.join(' ')).toContain(token) + }, 60_000) + + it('puts cells where they are on the tilted page, not where they would be on a straightened one', async () => { + const document = await readPdf(scan('scan-crooked-shaded.pdf'), 'scan-crooked-shaded.pdf', { ocr: pdfPageOcr({ engine: () => Promise.resolve(engine) }) }) + + const [page] = document.pages + const first = page.rows[0].cells[0] + const last = page.rows.at(-1)?.cells[0] + // The page was fed in 4.5 degrees clockwise: its left margin leans, so the last line starts a few points to the left of the first. On a straightened page both would start at 40. + expect(first.x - (last?.x ?? first.x)).toBeGreaterThan(3) + expect(page.rows[0].top).toBeGreaterThan(page.rows.at(-1)?.top ?? 0) + }, 60_000) + + it('reads one way of cleaning when told one, and nothing from the tilted and shaded scan when told none', async () => { + const once = createOcrEngine({ cleanup: { binarize: 'adaptive', denoise: true } }) + try { + const rows = await rowsOf('scan-crooked-shaded.pdf', once) + expect(rows.join(' ')).toContain('Pandina') + } finally { + await once.dispose?.() + } + + const raw = createOcrEngine({ cleanup: false }) + try { + await expect(rowsOf('scan-crooked-shaded.pdf', raw)).rejects.toThrow('OCR found no text on any page') + } finally { + await raw.dispose?.() + } + }, 60_000) + + it('names a language whose model is not installed, and what to install', async () => { + const italian = createOcrEngine({ languages: ['ita'] }) + try { + await expect(rowsOf('scan-clean.pdf', italian)).rejects.toThrow(/language data for "ita" is not installed: npm install @tesseract\.js-data\/ita/) + } finally { + await italian.dispose?.() + } + }, 60_000) +}) diff --git a/packages/ocr-tesseract/src/recognition/create-ocr-engine.use-case.ts b/packages/ocr-tesseract/src/recognition/create-ocr-engine.use-case.ts new file mode 100644 index 00000000..7b56dcb5 --- /dev/null +++ b/packages/ocr-tesseract/src/recognition/create-ocr-engine.use-case.ts @@ -0,0 +1,110 @@ +import type { OcrEngine, OcrLine, OcrPageImage, OcrWord } from '@opencraw/core' +import type { Block } from 'tesseract.js' +import { cleanPage } from '../page-cleanup' +import type { CleanPage, CleanupOptions } from '../page-cleanup' +import { DEFAULT_LANGUAGE, tesseractLanguages } from './language-codes.policy' +import { TesseractWorkers } from './tesseract-ocr.client' +import type { TesseractOptions } from './tesseract-ocr.client' + +/** How the engine reads pages. */ +export interface OcrEngineOptions extends TesseractOptions { + /** The languages to read when the recipe names none, as codes (`eng`, `ita`) or two-letter ones (`en`, `it`); default English. */ + languages?: readonly string[] + /** + * How each page is cleaned before it is read: one way, `false` for the picture as it is, or a list of ways tried in turn (the first whose reading is + * good enough is kept, else the best). Default: flatten and straighten, and if that reads poorly, take the grain out and binarize. + */ + cleanup?: CleanupOptions | readonly CleanupOptions[] | false +} + +interface Point { + x: number + y: number +} + +/** The ways a page is cleaned, in the order tried: the gentle one reads most scans; one that reads poorly (grain, a stained page) is tried again binarized. */ +const DEFAULT_PASSES: readonly CleanupOptions[] = [{ binarize: 'none' }, { binarize: 'adaptive', denoise: true }] + +/** A reading with this many characters the engine is sure of, and a high average confidence, is good enough to stop: the next way of cleaning is not tried. */ +const GOOD_CHARACTERS = 20 +const GOOD_CONFIDENCE = 85 +/** A word below this confidence is a guess: it does not count towards how good a reading is. */ +const SURE = 60 + +/** How good a reading is: the characters in the words the engine is sure of, and their average confidence. */ +function qualityOf (lines: readonly OcrLine[]): { characters: number, confidence: number } { + const sure = lines.flatMap(line => line.words).filter(word => word.confidence >= SURE) + const characters = sure.reduce((sum, word) => sum + word.text.trim().length, 0) + + return { characters, confidence: sure.length === 0 ? 0 : sure.reduce((sum, word) => sum + word.confidence, 0) / sure.length } +} + +/** A box on the cleaned picture as a box on the original: the corners, put back where they were, and the box around them. */ +function boxOnSource (clean: CleanPage, box: { x0: number, y0: number, x1: number, y1: number }): { x0: number, y0: number, x1: number, y1: number } { + const corners: Point[] = [{ x: box.x0, y: box.y0 }, { x: box.x1, y: box.y0 }, { x: box.x0, y: box.y1 }, { x: box.x1, y: box.y1 }].map(corner => clean.toSource(corner)) + const xs = corners.map(corner => corner.x) + const ys = corners.map(corner => corner.y) + + return { x0: Math.min(...xs), y0: Math.min(...ys), x1: Math.max(...xs), y1: Math.max(...ys) } +} + +/** The lines Tesseract found, with every position put back on the original picture, the baseline's slant included. */ +function linesOf (blocks: readonly Block[], clean: CleanPage): OcrLine[] { + return blocks.flatMap(block => block.paragraphs.flatMap(paragraph => paragraph.lines.map((line): OcrLine => { + const words: OcrWord[] = line.words.map(word => ({ text: word.text, confidence: word.confidence, ...boxOnSource(clean, word.bbox) })) + const { baseline } = line + // A line Tesseract found no baseline for has one with no length. + if (baseline.x1 <= baseline.x0) return { words } + const from = clean.toSource({ x: baseline.x0, y: baseline.y0 }) + const to = clean.toSource({ x: baseline.x1, y: baseline.y1 }) + + return { words, baseline: { x0: from.x, y0: from.y, x1: to.x, y1: to.y } } + }))) +} + +/** The ways to clean a page, as the engine was configured: a list is a list of passes, one way is one pass, `false` is the picture as it is. */ +function passes (cleanup: OcrEngineOptions['cleanup']): readonly (CleanupOptions | false)[] { + if (cleanup === undefined) return DEFAULT_PASSES + if (cleanup === false) return [false] + + return Array.isArray(cleanup) ? cleanup : [cleanup as CleanupOptions] +} + +/** A cleaned page that is the picture as it was given. */ +function untouched (image: OcrPageImage): CleanPage { + return { png: image.png, width: image.width, height: image.height, skewDegrees: 0, toSource: point => point } +} + +/** Reads the page with each way of cleaning it in turn: the first reading that is good enough is kept, else the one with the most characters the engine is sure of. */ +async function readPasses (ways: readonly (CleanupOptions | false)[], image: OcrPageImage, languages: readonly string[], workers: TesseractWorkers): Promise { + let best: { lines: OcrLine[], characters: number } | undefined + for (const way of ways) { + const clean = way === false ? untouched(image) : cleanPage(image.png, way) + const lines = linesOf(await workers.read(clean.png, languages, image.scale * 72), clean) + const quality = qualityOf(lines) + if (quality.characters >= GOOD_CHARACTERS && quality.confidence >= GOOD_CONFIDENCE) return lines + if (best === undefined || quality.characters > best.characters) best = { lines, characters: quality.characters } + } + + return best?.lines ?? [] +} + +/** + * An OCR engine for `@opencraw/core`: each page picture is cleaned (background flattened, lines levelled; if that reads poorly, the grain taken out and + * the ink made black on white), read by Tesseract, and every word handed back where it sits on the picture it was given, whatever the cleaning did to + * the geometry. + * + * @param options - Default languages, cleanup, and where models live. + * @returns The engine. Call `dispose` when done, to stop the Tesseract worker threads. + */ +export function createOcrEngine (options: OcrEngineOptions = {}): OcrEngine { + const workers = new TesseractWorkers(options) + const defaults = tesseractLanguages(options.languages ?? [DEFAULT_LANGUAGE]) + const ways = passes(options.cleanup) + + return { + name: 'tesseract', + recognize: async (image, requested) => await readPasses(ways, image, requested.languages.length > 0 ? tesseractLanguages(requested.languages) : defaults, workers), + dispose: async () => { await workers.close() }, + } +} diff --git a/packages/ocr-tesseract/src/recognition/index.ts b/packages/ocr-tesseract/src/recognition/index.ts new file mode 100644 index 00000000..97f22c6a --- /dev/null +++ b/packages/ocr-tesseract/src/recognition/index.ts @@ -0,0 +1,4 @@ +export { createOcrEngine } from './create-ocr-engine.use-case' +export type { OcrEngineOptions } from './create-ocr-engine.use-case' +export { OcrLanguageError } from './ocr-language.error' +export { tesseractLanguages } from './language-codes.policy' diff --git a/packages/ocr-tesseract/src/recognition/language-codes.policy.ts b/packages/ocr-tesseract/src/recognition/language-codes.policy.ts new file mode 100644 index 00000000..c59b89ec --- /dev/null +++ b/packages/ocr-tesseract/src/recognition/language-codes.policy.ts @@ -0,0 +1,57 @@ +/** The Tesseract code of each two-letter language code a recipe may use, for the languages with a model people ask for. */ +const TESSERACT_CODES: Readonly> = { + ar: 'ara', + bg: 'bul', + ca: 'cat', + cs: 'ces', + da: 'dan', + de: 'deu', + el: 'ell', + en: 'eng', + es: 'spa', + et: 'est', + fi: 'fin', + fr: 'fra', + he: 'heb', + hi: 'hin', + hr: 'hrv', + hu: 'hun', + id: 'ind', + it: 'ita', + ja: 'jpn', + ko: 'kor', + lt: 'lit', + lv: 'lav', + nb: 'nor', + nl: 'nld', + no: 'nor', + pl: 'pol', + pt: 'por', + ro: 'ron', + ru: 'rus', + sk: 'slk', + sl: 'slv', + sv: 'swe', + th: 'tha', + tr: 'tur', + uk: 'ukr', + vi: 'vie', + zh: 'chi_sim', +} + +/** The language a page is read in when nothing says. */ +export const DEFAULT_LANGUAGE = 'eng' + +/** + * The Tesseract languages for the ones asked for: two-letter codes (`it`, `pt`) become Tesseract's (`ita`, `por`), Tesseract's own are kept + * (`eng`, `chi_sim`, `por+eng` as two), lower-cased and without repeats. English when none is asked for. + * + * @param languages - The languages, as a recipe wrote them. + * @returns The Tesseract codes, in the order asked for. + */ +export function tesseractLanguages (languages: readonly string[]): string[] { + const codes = languages.flatMap(language => language.toLowerCase().split('+')).map(code => code.trim()).filter(code => code !== '').map(code => TESSERACT_CODES[code] ?? code) + const unique = [...new Set(codes)] + + return unique.length === 0 ? [DEFAULT_LANGUAGE] : unique +} diff --git a/packages/ocr-tesseract/src/recognition/language-data.client.ts b/packages/ocr-tesseract/src/recognition/language-data.client.ts new file mode 100644 index 00000000..ec8da1c2 --- /dev/null +++ b/packages/ocr-tesseract/src/recognition/language-data.client.ts @@ -0,0 +1,51 @@ +import { copyFileSync, existsSync, mkdirSync } from 'node:fs' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import { basename, dirname, join } from 'node:path' +import { OcrLanguageError } from './ocr-language.error' + +/** The model the data packages keep the small LSTM ("best", quantised) models in: the size of a quick read, the accuracy of the slow one. */ +const MODEL_FOLDER = '4.0.0_best_int' + +/** + * Where a language's model file is: in `langPath` when the caller keeps its own models, else in the `@tesseract.js-data/` package (English + * comes with this package; any other language is one `npm install @tesseract.js-data/` away). + */ +function modelFile (language: string, langPath: string | undefined): string { + if (langPath !== undefined) { + const own = join(langPath, `${language}.traineddata.gz`) + if (!existsSync(own)) throw new OcrLanguageError(`there is no "${language}.traineddata.gz" in ${langPath}`) + + return own + } + const require = createRequire(import.meta.url) + try { + return join(dirname(require.resolve(`@tesseract.js-data/${language}/package.json`)), MODEL_FOLDER, `${language}.traineddata.gz`) + } catch (error) { + throw new OcrLanguageError(`the OCR language data for "${language}" is not installed: npm install @tesseract.js-data/${language}, or give "langPath" a folder with ${language}.traineddata.gz`, { cause: error }) + } +} + +/** + * The folder Tesseract reads the models of these languages from. One language already lies in a folder of its own; several are gathered into one + * (a folder under the system's temp folder, made once and reused) because Tesseract takes a single folder. + * + * @param languages - The Tesseract codes of the languages. + * @param langPath - A folder of the caller's own models, if it keeps any. + * @returns The folder. + * @throws OcrLanguageError for a language with no model installed. + */ +export function languageFolder (languages: readonly string[], langPath: string | undefined): string { + const files = languages.map(language => modelFile(language, langPath)) + if (langPath !== undefined) return langPath + const [first] = files + if (first !== undefined && files.length === 1) return dirname(first) + const folder = join(tmpdir(), 'opencraw-ocr-models', [...languages].sort((a, b) => a.localeCompare(b)).join('+')) + mkdirSync(folder, { recursive: true }) + for (const file of files) { + const target = join(folder, basename(file)) + if (!existsSync(target)) copyFileSync(file, target) + } + + return folder +} diff --git a/packages/ocr-tesseract/src/recognition/ocr-language.error.ts b/packages/ocr-tesseract/src/recognition/ocr-language.error.ts new file mode 100644 index 00000000..6cc00e49 --- /dev/null +++ b/packages/ocr-tesseract/src/recognition/ocr-language.error.ts @@ -0,0 +1,4 @@ +/** A language the page is to be read in has no model installed, with what to install. */ +export class OcrLanguageError extends Error { + override readonly name = 'OcrLanguageError' +} diff --git a/packages/ocr-tesseract/src/recognition/request-ocr.integration.test.ts b/packages/ocr-tesseract/src/recognition/request-ocr.integration.test.ts new file mode 100644 index 00000000..04f12dcd --- /dev/null +++ b/packages/ocr-tesseract/src/recognition/request-ocr.integration.test.ts @@ -0,0 +1,31 @@ +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' +import { HttpClient } from '@opencraw/core' + +const scan = pathToFileURL(join(__dirname, '..', '..', '..', 'core', 'src', 'pdf-ocr', 'fixtures', 'scan-clean.pdf')).href + +describe('a request for a scanned PDF with "ocr"', () => { + it('reads the scan through the installed engine and stops its workers', async () => { + const client = await HttpClient.open() + try { + const response = await client.send({ url: scan, ocr: true }) + + expect(response.format).toBe('pdf') + expect(response.body.kind).toBe('pdf') + const text = response.body.kind === 'pdf' ? response.body.pages.flatMap(page => page.rows).map(row => row.cells.map(cell => cell.text).join(' ')).join('\n') : '' + expect(text).toContain('Totale ordini') + expect(text).toContain('24950') + } finally { + await client.dispose() + } + }, 60_000) + + it('says what is wrong when the request does not ask for OCR', async () => { + const client = await HttpClient.open() + try { + await expect(client.send({ url: scan })).rejects.toThrow('ocr') + } finally { + await client.dispose() + } + }, 60_000) +}) diff --git a/packages/ocr-tesseract/src/recognition/tesseract-ocr.client.ts b/packages/ocr-tesseract/src/recognition/tesseract-ocr.client.ts new file mode 100644 index 00000000..773c5c4a --- /dev/null +++ b/packages/ocr-tesseract/src/recognition/tesseract-ocr.client.ts @@ -0,0 +1,73 @@ +import { createWorker, OEM } from 'tesseract.js' +import type { Block, PSM, Worker } from 'tesseract.js' +import { languageFolder } from './language-data.client' + +/** The page segmentation Tesseract is run with: it finds the blocks and lines itself (`3`), which is what a page of columns, tables and paragraphs needs. */ +const AUTO_SEGMENTATION = '3' + +export interface TesseractOptions { + /** A folder of `.traineddata.gz` files of the caller's own; default: the `@tesseract.js-data` packages. */ + langPath?: string + /** Tesseract's page segmentation mode; default 3, automatic. */ + segmentation?: string +} + +/** + * Tesseract, in worker threads running WebAssembly, one per set of languages and started when a page first needs it. Models come from disk, + * never from a CDN, and nothing is cached in the working directory. Pages queue on their worker. + */ +export class TesseractWorkers { + private readonly workers = new Map>() + + constructor (private readonly options: TesseractOptions = {}) {} + + private async start (languages: readonly string[]): Promise { + const key = languages.join('+') + const existing = this.workers.get(key) + if (existing !== undefined) return await existing + const pending = createWorker([...languages], OEM.LSTM_ONLY, { langPath: languageFolder(languages, this.options.langPath), gzip: true, cacheMethod: 'none' }) + this.workers.set(key, pending) + try { + const worker = await pending + await worker.setParameters({ tessedit_pageseg_mode: (this.options.segmentation ?? AUTO_SEGMENTATION) as PSM, preserve_interword_spaces: '1' }) + + return worker + } catch (error) { + // A failed start is not kept: the next page tries again (a model installed in the meantime is found). + this.workers.delete(key) + throw error + } + } + + /** + * Reads one picture. + * + * @param png - The page, cleaned. + * @param languages - The Tesseract codes of the languages it is written in. + * @param dpi - The picture's resolution: Tesseract sizes its reading by it, and guesses (and says so on the console) when it is not told. + * @returns Tesseract's blocks: paragraphs, lines and words with their boxes and confidence. + */ + async read (png: Uint8Array, languages: readonly string[], dpi: number): Promise { + const worker = await this.start(languages) + const resolution = Math.max(70, Math.round(dpi)) + await worker.setParameters({ user_defined_dpi: String(resolution) }) + const { data } = await worker.recognize(Buffer.from(png), {}, { blocks: true }) + + return data.blocks ?? [] + } + + /** Stops every worker thread. */ + async close (): Promise { + // eslint-disable-next-line unicorn/prefer-iterator-to-array -- this package's lib has no Iterator#toArray + const started = [...this.workers.values()] + this.workers.clear() + for (const pending of started) { + try { + const worker = await pending + await worker.terminate() + } catch { + // A worker that never started has nothing to stop. + } + } + } +} diff --git a/packages/ocr-tesseract/tsconfig.json b/packages/ocr-tesseract/tsconfig.json new file mode 100644 index 00000000..62ebbd94 --- /dev/null +++ b/packages/ocr-tesseract/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.base.json", + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + }, + { + "path": "./tsconfig.spec.json" + } + ] +} diff --git a/packages/ocr-tesseract/tsconfig.lib.json b/packages/ocr-tesseract/tsconfig.lib.json new file mode 100644 index 00000000..12204465 --- /dev/null +++ b/packages/ocr-tesseract/tsconfig.lib.json @@ -0,0 +1,29 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "tsBuildInfoFile": "dist/tsconfig.lib.tsbuildinfo", + "emitDeclarationOnly": true, + "module": "esnext", + "moduleResolution": "bundler", + "forceConsistentCasingInFileNames": true, + "types": [ + "node" + ] + }, + "include": [ + "src/**/*.ts" + ], + "references": [ + { + "path": "../core/tsconfig.lib.json" + } + ], + "exclude": [ + "jest.config.ts", + "jest.config.cts", + "src/**/*.spec.ts", + "src/**/*.test.ts" + ] +} diff --git a/packages/ocr-tesseract/tsconfig.spec.json b/packages/ocr-tesseract/tsconfig.spec.json new file mode 100644 index 00000000..2d74c1fc --- /dev/null +++ b/packages/ocr-tesseract/tsconfig.spec.json @@ -0,0 +1,25 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./out-tsc/jest", + "types": [ + "jest", + "node" + ], + "module": "esnext", + "moduleResolution": "bundler", + "forceConsistentCasingInFileNames": true + }, + "include": [ + "jest.config.ts", + "jest.config.cts", + "src/**/*.test.ts", + "src/**/*.spec.ts", + "src/**/*.d.ts" + ], + "references": [ + { + "path": "./tsconfig.lib.json" + } + ] +} diff --git a/tsconfig.json b/tsconfig.json index 9c27a0f9..274afad5 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -41,6 +41,9 @@ }, { "path": "./apps/vscode-extension" + }, + { + "path": "./packages/ocr-tesseract" } ], "compilerOptions": {}