diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..e2804ef8 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,54 @@ +name: Tests + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + name: ${{ matrix.image }} / PostgreSQL 15 + runs-on: ubuntu-22.04 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + image: [Pharo64-13, Moose64-13] + services: + postgres: + image: postgres:15 + env: + POSTGRES_DB: famixngsql_ci + POSTGRES_USER: famixngsql_ci + POSTGRES_PASSWORD: famixngsql_ci + ports: + - 55432:5432 + options: >- + --health-cmd "pg_isready -U famixngsql_ci -d famixngsql_ci" + --health-interval 5s + --health-timeout 5s + --health-retries 12 + env: + FAMIXNGSQL_TEST_HOST: 127.0.0.1 + FAMIXNGSQL_TEST_PORT: '55432' + FAMIXNGSQL_TEST_DATABASE: famixngsql_ci + FAMIXNGSQL_TEST_USER: famixngsql_ci + FAMIXNGSQL_TEST_PASSWORD: famixngsql_ci + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: hpi-swa/setup-smalltalkCI@c0a3a3b942ac515b6c3846ca2f65802d353757ef # v1 + with: + smalltalk-image: ${{ matrix.image }} + - name: Load and test in a clean image + run: smalltalkci -s '${{ matrix.image }}' .smalltalk.ston + - name: Upload JUnit results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ matrix.image }}-test-results + path: FAMIXNGSQL-2-Pharo13-Moose13.xml + if-no-files-found: warn + retention-days: 14 diff --git a/.gitignore b/.gitignore index e43b0f98..61d9ba06 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .DS_Store +artifacts/ diff --git a/.smalltalk.ston b/.smalltalk.ston index 94c2ab51..fd66e1a5 100644 --- a/.smalltalk.ston +++ b/.smalltalk.ston @@ -1,9 +1,19 @@ SmalltalkCISpec { + #name : 'FAMIXNGSQL-2-Pharo13-Moose13', #loading : [ SCIMetacelloLoadSpec { #baseline : 'FAMIXNGSQL', #directory : 'src', + #load : [ 'Tests', 'GeneratorTests' ], + #onConflict : #useIncoming, + #onUpgrade : #useIncoming, #platforms : [ #pharo ] } - ] + ], + #preTesting : 'scripts/ci/configure-database.st', + #testing : { + #packages : [ 'FamixNGSQL-Importer-Tests', 'FAMIXNGSQLMetamodelGenerator-Tests', 'PgMetadata-tests' ], + #defaultTimeout : 120, + #failOnZeroTests : true + } } diff --git a/.travis.yml b/.travis.yml index ddc5dd78..48bd6311 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,27 @@ language: smalltalk -sudo: false - +dist: jammy +os: linux +services: + - docker +smalltalk: + - Pharo64-13 + - Moose64-13 +smalltalk_edge: + source: hpi-swa/smalltalkCI + branch: master +env: + global: + - FAMIXNGSQL_TEST_HOST=127.0.0.1 + - FAMIXNGSQL_TEST_PORT=55432 + - FAMIXNGSQL_TEST_DATABASE=famixngsql_ci + - FAMIXNGSQL_TEST_USER=famixngsql_ci + - FAMIXNGSQL_TEST_PASSWORD=famixngsql_ci +before_script: + - docker run --detach --name famixngsql-postgres --publish 127.0.0.1:55432:5432 --env POSTGRES_DB=famixngsql_ci --env POSTGRES_USER=famixngsql_ci --env POSTGRES_PASSWORD=famixngsql_ci postgres:15 + - bash scripts/ci/wait-for-postgres.sh +script: + - smalltalkci .smalltalk.ston notifications: email: on_success: never on_failure: always - -os: - - linux - - osx - -smalltalk_edge: - source: peteruhnak/smalltalkCI - branch: master - -smalltalk: - - Pharo-7.0 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..ad62985b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,64 @@ +# Changelog + +## 2.0.0 — 2026-09-17 + +This is the Pharo 13 / Moose 13 migration of FAMIXNGSQL. It supersedes the +historical Pharo 7 release; the `v1.0.0` tag remains unchanged. + +### Breaking changes + +- Use `github://deem0n/FAMIXNGSQL:v2.0.0/src` to install the release. Packages use + Tonel under `src`; historical `/repository` loading instructions do not apply. +- Use `FmxSQLModel` in place of the historical `FmxSQLMooseModel`. +- Generated classes and traits target the current Famix infrastructure. Pharo 7, + Pharo 10 and older Moose versions are not supported by this release's tests. +- The default group is `Core`. Historical GT/Telescope/connection-manager UI + packages are isolated in `LegacyUI` and are not validated for this release. +- Regenerated containment and source-anchor relations may differ from older + MSE exports. Cross-version import of historical MSE files is not guaranteed. + +### Model and importer + +- Restore namespace ownership, association direction and source-holder/anchor + relationships on the current metamodel builder. +- Preserve CHECK and exclusion constraint calls to routines and inverse links. +- Integrate updated PgMetadata extraction through upstream P3 code, including + materialized views, partitioned/foreign tables, routine OIDs/languages, + parameter order/modes and trigger ownership on tables or views. +- Read catalogs in a read-only, repeatable-read transaction and close the + metadata connection after extraction. +- Use separate SQL and PL/pgSQL parser entry points. Preserve source when + syntax, language or name resolution is unsupported. +- Record per-object analysis status, diagnostics and timeouts. Ambiguous calls + retain candidate routines; runtime-dependent dynamic SQL targets are not guessed. +- Preserve entity identity, inverse relations and source anchors through MSE + export/import. Export diagnostics separately as JSON. +- Exclude the abstract structural base generator from global regeneration. + +### Tests and continuous integration + +- Share smalltalkCI configuration between Travis CI and GitHub Actions. +- Test both a plain Pharo 13 image and the ready-made Moose 13 / Pharo 13 image + against a disposable PostgreSQL 15 service. +- Run importer, metamodel-generator selection and PgMetadata suites. Fixtures + create unique schemas and clean them up after each scenario. + +### Known limits + +- SQL/PL/pgSQL parsing and semantic analysis are incomplete. The documented `mi` + import retained 70,143 unique entities but still reported 520 parser syntax + failures and additional reference-resolution errors. Consult the analysis + report rather than treating a completed import as complete source coverage. +- The upstream Moose 13 image is a rolling development build. Other runtime + combinations and `LegacyUI` remain unvalidated. +- Some transitive dependencies still follow upstream branches; the dependency + stack is not fully frozen by the FAMIXNGSQL tag alone. +- Package ownership after metamodel regeneration remains tracked in issue #8. + +See [README](README.md) for installation and examples, and the +[migration audit](docs/pharo13-migration-audit.md) for detailed evidence. + +## 1.0.0 — 2019-09-02 + +Historical upstream release for Pharo 7: +[release notes](https://github.com/juliendelplanque/FAMIXNGSQL/releases/tag/v1.0.0). diff --git a/README.md b/README.md index d450eebe..fa137cf6 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,489 @@ -# FAMIXNGSQL [![Build Status](https://travis-ci.org/juliendelplanque/FAMIXNGSQL.svg?branch=master)](https://travis-ci.org/juliendelplanque/FAMIXNGSQL) -A meta-model for SQL databases using FAMIXNG with its generator. +# FAMIXNGSQL -## Install -In a Pharo 7.0 image, open a playground and execute the following script: +[![Tests](https://github.com/deem0n/FAMIXNGSQL/actions/workflows/tests.yml/badge.svg)](https://github.com/deem0n/FAMIXNGSQL/actions/workflows/tests.yml) +FAMIXNGSQL builds a FAMIX model of a PostgreSQL database: schemas, tables, views, +constraints, routines, triggers, source code and references between entities. +Version **2.0.0** includes the Pharo 13 / Moose 13 migration and the updated +PgMetadata and PostgreSQLParser integrations. See the [changelog](CHANGELOG.md) +for breaking changes from `v1.0.0`. + +The import has two stages: + +1. [PgMetadata](https://github.com/deem0n/PgMetadata) reads PostgreSQL catalogs + through [P3](https://github.com/deem0n/P3) and builds a metadata model. +2. FAMIXNGSQL creates `FmxSQL*` entities, parses supported SQL/PL/pgSQL source + with [PostgreSQLParser](https://github.com/deem0n/PostgreSQLParser), and resolves + references into the model. It records incomplete analysis explicitly. + +Catalog extraction and source analysis have different coverage. A successful +import preserves objects and source even when parts of their code cannot be +parsed or resolved. See [the migration audit](docs/pharo13-migration-audit.md) +for the original-model comparison, validation evidence and remaining work. + +## Compatibility and dependencies + +Validated on **2026-09-17**: + +| Component | Tested version or revision | +| --- | --- | +| Pharo | **13.1.0SNAPSHOT** | +| Moose | **13.0.0** | +| PostgreSQL | **15.14** (Postgres.app) | +| PgMetadata | `5abe3134238b4f84f7d1e02e28d9aee45aee5c61`, included in `deem0n/PgMetadata:master` | +| PostgreSQLParser | `f9d1b0850cb7cde2839760d87f54f04ba9de315e`, group `core-no-gui` | +| SymbolResolver | `c6eb29c46cd36cabb8dff5d19329085ee8fa38c7` | +| P3 | `d45f0d358f41ff809fd85f26046a63630a3a7bef`, synchronized with upstream `svenvc/P3` | +| Famix | `4fd41546375e0ab86a4920eff547cc898e22db0b`, the Moose 13 core used for validation | +| PetitParser | `5bdaf9f36793bf355e863c1accfbcc6162a6b034` | + +Validation includes the existing Pharo 13/Moose 13 image and clean CI installs +in **Pharo64-13** and **Moose64-13**, each with PostgreSQL 15. Pharo 7, Pharo 10, +other Moose versions and other PostgreSQL versions are not certified by this +migration. +Historical implementations remain available in Git history and the +[original upstream repository](https://github.com/juliendelplanque/FAMIXNGSQL). + +PgMetadata alone does not require Moose, but the FAMIXNGSQL importer does. +The baseline specifies commit revisions for PgMetadata, PostgreSQLParser, +SymbolResolver, Famix and PetitParser. Upstream baselines still reference +moving branches, including Fame, P3 and other transitive dependencies, so this +is not yet a fully frozen dependency stack. The table records the revisions +observed during validation. +Deleting this repository's merged development branches does not remove the +pinned dependency commits. + +## Installation + +In a Pharo 13 or Moose 13/Pharo 13 Playground: + +```smalltalk +Metacello new + baseline: 'FAMIXNGSQL'; + repository: 'github://deem0n/FAMIXNGSQL:v2.0.0/src'; + load: 'Core'. +``` + +Keep `/src` in this repository URL. The default group also loads `Core`. +Use `:master/src` instead when intentionally testing unreleased development. + +To start with Moose preinstalled, download the official +[Moose 13 / Pharo 13 image](https://github.com/moosetechnology/Moose/releases/download/continuous/Moose13-development-Pharo64-13.zip) +or select it in Pharo Launcher. It is a rolling **development** image, not an +immutable Moose 13 release. smalltalkCI selects it with `Moose64-13`. + +| Group | Purpose | +| --- | --- | +| `Core` | Generated SQL metamodel and importer with their dependencies | +| `Tests` | Core, importer tests and PgMetadata scenario-test support | +| `Generator` | Core and the metamodel generator | +| `GeneratorTests` | Generator plus its global-regeneration selection regression test | +| `LegacyUI` | Historical GT Inspector, analysis, Telescope and connection-manager packages | + +`LegacyUI` has not been ported or validated on the current image. Core usage does +not require those UI packages or a Genie MCP server. Loading modern Genie in a +historical Pharo 7 image is not part of the installation procedure. + +## Connect and build a model + +This example models **all application schemas** in the local `mi` database, +using role `bi` on port `5435`: + +```smalltalk +| connection builder model | +connection := PgConnection + hostname: 'localhost' + port: 5435 + database: 'mi' + user: 'bi' + password: nil. + +builder := FmxSQLModelBuilder new + databaseName: 'mi'; + connection: connection; + analysisTimeout: 5 seconds; + yourself. + +model := builder buildModel. +model inspect. ``` + +Replace the connection parameters for another server and keep both database +names consistent. `password: nil` works with the existing local authentication +used for validation; provide a password when required. It does not automatically +read `.pgpass`. The role must be able to connect and read the required catalogs. + +Metadata extraction uses a **read-only, repeatable-read transaction** and closes +the connection afterwards. Parsing and reference resolution then operate on the +extracted source. The importer does not execute application routine bodies. + +Application schemas exclude names beginning with `pg_` and +`information_schema`. Non-system extension and test schemas, such as `pgtap`, +are included. There is currently no schema-allowlist option on the builder. +System types and routines needed for reference resolution can appear as stubs. + +`analysisTimeout:` limits source analysis **per routine or view**, not the whole +import or its database connection. Five seconds is the default. A large model +can take time to build; raising this value does not add missing parser support. + +## What the model contains + +| Area | Entities and relationships | +| --- | --- | +| Schemas | `FmxSQLNamespace namespaceEntities` and inverse `parentNamespace` | +| Tables | Ordinary, partitioned and foreign tables; columns, types and table inheritance | +| Views | Ordinary and materialized views, columns and original source; `isMaterialized` distinguishes the latter | +| Constraints | Primary/foreign keys, unique, not-null, CHECK and exclusion constraints | +| Constraint calls | CHECK/exclusion `storedProceduresCalled` and inverse routine relations, extracted from catalog dependencies | +| Routines | Functions/procedures, PostgreSQL OIDs, language, source, ordered parameters and parameter modes | +| Triggers | Owning table/view, invoked trigger routine, event/timing metadata and NEW/OLD references where analysis succeeds | +| Source analysis | Queries, clauses, variables, calls and structural references for supported syntax | +| Source anchors | Original source ranges attached to queries, clauses and references | + +`FmxSQLStoredProcedure` is the model's historical name for general routines; it +also represents PostgreSQL functions. Trigger routines specialize it. Names +alone do not identify overloads: retain `postgresOid` and parameter information. +All generated class names use **`FmxSQL`**. A workspace variable referring to a +model of `mi` does not introduce a separate class prefix. + +The model preserves original routine/view source independently of whether the +visitor could build a complete semantic representation. Unsupported languages +remain represented as catalog entities with source. + +## Explore an imported model + +In the same Playground used in [Connect and build a model](#connect-and-build-a-model), +`model` refers to the `FmxSQLModel` returned by `buildModel`. Count selected entity +kinds with: + +```smalltalk +Dictionary newFrom: { + #entities -> model entities size . + #namespaces -> (model allWithType: FmxSQLNamespace) size . + #tables -> (model allWithType: FmxSQLTable) size . + #foreignTables -> (model allWithType: FmxSQLForeignTable) size . + #views -> (model allWithType: FmxSQLView) size . + #routines -> ((model allWithSubTypesOf: FmxSQLStoredProcedure) reject: #isStub) size . + #triggers -> (model allWithType: FmxSQLTrigger) size + } +``` + +Model counts can include synthetic/system stubs; they need not equal a direct +count of application catalog rows. `allWithType:` selects the exact class, +whereas `allWithSubTypesOf:` includes specialized entities. + +The remaining examples in this section run in the model Inspector, where `self` +refers to the inspected model. To run the counting example there, replace `model` +with `self`; do not assign to `self`. + +Inspect routines, including trigger routines, with their source and parameters: + +```smalltalk +((self allWithSubTypesOf: FmxSQLStoredProcedure) reject: #isStub) + collect: [ :routine | + { routine parentNamespace name. + routine name. + routine postgresOid. + routine languageName. + routine source. + (routine parameters sorted: [ :a :b | a position < b position ]) } ]. +``` + +Inspect each trigger's owner and invoked routine: + +```smalltalk +(self allWithType: FmxSQLTrigger) collect: [ :trigger | + { trigger parentNamespace name. + trigger name. + trigger table name. + trigger storedProcedure name } ]. +``` + +The `table` relation also accepts a view, including the owner of an +`INSTEAD OF` trigger. + +### SQL queries and Moose navigation (master after v2.0.0) + +SQL relations use `sqlQuery` so that `query` remains available for Moose's +navigation API: + +| Entity | SQL accessor | Meaning | +| --- | --- | --- | +| `FmxSQLView` | `sqlQuery` / `sqlQuery:` | SELECT query defining the view | +| `FmxSQLDerivedTable` | `sqlQuery` / `sqlQuery:` | Query defining the derived table | +| `FmxSQLClause` and subclasses | `sqlQuery` / `sqlQuery:` | Query containing the clause | +| `FmxSQLCursor` | `sqlQuery` | Single query in the cursor's `queries` collection | + +For example, in a view's Inspector: + +```smalltalk +self sqlQuery. "Parsed SQL definition; can be nil after incomplete analysis" +self query. "MooseQuery navigation object" +self queryLocal: #in with: FmxSQLViewReference. +``` + +This is a breaking change from v2.0.0: update SQL-domain callers of `query` +and `query:`. There are no compatibility aliases or MSE property translations. +The parser AST API is unchanged. The metamodel generator defines the new names, +including the opposite relations, so regeneration preserves them. + +Use a fresh image and rebuild models from PostgreSQL, or explicitly preserve +and restore the old slot values when upgrading a populated image. Older MSE +files that contain a clause's `query` property require migration before import; +new exports use `sqlQuery`. Keep the old image/export until migration is verified. + +### Standard Moose Architectural Map (master after v2.0.0) + +In a Moose 13 image, execute this in the imported model's Inspector: + +```smalltalk +| schemas browser | +schemas := (self allWithType: FmxSQLNamespace) reject: [ :schema | + (schema name beginsWith: 'pg_') or: [ schema name = 'information_schema' ] ]. +browser := MiArchitecturalMapBrowser new. +browser open. +browser followEntity: schemas. +browser beFrozen. +``` + +This uses the standard browser, model, builder and default containment query. +SQL references implement Moose's dependency-query protocol because they can be +contained in routine arguments and expression groups. The map can therefore +discover association types and follow SQL bodies without custom map adapters. +Double-click a schema to expand or collapse it. Freezing keeps the map on the +selected schemas when another Moose browser changes its selection. + +The diagram reflects dependencies present in the imported model; incomplete +source analysis still limits the relationships available to display. The plain +Pharo `Core` installation does not include the Architectural Map UI. + +## Understand analysis coverage + +In the model Inspector: + +```smalltalk +| report | +report := self analysisReport. +{ report at: 'routineCounts'. + report at: 'viewCounts'. + report at: 'errorCount'. + report at: 'warningCount' }. +``` + +Inspect routines requiring further analysis: + +```smalltalk +(self analysisReport at: 'routines') reject: [ :entry | + (entry at: 'status') = 'visited' ]. +``` + +Each routine entry includes its schema, name, OID, language, status and +individual diagnostics. Views have corresponding entries under `views`. + +| Status | Meaning | +| --- | --- | +| `visited` | Parser and visitor finished without a recorded issue; this does not prove all semantics were modeled | +| `partial` | Analysis completed with recorded warnings/errors, including unresolved references or dynamic SQL | +| `failed` | Parsing or visiting failed; the original source remains available | +| `timedOut` | Analysis exceeded the per-entity time limit | +| `unsupportedLanguage` | Routine language has no supported source-analysis path | +| `catalogOnly` | Catalog entity intentionally not analyzed as a routine body, such as an aggregate | +| `untracked` | No analysis status is present, for example on a manually created entity | + +Known limitations include incomplete PostgreSQL grammar coverage, CTE/recursive +query handling, composite/record fields, accurate `search_path` resolution and +routine overload selection. Ambiguous calls retain candidate routines instead +of choosing one arbitrarily. Runtime-dependent dynamic SQL targets are not +guessed. Visitor failures are recorded while the import continues. + +The measured `mi` run produced **70,143 unique entities** and retained **569 +triggers**, **2,461 application routines** and **85 views**. It still had **520 +syntax failures** in the Smalltalk parser. These are not PostgreSQL reporting +invalid stored code. There were also other analysis errors and warnings; syntax +failures alone are not the complete coverage report. See the +[migration audit](docs/pharo13-migration-audit.md) for the dated status breakdown. + +## Export and reload + +Export both the model and its diagnostics from the model Inspector: + +```smalltalk +'mi.mse' asFileReference writeStreamDo: [ :stream | + self exportToMSEStream: stream ]. +'mi-analysis.json' asFileReference writeStreamDo: [ :stream | + stream nextPutAll: (NeoJSONWriter toString: self analysisReport) ]. +``` + +Files are written relative to the image's working directory. Choose new paths +when retaining previous exports. `NeoJSONWriter` is available in the tested +Moose image; the FAMIXNGSQL baseline does not separately declare NeoJSON. + +Reload the model in a Playground with the metamodel and importer loaded: + +```smalltalk +| restored | +restored := 'mi.mse' asFileReference readStreamDo: [ :stream | + FmxSQLModel importFromMSEStream: stream ]. +restored inspect. +``` + +Read the separate report: + +```smalltalk +'mi-analysis.json' asFileReference readStreamDo: [ :stream | + NeoJSONReader fromString: stream contents ]. +``` + +MSE preserves modeled source and relations, but the analysis/error caches are +not serialized. Use the companion JSON to inspect the original import's +coverage; do not expect `restored analysisReport` to reconstruct it. + +The validation export was checked for unique IDs and missing references and +reloaded with matching entity, routine, trigger and anchor counts. Routine/view +source text matched exactly. Database-derived files under `artifacts/` are +ignored by Git and are **not published in this repository**. + +## Tests + +Load the importer tests and their PgMetadata fixture support: + +```smalltalk +Metacello new + baseline: 'FAMIXNGSQL'; + repository: 'github://deem0n/FAMIXNGSQL:v2.0.0/src'; + load: 'Tests'. +``` + +The database scenario performs DDL. Create a **disposable database** in advance +and configure it explicitly; do not use `mi` or another application database: + +```smalltalk +PgScenarioTest connectionParameters: + (PgConnection + hostname: 'localhost' + port: 5435 + database: 'pgmetadata_validation_20260917' + user: 'bi' + password: nil). +``` + +The role must be allowed to create schemas, tables, routines and triggers in +that database. Each scenario creates a UUID-named schema and drops it during +teardown. The database itself must already exist. + +Run all importer test classes: + +```smalltalk +| suite result | +suite := TestSuite named: 'FAMIXNGSQL importer'. +{ FmxSQLSymbolResolutionVisitorTest. + FmxSQLAnalysisReportingTest. + FmxSQLDatabaseImportTest } + do: [ :testClass | suite addTest: testClass suite ]. +result := suite run. +result inspect. +``` + +All **40 importer tests** passed in the validated image. Coverage includes +symbol resolution, overload candidates, a real PostgreSQL schema with triggers +and constraint calls, timeouts, retained source, inverse relations, and MSE +identity/source round trips. Dependency suites previously passed 24 PgMetadata, +95 P3 and 246 PostgreSQLParser tests; see the audit for the validation scope. + +## Continuous integration + +[GitHub Actions](https://github.com/deem0n/FAMIXNGSQL/actions/workflows/tests.yml) +and [.travis.yml](.travis.yml) use the same [.smalltalk.ston](.smalltalk.ston). +Both test `Pharo64-13` and the ready-made `Moose64-13` image on Linux, against a +disposable PostgreSQL 15 container. This is a two-image compatibility matrix; +it does not establish support for other Pharo or Moose versions. + +Each job loads `Tests` and `GeneratorTests` from the checked-out source and runs +**65 tests**: 40 importer, 1 generator-selection and 24 PgMetadata tests. Tests +that need a database always use explicit `FAMIXNGSQL_TEST_*` settings. Missing +settings fail the job instead of falling back to a local/application database. +The container contains no application data and is discarded after the job. + +GitHub Actions runs on pushes, pull requests and manual **Run workflow** +requests. JUnit XML reports are uploaded as artifacts for each image. Travis +requires this repository to be enabled in the owner's Travis account; the YAML +configuration alone does not activate that external service. + +To run the same suite locally with smalltalkCI and an existing disposable +database, set the connection parameters explicitly: + +```sh +export FAMIXNGSQL_TEST_HOST=127.0.0.1 +export FAMIXNGSQL_TEST_PORT=55432 +export FAMIXNGSQL_TEST_DATABASE=famixngsql_ci +export FAMIXNGSQL_TEST_USER=famixngsql_ci +export FAMIXNGSQL_TEST_PASSWORD=famixngsql_ci +smalltalkci -s Moose64-13 .smalltalk.ston +``` + +Substitute your disposable database's settings. `Pharo64-13` selects the plain +image installation path. The passwords in the CI configurations are only for +the disposable service created inside each job. + +## Metamodel development + +Generated entities live in `src/FamixNGSQL`; importer extensions live in +`src/FamixNGSQL-Importer`. Change the generator in +`src/FAMIXNGSQLMetamodelGenerator` when changing the metamodel, then regenerate +and review the generated diff. Keep handwritten importer behavior separate. + +Load the generator: + +```smalltalk +Metacello new + baseline: 'FAMIXNGSQL'; + repository: 'github://deem0n/FAMIXNGSQL:v2.0.0/src'; + load: 'Generator'. +``` + +Construct its definitions without installing regenerated classes: + +```smalltalk +FmxSQLMetamodelGenerator new define. +``` + +`FmxSQLStructuralMetamodelGenerator` is an abstract base, so Moose's global +regeneration selects only the concrete `FmxSQLMetamodelGenerator`. If an older +loaded version reports `SubclassResponsibility` for the base class's `prefix`, +load the updated generator, abandon the failed regeneration, and start it again. +Its already-created generator list still contains the abstract class; resuming +that old operation does not rebuild the list. + +The selection regression can be checked without regenerating classes: + +```smalltalk Metacello new - repository: 'github://juliendelplanque/FAMIXNGSQL/src'; - baseline: 'FAMIXNGSQL'; - load + baseline: 'FAMIXNGSQL'; + repository: 'github://deem0n/FAMIXNGSQL:v2.0.0/src'; + load: 'GeneratorTests'. +FmxSQLMetamodelGeneratorTest suite run inspect. ``` -## Developing the meta-model generator -This section presents some tips and tricks that help in the development of the -meta-model. +Perform actual regeneration in a disposable development image after releasing +model instances. `FmxSQLMetamodelGenerator new generateWithCleaning` removes and +recreates the generated package; reload the handwritten extensions afterwards. +Live migration of a large model can take minutes. Extension-package ownership +across regeneration still needs a regression check; it is tracked in +[issue #8](https://github.com/deem0n/FAMIXNGSQL/issues/8). + +Export generated source to an empty directory and review replacement of the +old generated package. Overlaying an export leaves stale definitions behind. +In particular, `FmxSQLUnknownSourceLanguage` is obsolete: its former +`FamixTUnknownSourceLanguage` trait no longer exists in the tested Moose image. +Do not restore that class merely to silence an old-image load warning. + +The generator preserves namespace ownership, source-holder/anchor direction, +CHECK/exclusion routine links and reciprocal associations. MSE regression tests +must continue checking identity and inverse relations, not only whether an +export can be parsed. + +## License and origins -### Modifying the meta-model generator -If you modified the generator and want to re-generate the meta-model, you have -to: -1. Ensure that no more instances of the meta-model exist in the system. -2. Run `FmxNewSQLMetamodelGenerator class>>#regenerateMetaModel` +Distributed under the [MIT license](LICENSE). This fork builds on +[Julien Delplanque's FAMIXNGSQL](https://github.com/juliendelplanque/FAMIXNGSQL), +with updated metadata extraction, SQL parsing and Moose metamodel integration. diff --git a/docs/pharo13-migration-audit.md b/docs/pharo13-migration-audit.md new file mode 100644 index 00000000..d174279d --- /dev/null +++ b/docs/pharo13-migration-audit.md @@ -0,0 +1,142 @@ +# Pharo 13 / Moose 13 migration audit + +## Reference sources + +- Original upstream cached revision: `d8c6427245ff8fbae6401e956db2b0759bb5a2d0`. +- User fork before Moose migration: `7af7700305360d03f4af5292c3f73418979ec434`. +- Existing incomplete Moose 10 migration: `83d53793d4509965f47d4c12def300a55fd26750`. +- The requested exclusion-constraint/routine relation is an explicit requirement, + even though it is absent from these checked-in behavioural generator versions. + +## Invariants to preserve + +Namespaces own named SQL objects; tables/views own columns; routines own ordered +parameters, local variables and queries; queries own clauses; references retain +source ranges and source/target direction. Trigger entities retain their owning +table or view and the invoked routine. CHECK and exclusion constraints retain +called routines with inverse links from each routine. Unsupported code remains +available as source and failures must be reported, never treated as successful +analysis. + +## Compatibility changes reviewed + +- Mark `FmxSQLStructuralMetamodelGenerator` abstract while keeping its concrete + subclass `FmxSQLMetamodelGenerator` eligible for generation. Otherwise Moose's + global regeneration attempts to instantiate the structural base and signals + `SubclassResponsibility` for `prefix`. The `GeneratorTests` group verifies + selection without installing regenerated classes; its regression failed + before the fix and passed afterwards in the running Pharo 13 image. +- Regenerate the checked-in `FamixNGSQL` package on current Moose. Its old + `FmxSQLUnknownSourceLanguage` definition referenced the removed + `FamixTUnknownSourceLanguage` trait and caused a post-mortem load warning. + Current generation does not emit that class. Remove stale generated files + when exporting; exporting over an existing directory alone leaves them behind. +- Empty `TNamedEntity`, `TWithImmediateSource`, and `TWithAccesses` placeholders + from the Moose 10 migration lost namespace, source and access behaviour. + Restore the namespace relation and use current source/access traits. +- `TSourceAnchor` describes an anchor, not the query/clause holding an anchor. + Restore source-entity behaviour on queries, clauses and reference groups. +- Restore association and entity dependency-query traits. Use `FmxSQLModel` + consistently so source/target metadata comes from the SQL metamodel. +- Remove the duplicate CheckConstraint superclass declaration; it is already + defined by the structural generator. +- Moose 13 no longer has `withoutPrimaryContainer`. Preserve both relation + endpoints and composition through the existing `<>-`/`<>-*` commands. + There is no replacement that disables the relation or drops its inverse. + For the one-to-one view/query and derived-table/query relations, explicitly + mark the owner query property derived and its container inverse not derived. + This preserves composition while exporting each query once; retaining the + default direction produces duplicate MSE IDs. +- Use a trigger-owner trait shared by tables and views, preserving the public + `table` / `triggers` relation names. +- Keep the CHECK/routine relation and add the requested exclusion/routine + relation. Populate both from catalog dependencies, including exclusion index + expressions, with OID-based identity. +- Preserve materialized views and source definitions, routine OIDs/languages, + unnamed parameters, argument order/modes, and SQL-standard routine bodies. +- Dispatch SQL and PL/pgSQL bodies to their respective grammar entry points. + Parsing must preserve original source offsets, including semicolons in strings. + +## Validation scope + +Tests and live coverage results are recorded separately. Passing structural tests +is not proof of complete parsing of real database source. A full-model build must +report failed/partial analysis and retain the corresponding original source. + +## Measured validation (2026-09-17) + +Validated in the running Pharo 13 / Moose 13 image against PostgreSQL 15.14. +P3 master was fast-forwarded to upstream `d45f0d358f41ff809fd85f26046a63630a3a7bef` +and pushed; its 95 tests passed against a disposable database. PgMetadata's 24 +tests passed, including constraint/routine links and the read-only extraction +transaction. PostgreSQLParser passed 133 grammar tests, 103 AST-builder tests, +7 facade tests, and 3 AST tests. The importer has 40 passing tests (including a +real PostgreSQL integration scenario and MSE source-anchor export). + +The regenerated metamodel was exported to an empty directory and reloaded +successfully. No generated definition refers to `FamixTUnknownSourceLanguage`. +The production importer and its tests now have separate packages. + +The final model contains 70,143 unique entities. The earlier raw count included +duplicate registrations and has been superseded. Application catalog counts +were independently checked with SQL: 47 schemas, 359 tables, 5 foreign +tables, 85 ordinary/materialized views, 2,461 routines, and 569 triggers. +Generated system stubs are additional model entities. + +Routine statuses: 937 visited, 286 partial, 718 failed, 7 timed out, 511 in +unsupported languages, and 2 aggregate catalog objects. View statuses: 42 +visited, 12 partial, 31 failed. Syntax failures across routines/views fell from +799 to 520 after the schema-qualified call, declaration, and EXECUTE repairs. +The larger model exposes more unresolved references; there are 1,836 recorded +errors and 347 warnings overall. These counts are observations, not a claim of +complete PostgreSQL semantics or dependency coverage. + +The MSE artifact retains source and structural/reference relations. A separate +JSON report records per-entity status and diagnostics. Local database artifacts +are ignored by Git. Legacy UI validation has not been performed. Fresh-image +validation was added during release preparation; see the CI section below. + +## Remaining work + +- CTEs, recursive queries, set-returning functions, and other modern PostgreSQL + syntax still exceed the old grammar/visitor in many cases. +- Resolve unqualified object names with the actual PostgreSQL search_path and + overload rules. Calls currently retain all matching overload candidates. +- Model composite/record type fields and more PL/pgSQL statement semantics. +- Analyse constant dynamic SQL where possible; runtime-dependent SQL is reported + explicitly and its targets are not guessed. +- Remove remaining visitor Halts with supported semantics and regression tests; + the importer currently records them as failures and continues. +- Freeze the remaining transitive dependency branches. Clean-image validation + now passes on the two CI targets described below. + +## Export integrity fixes + +Reference factories already register their entities in the model. Remove the +second registration in the visitor and in trigger-column linking. Register +synthetic tableoid columns and copied trigger source anchors as model members. +A final relation-graph pass also includes entities linked before a visitor failed. +Round-trip tests check query identity, inverse relationships, entity counts, and +source-anchor text, beyond checking that the MSE parser accepts the file. + +Final MSE verification: 70,143 records, 70,143 distinct IDs, zero undefined +reference IDs. Reloaded model counts match: 70,143 entities, 569 triggers, +2,461 application routines, 18,457 source anchors, and 13,386 anchored references. +Routine and view source text was checked for exact equality after reloading. +The snapshots are `artifacts/mi-20260917.mse` and +`artifacts/mi-20260917-analysis.json` in the local working repository. + +## Release CI validation (2026-09-17) + +GitHub Actions run [35272163591](https://github.com/deem0n/FAMIXNGSQL/actions/runs/35272163591) +loaded the checkout in both plain `Pharo64-13` and official prebuilt +`Moose64-13` images. Both jobs passed all 65 tests: 40 importer tests, the +generator-selection regression and 24 PgMetadata tests, against PostgreSQL 15 +in disposable containers. This supersedes the earlier fresh-image validation +gap; it does not extend the supported matrix to older Pharo/Moose versions. + +Travis and GitHub Actions share `.smalltalk.ston`. Travis execution requires +the repository's external Travis integration to be enabled separately. + +The first CI attempt passed the tests but failed writing JUnit XML because the +suite name contained a slash. The suite name is now filesystem-safe. diff --git a/scripts/ci/configure-database.st b/scripts/ci/configure-database.st new file mode 100644 index 00000000..7f668e2d --- /dev/null +++ b/scripts/ci/configure-database.st @@ -0,0 +1,22 @@ +"Only explicit CI settings are accepted: never fall back to an application database." +| environment required value | +(Smalltalk version beginsWith: 'Pharo13.') ifFalse: [ + self error: 'This CI target requires Pharo 13' ]. +Smalltalk globals at: #MooseVersion ifPresent: [ :versionClass | + versionClass current major asString = '13' ifFalse: [ + self error: 'The prebuilt Moose CI target requires Moose 13' ] ]. +environment := Smalltalk os environment. +required := [ :name | + value := environment at: name ifAbsent: [ self error: 'Missing CI setting: ', name ]. + value ifEmpty: [ self error: 'Empty CI setting: ', name ]. + value ]. +PgScenarioTest connectionParameters: + (PgConnection + hostname: (required value: 'FAMIXNGSQL_TEST_HOST') + port: (required value: 'FAMIXNGSQL_TEST_PORT') asInteger + database: (required value: 'FAMIXNGSQL_TEST_DATABASE') + user: (required value: 'FAMIXNGSQL_TEST_USER') + password: (required value: 'FAMIXNGSQL_TEST_PASSWORD')). +Transcript + show: 'Runtime: ', Smalltalk version; cr; + show: 'Database scenarios configured for a disposable CI database.'; cr. diff --git a/scripts/ci/wait-for-postgres.sh b/scripts/ci/wait-for-postgres.sh new file mode 100644 index 00000000..7af7ed83 --- /dev/null +++ b/scripts/ci/wait-for-postgres.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +for attempt in {1..60}; do + if docker exec famixngsql-postgres pg_isready --username=famixngsql_ci --dbname=famixngsql_ci; then + exit 0 + fi + sleep 1 +done + +docker logs famixngsql-postgres +echo 'PostgreSQL did not become ready within 60 seconds.' >&2 +exit 1 diff --git a/src/BaselineOfFAMIXNGSQL/BaselineOfFAMIXNGSQL.class.st b/src/BaselineOfFAMIXNGSQL/BaselineOfFAMIXNGSQL.class.st index af320210..e7862502 100644 --- a/src/BaselineOfFAMIXNGSQL/BaselineOfFAMIXNGSQL.class.st +++ b/src/BaselineOfFAMIXNGSQL/BaselineOfFAMIXNGSQL.class.st @@ -12,9 +12,13 @@ BaselineOfFAMIXNGSQL >> baseline: spec [ do: [ spec package: #FAMIXNGSQLMetamodelGenerator with: [ spec requires: #('Famix') ]; + package: #'FAMIXNGSQLMetamodelGenerator-Tests' + with: [ spec requires: #('FAMIXNGSQLMetamodelGenerator') ]; package: #'FamixNGSQL-Importer' with: [ spec - requires: #('PostgreSQLParser' 'PgMetadata' 'SymbolResolver' 'PetitParser') ]; + requires: #('FamixNGSQL' 'PostgreSQLParser' 'PgMetadata' 'SymbolResolver' 'PetitParser') ]; + package: #'FamixNGSQL-Importer-Tests' + with: [ spec requires: #('FamixNGSQL-Importer' 'PgMetadata-Tests') ]; package: #'FamixNGSQL-GTInspectorExtension' with: [ spec requires: #('FamixNGSQL-Importer') ]; package: #FAMIXNGSQLAnalysis @@ -25,7 +29,7 @@ BaselineOfFAMIXNGSQL >> baseline: spec [ spec baseline: 'Famix' with: [ spec - repository: 'github://jecisc/Famix:v4.x.x/src'; + repository: 'github://moosetechnology/Famix:4fd41546375e0ab86a4920eff547cc898e22db0b/src'; loads: #('Core' 'Basic') ]. spec baseline: 'SymbolResolver' @@ -33,16 +37,20 @@ BaselineOfFAMIXNGSQL >> baseline: spec [ spec baseline: 'PostgreSQLParser' with: [ spec - repository: 'github://juliendelplanque/PostgreSQLParser/src'; - loads: #('core') ]. + repository: 'github://deem0n/PostgreSQLParser:f9d1b0850cb7cde2839760d87f54f04ba9de315e/src'; + loads: #('core-no-gui') ]. spec baseline: 'PetitParser' with: [ spec loads: #('Preprocessor'); - repository: 'github://moosetechnology/PetitParser/src' ]. + repository: 'github://moosetechnology/PetitParser:5bdaf9f36793bf355e863c1accfbcc6162a6b034/src' ]. spec baseline: 'PgMetadata' - with: [ spec repository: 'github://olivierauverlot/PgMetadata' ]. + with: [ spec + repository: 'github://deem0n/PgMetadata:5abe3134238b4f84f7d1e02e28d9aee45aee5c61'; + loads: #('Core') ]. + spec project: 'PgMetadata-Tests' copyFrom: 'PgMetadata' + with: [ spec loads: #('Tests') ]. spec baseline: 'DBConnectionsManager' with: [ spec @@ -51,5 +59,12 @@ BaselineOfFAMIXNGSQL >> baseline: spec [ spec baseline: 'TelescopeCytoscape' with: [ spec - repository: 'github://TelescopeSt/TelescopeCytoscape:v2.x.x/src' ] ] + repository: 'github://TelescopeSt/TelescopeCytoscape:v2.x.x/src' ]. + spec + group: 'default' with: #('Core'); + group: 'Core' with: #('FamixNGSQL' 'FamixNGSQL-Importer'); + group: 'Generator' with: #('Core' 'FAMIXNGSQLMetamodelGenerator'); + group: 'GeneratorTests' with: #('Generator' 'FAMIXNGSQLMetamodelGenerator-Tests'); + group: 'Tests' with: #('Core' 'FamixNGSQL-Importer-Tests'); + group: 'LegacyUI' with: #('Core' 'FamixNGSQL-GTInspectorExtension' 'FAMIXNGSQLAnalysis' 'FAMIXNGSQLAnalysis-ConnectionsManager') ] ] diff --git a/src/FAMIXNGSQLMetamodelGenerator-Tests/FmxSQLMetamodelGeneratorTest.class.st b/src/FAMIXNGSQLMetamodelGenerator-Tests/FmxSQLMetamodelGeneratorTest.class.st new file mode 100644 index 00000000..4cc2d201 --- /dev/null +++ b/src/FAMIXNGSQLMetamodelGenerator-Tests/FmxSQLMetamodelGeneratorTest.class.st @@ -0,0 +1,17 @@ +Class { + #name : #FmxSQLMetamodelGeneratorTest, + #superclass : #TestCase, + #category : #'FAMIXNGSQLMetamodelGenerator-Tests' +} + +{ #category : #tests } +FmxSQLMetamodelGeneratorTest >> testGlobalRegenerationSkipsStructuralBase [ + | generators | + self assert: FmxSQLStructuralMetamodelGenerator isAbstract. + self deny: FmxSQLMetamodelGenerator isAbstract. + generators := FamixMetamodelGenerator metamodelsToGenerate. + self deny: (generators includes: FmxSQLStructuralMetamodelGenerator). + self assert: (generators includes: FmxSQLMetamodelGenerator). + self assert: FmxSQLMetamodelGenerator prefix equals: #FmxSQL. + self assert: FmxSQLMetamodelGenerator packageName equals: #FamixNGSQL +] diff --git a/src/FAMIXNGSQLMetamodelGenerator-Tests/package.st b/src/FAMIXNGSQLMetamodelGenerator-Tests/package.st new file mode 100644 index 00000000..b5c21544 --- /dev/null +++ b/src/FAMIXNGSQLMetamodelGenerator-Tests/package.st @@ -0,0 +1 @@ +Package { #name : #'FAMIXNGSQLMetamodelGenerator-Tests' } diff --git a/src/FAMIXNGSQLMetamodelGenerator/FmxSQLMetamodelGenerator.class.st b/src/FAMIXNGSQLMetamodelGenerator/FmxSQLMetamodelGenerator.class.st index 92662a2c..1f6a80ae 100644 --- a/src/FAMIXNGSQLMetamodelGenerator/FmxSQLMetamodelGenerator.class.st +++ b/src/FAMIXNGSQLMetamodelGenerator/FmxSQLMetamodelGenerator.class.st @@ -67,7 +67,10 @@ Class { 'tWithEntitySourceAnchor', 'tDefiningDerivedTables', 'qualifiedReference', - 'tWithQualifiedReference' + 'tWithQualifiedReference', + 'tWithImmediateSource', + 'tWithAccesses', + 'tWithTriggers' ], #category : #FAMIXNGSQLMetamodelGenerator } @@ -77,18 +80,18 @@ FmxSQLMetamodelGenerator class >> generate [ "This method is just there to make regeneration of the MM easy."