Standardize naming and use master checkouts for workspace dependencies

.github/workflows/CI.yml

@@ -51,12 +51,15 @@ jobs:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
       - uses: julia-actions/cache@v3
-      # The script resolves KernelSmoothing (master) before FDA (main), also on 1.10.
-      - name: Resolve ecosystem URL dependencies including FDA
-        shell: julia --color=yes {0}
-        run: include(joinpath(ENV["GITHUB_WORKSPACE"], "scripts", "instantiate.jl"))
+      - name: Check naming conventions
+        run: ${{ runner.os == 'Windows' && 'python' || 'python3' }} scripts/check_naming.py
+      - name: Resolve dependencies from master checkouts
+        run: julia --color=yes scripts/instantiate.jl
       - uses: julia-actions/julia-buildpkg@v1
-      - uses: julia-actions/julia-runtest@v1
+      - name: Test against dependency master checkouts
+        run: julia --color=yes scripts/test_master.jl
+        env:
+          JULIA_TEST_COVERAGE: "true"
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v7
         with:

.gitignore

@@ -11,3 +11,9 @@
 # Local development and generated experiment outputs
 .dev/
 results/
+
+# Generated master dependency test environments
+.master-test-env/
+
+# Local Julia-minor environments are regenerated from master checkouts
+/Manifest-v*.toml

CONTRIBUTING.md

@@ -0,0 +1,14 @@
+# Development conventions
+
+Types and modules use UpperCamelCase. Descriptive functions, fields and keyword
+arguments use snake_case; mutating functions end in `!`. Retain standard numerical
+abbreviations, mathematical identifiers and names required by upstream interfaces.
+Run `python3 scripts/check_naming.py` to check new declarations.
+
+Run `julia scripts/instantiate.jl` before tests. Workspace dependencies use sibling
+master checkouts, including transitive dependencies. Missing checkouts are cloned
+from master. Existing checkouts are never reset or silently updated. Package
+version numbering and registry changes are deferred.
+
+Run the complete tests with `julia scripts/test_master.jl`. This uses an isolated
+master-based environment on Julia 1.10 and newer without changing package deps.

Project.toml

@@ -13,10 +13,10 @@ ProximalBase = "0a755509-c5b1-5d6f-9b24-59bcda3c37a3"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 
 [sources]
-CovarianceSelection = {rev = "master", url = "https://github.com/mlakolar/CovarianceSelection.jl"}
-FunctionalDataAnalysis = {rev = "main", url = "https://github.com/mlakolar/FunctionalDataAnalysis.jl"}
-KernelSmoothing = {rev = "master", url = "https://github.com/mlakolar/KernelSmoothing.jl"}
-ProximalBase = {rev = "master", url = "https://github.com/mlakolar/ProximalBase.jl"}
+CovarianceSelection = {path = "../CovarianceSelection.jl"}
+FunctionalDataAnalysis = {path = "../FunctionalDataAnalysis.jl"}
+KernelSmoothing = {path = "../KernelSmoothing.jl"}
+ProximalBase = {path = "../ProximalBase.jl"}
 
 [compat]
 Aqua = "0.8"

README.md

@@ -13,7 +13,7 @@ Estimation of Gaussian graphical models for multivariate functional data:
 The package uses current `master` versions of **ProximalBase** and
 **CovarianceSelection**, with **ProximalSolvers** as a transitive dependency,
 and the registered **BSplineKit** for B-spline bases. **FunctionalDataAnalysis**
-(FDA, `main`) supplies data types, trapezoidal quadrature, dense FPCA and PACE.
+(FDA, `master`) supplies data types, trapezoidal quadrature, dense FPCA and PACE.
 **KernelSmoothing** (`master`) supplies PACE kernels. FDA and KernelSmoothing
 must be registered before FGM can be registered.
 
@@ -23,9 +23,10 @@ Julia 1.10 or newer is required. Resolve ecosystem dependencies explicitly:
 
 ```sh
 julia scripts/instantiate.jl
+julia scripts/test_master.jl
 ```
 
-The generated Manifest records exact dependency commits. The setup script
+The generated Manifest uses the sibling master checkouts. The setup script
 preserves the package Project file. For an existing resolved environment,
 `julia --project -e 'using Pkg; Pkg.instantiate()'` reuses its Manifest.
 
@@ -70,7 +71,7 @@ curves = FDA.MultivariateFunctionalData([
     FDA.GridFunctionalData(values[:, j, :], grid) for j in axes(values, 2)
 ])
 values_copy = FDA.to_array(curves)
-fpca = FDA.fit(FDA.FPCA, curves; basis=:marginal, ncomponents=2, maxK=2)
+fpca = FDA.fit(FDA.FPCA, curves; basis=:marginal, ncomponents=2, max_k=2)
 scores = FDA.score_array(fpca)  # subjects × nodes × components
 heldout_scores = FDA.predict(fpca, curves)  # one matrix per node
 reconstructed = FDA.reconstruct(fpca, heldout_scores)

docs/M4_MIGRATION.md

@@ -21,7 +21,7 @@ The audit follow-up continued from **d7840ffa8095040868edc1eabc4dca0e5693408a**.
   response columns. A validation self-test checks independent component
   sign flips, input preservation, and detection of a coefficient change.
 - **F2:** The qiao2019 experiment requests `basis=:marginal` and
-  `maxK=opts["components"]`, matching the package's fixed truncation.
+  `max_k=opts["components"]`, matching the package's fixed truncation.
 - **F3:** CSV provenance appends `+dirty` to the recorded HEAD when tracked
   or untracked working-tree changes exist. Source and worker SHA-256 digests
   still identify the estimator source and exact comparison worker. The CSV
@@ -37,9 +37,9 @@ does not change package estimator code or expected-value tolerances.
 
 1. FDA PR #16 fixed pooled rank upstream. The refreshed FDA `main` is
    **1b75c014778ad374b30fad70fa9083591e1bb546**, source tree
-   `f3e6290f513a5c4350ffa7b184eee62a423518f2`. Its pooled `maxK` default is
+   `f3e6290f513a5c4350ffa7b184eee62a423518f2`. Its pooled `max_k` default is
    `max(1, min(g-2, p*(n-1)-1))`, with explicit cap `min(g, p*(n-1))`.
-   FGM calls FDA's pooled route directly. The existing `maxK=ncomponents`
+   FGM calls FDA's pooled route directly. The existing `max_k=ncomponents`
    requests the caller's fixed truncation, including the structural rank cap;
    there is no local rank workaround or replacement eigenstep. A regression
    fixture now retains four pooled components from two nodes with only three
@@ -101,7 +101,7 @@ Such a case remains an error; no ridge or pseudoinverse was introduced.
   `SparseFunctionalData`, `FPCAResult`, `fit_fpca`, `transform_fpca`,
   `reconstruct`, `SparseFPCAFit`, `fit_sparse_fpca`. The README lists each.
   The exported `quadrature_weights` is now FDA's function.
-- Project sources specify FDA `main` and KernelSmoothing `master`. CI invokes
+- Project sources use sibling FDA and KernelSmoothing `master` checkouts. CI invokes
   `scripts/instantiate.jl`, which resolves KernelSmoothing before FDA in one
   URL-resolution transaction, including on Julia 1.10. ProximalSolvers stays
   transitive. The script restores Project.toml after **all** Pkg operations,
@@ -112,10 +112,10 @@ Such a case remains an error; no ridge or pseudoinverse was introduced.
 
 | Route | Resolved defaults / required choices | Source and reason |
 | --- | --- | --- |
-| Dense FPCA | marginal for glasso/neighborhood/doubly; pooled for separable; explicit integer `ncomponents`, `maxK=ncomponents`; no presmoothing | FGM estimator definitions and caller truncation; explicit maxK avoids FDA's automatic smaller cap |
+| Dense FPCA | marginal for glasso/neighborhood/doubly; pooled for separable; explicit integer `ncomponents`, `max_k=ncomponents`; no presmoothing | FGM estimator definitions and caller truncation; explicit max_k avoids FDA's automatic smaller cap |
 | Dense eigenstep | trapezoidal quadrature; covariance/eigenvalues `n−1`; `eigen_tol=1e-10`, `sign_tol=1e-12`; mean alignment then largest absolute entry | FDA FPCA and multivariate FPCA docstrings, following fdapace 0.6.0 except documented FDA numerical sign/cutoff rules |
 | Sparse PACE | required work grid and bandwidth; marginal PACE; Gaussian kernel, optional Epanechnikov; `weighting=:subj`; same bandwidth for mean, covariance, diagonal; `binning=nothing` | Preserve FGM settings, rather than fdapace automatic bandwidth/binning defaults; kernels map to KernelSmoothing objects |
-| Sparse spectrum | `maxK=length(grid)`, `eigen_tol=100eps(Float64)`, FDA `sign_tol=1e-12`; all retained positive modes in conditional observation covariance | Preserve FGM full-mode conditional-score algorithm and former relative positive-mode cutoff, with FDA signs |
+| Sparse spectrum | `max_k=length(grid)`, `eigen_tol=100eps(Float64)`, FDA `sign_tol=1e-12`; all retained positive modes in conditional observation covariance | Preserve FGM full-mode conditional-score algorithm and former relative positive-mode cutoff, with FDA signs |
 | Sparse noise | `nothing` estimates; supplied nonnegative scalar or vector passes through per node | Existing FGM interface; adopted FDA middle-half integral using the raw smoothed covariance diagonal, as approved above |
 | Score covariance | `center=true`, `corrected=false` (`1/n`) | Existing FGM likelihood scaling; `corrected=true` selects `1/(n−1)` |
 | P-spline | required `nbasis`; degree 3, penalty 0, second coefficient differences, trapezoidal loss | Existing FGM cubic unpenalized defaults; FDA derivative penalty does not reproduce the objective |
@@ -128,7 +128,7 @@ Such a case remains an error; no ridge or pseudoinverse was introduced.
 | Simulation | n 100, p 50, ncomponents 5, ntime 100, model `:ar2`, noise_sd 0.5, random_diagonal_margin 0.1 | Existing FGM generating-design implementation; settings and caller RNG remain explicit |
 
 The migration adds no new automatic tuning. The additional implementation
-choices are explicit maxK overrides, no sparse binning, retention of FGM's
+choices are explicit max_k overrides, no sparse binning, retention of FGM's
 sparse eigenvalue cutoff, per-candidate tuning fits to preserve valid lower
 ranks, and metadata storage. There are no outstanding author decisions.
 The linked external plan was inaccessible through the browser;
@@ -277,7 +277,7 @@ julia +lts --startup-file=no --project=. validation/m4_compare.jl results/m4-aud
 
 The generated table was copied to `validation/m4_comparison.csv` as the final
 tracked artifact update. Its neighborhood settings now show the estimator's
-own `maxK=2`. All previously reported metric cells remain unchanged; the new
+own `max_k=2`. All previously reported metric cells remain unchanged; the new
 coefficient column measures the neighborhood output that was missing before.
 The full-suite, Aqua, Julia-release, and documentation checks remain the
 controller's responsibility.

experiments/qiao2019.jl

@@ -206,7 +206,7 @@ function run_experiment(opts, arguments)
                     ncomponents=5, ntime=opts["ntime"], model, noise_sd=0.5)
                 smoothed = smooth_bspline(sim.data; nbasis=opts["nbasis"])
                 fpca = FDA.fit(FDA.FPCA,smoothed; method=:dense,basis=:marginal,
-                    ncomponents=opts["components"],maxK=opts["components"])
+                    ncomponents=opts["components"],max_k=opts["components"])
                 bounds = penalty_bounds(smoothed, fpca)
                 empty_graph = falses(opts["p"], opts["p"])
                 complete_graph = trues(opts["p"], opts["p"])

scripts/check_naming.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""Check new Julia declarations; compatibility files and mathematical names are explicit exceptions."""
+import argparse
+from pathlib import Path
+import re
+
+# Matrix caches and scientific notation retain their conventional identifiers.
+MATHEMATICAL = {'A','B','C','D','G','H','I','J','K','L','M','N','P','Q','R','S','T','U','V','W','X','Y','Z',
+                'Ax','At','Bt','Xt','Yt','XtY','XtX','sUtY','Ws','SX','SY','Xn','Yn','SXn','SYn',
+                'Xa','Ya','SXa','SYa','Xr','Yr','posA','posR','T0','F','Sx','Sy','Sxy','Sxx','Syy','AD','Vt','AU','D_calls','Dt_calls','Dx','Dxp','Dty','Dtyp','Bpath'}
+FUNCTION = re.compile(r'^\s*(?:function\s+)([\w.!]+)\s*\(|^\s*([\w.!]+)\([^\n]*\)\s*(?:where\s+[^=]+)?=(?!=)')
+FIELD = re.compile(r'^\s*(\w+)\s*::')
+LEGACY_KEYWORDS = {'printEvery','normD','maxK','requested_maxK','epsTol','maxIter','checkEvery','bootSamples','estimSupport'}
+FACTORIES = {'IndBallLinf','Constrained','_AtomIterate','CDLeastSquaresLoss','CDWeightedLSLoss','CDSqrtLassoLoss','CDSqrtQuadraticLoss','CDQuadraticLoss'}
+TYPE = re.compile(r'^\s*(?:mutable\s+)?(?:struct|abstract\s+type|primitive\s+type)\s+(\w+)')
+
+def check(root):
+    errors=[]
+    types={m.group(1) for path in (root/'src').rglob('*.jl')
+           for line in path.read_text(encoding="utf-8").splitlines() if (m := TYPE.match(line))}
+    types |= {m.group(1) for path in (root/'src').rglob('*.jl') for m in re.finditer(r'^\s*const\s+(\w+)\s*=',path.read_text(encoding="utf-8"),re.M)}
+    for path in sorted((root/'src').rglob('*.jl')):
+        if 'deprecated' in path.stem: continue
+        in_doc=False; in_struct=False
+        lines=path.read_text(encoding="utf-8").splitlines()
+        for number,line in enumerate(lines,1):
+            if line.count('"""') % 2: in_doc=not in_doc; continue
+            if in_doc: continue
+            line=line.split('#',1)[0]
+            typ=TYPE.match(line)
+            if typ:
+                name=typ.group(1)
+                if not name.lstrip('_')[0].isupper():errors.append((path,number,'type',name))
+                in_struct='struct' in line
+                continue
+            if re.match(r'^\s*end\b',line):in_struct=False
+            function=FUNCTION.match(line)
+            if function:
+                name=function.group(1) or function.group(2)
+                # Qualified names implement external interfaces; constructors must name an actual type.
+                if '.' not in name and name not in types and name not in FACTORIES and name not in MATHEMATICAL and re.search('[A-Z]',name):
+                    errors.append((path,number,'function',name))
+                if ';' in line or line.lstrip().startswith('function '):
+                    signature=line[line.find('(')+1:]
+                    depth=1 + signature.count('(')-signature.count(')')
+                    offset=number
+                    while depth>0 and offset<len(lines):
+                        more=lines[offset].split('#',1)[0]
+                        signature+=' '+more
+                        depth+=more.count('(')-more.count(')')
+                        offset+=1
+                    # Only the top level keyword list; ignore nested constructor defaults.
+                    depth=0; keyword=False; segment=''
+                    for char in signature:
+                        if char in '([{': depth+=1
+                        elif char in ')]}':
+                            if depth==0: char=','
+                            else: depth-=1
+                        if char==';' and depth==0:
+                            keyword=True; segment=''; continue
+                        if char==',' and depth==0:
+                            if keyword:
+                                match=re.match(r'\s*(\w+)\s*(?:::[^=]+)?=',segment)
+                                if match:
+                                    key=match.group(1)
+                                    if re.search('[a-z][A-Z]',key) and key not in LEGACY_KEYWORDS:
+                                        errors.append((path,number,'keyword',key))
+                            segment=''
+                        else: segment+=char
+            field=FIELD.match(line) if in_struct else None
+            if field:
+                name=field.group(1)
+                if re.search('[A-Z]',name) and name not in MATHEMATICAL:
+                    errors.append((path,number,'field',name))
+    return errors
+
+if __name__=='__main__':
+    parser=argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('packages',nargs='*',type=Path)
+    args=parser.parse_args()
+    roots=args.packages or [Path(__file__).resolve().parents[1]]
+    errors=[item for root in roots for item in check(root)]
+    for path,line,kind,name in errors: print(f'{path}:{line}: noncanonical {kind}: {name}')
+    if errors: raise SystemExit(1)
+    print(f'Naming declarations checked in {len(roots)} package(s).')

scripts/instantiate.jl

@@ -1,31 +1,6 @@
-# Explicit URL dependencies work on Julia 1.10 as well as newer Pkg versions.
-# Run on the user's compute host; Manifest.toml records exact resolved commits.
-using Pkg, TOML
-root = normpath(joinpath(@__DIR__, ".."))
-Pkg.activate(root)
-deps = TOML.parsefile(joinpath(root, "Project.toml"))["deps"]
-urls = [
-    ("ProximalBase", "https://github.com/mlakolar/ProximalBase.jl", "master"),
-    ("ProximalSolvers", "https://github.com/mlakolar/ProximalSolvers.jl", "master"),
-    ("CovarianceSelection", "https://github.com/mlakolar/CovarianceSelection.jl", "master"),
-    ("KernelSmoothing", "https://github.com/mlakolar/KernelSmoothing.jl", "master"),
-    ("FunctionalDataAnalysis", "https://github.com/mlakolar/FunctionalDataAnalysis.jl", "main"),
-]
-specs = Pkg.PackageSpec[]
-for (name, url, rev) in urls
-    if haskey(deps, name) || (name == "ProximalSolvers" && haskey(deps, "CovarianceSelection"))
-        push!(specs, Pkg.PackageSpec(; name, url, rev))
-    end
-end
-# ProximalSolvers must be resolved alongside its unregistered downstream
-# package, but must not become a stale direct dependency of this package.
-project_path = joinpath(root, "Project.toml")
-original_project = read(project_path, String)
-try
-    isempty(specs) || Pkg.add(specs)
-    Pkg.resolve()
-    Pkg.instantiate()
-finally
-    # Newer Pkg versions can also rewrite sources during resolve/instantiate.
-    write(project_path, original_project)
-end
+# Resolve every workspace dependency from a local master checkout.
+# Missing sibling checkouts are cloned from their master branches.
+include(joinpath(@__DIR__, "master_dependencies.jl"))
+package_root = dirname(@__DIR__)
+environment = isempty(ARGS) ? package_root : abspath(ARGS[1])
+instantiate_master(environment; workspace=dirname(package_root), owner=package_root)