diff --git a/products/cscl/dbt_project.yml b/products/cscl/dbt_project.yml index ed74ce5e4e..ee26409f05 100644 --- a/products/cscl/dbt_project.yml +++ b/products/cscl/dbt_project.yml @@ -42,5 +42,46 @@ models: on-run-start: - '{{ create_pg_functions() }}' +vars: + # qa__context_geoms: buffer applied to the SUBTRACTED side before computing + # build_minus_legacy/legacy_minus_build, to separate real spurs/gaps from needle-sliver + # noise - NOT a proxy for perimeter/length. Confirmed on PUMA 3808 (chat log 2026-10-06): + # its tract-dissolve needle noise hugs the true boundary within 0.004 ft even at 2,838 ft + # long (perimeter is not a usable signal - some noise needles are LONGER than the real + # defect below), while its one genuine self-touching-vertex spur reaches 104.5 ft from + # the true boundary despite a smaller, 501 ft perimeter. A 1 ft buffer fully absorbs the + # former (needle max gap is two orders of magnitude under 1 ft) while only trimming the + # base of the latter, leaving its real extent visible. + context_geom_noise_buffer_ft: 1.0 + # AtomicPolygon topology (models/intermediate/atomicpolygon_topology) and the district + # boundaries built from it (macros/district_boundary.sql). + # + # Node merge tolerance (ft): two independently digitized copies of the same corner + # differ by small multiples of the source's 2^-14 ft quantization (observed up to + # 0.000307 ft). 2 x quantum covers every observed case; ST_DWithin measures true + # distance, so this also covers a diagonal offset of one quantum per axis. + ap_topology_node_merge_tolerance_ft: 0.0003662109375 + # Scope: an AtomicPolygon is in scope if its 2010 tract's PUMA is in + # ap_topology_scope_pumas OR its borocode is in ap_topology_scope_boros. Citywide by + # default (full rebuild ~3 min); narrow it when iterating on vertices/nodes/edges. + # Must be a union of whole districts for any district type compared against prod. + ap_topology_scope_pumas: [] + ap_topology_scope_boros: ['1', '2', '3', '4', '5'] + # Orphan edge splitting (int__topology__edges): an alternate path may replace a + # one-owner edge only if its length matches the direct distance within this + # tolerance - by the triangle inequality, i.e. only if every intermediate vertex is + # collinear. Observed real cases are 0.00001-0.00006 ft off the chord; real lot-line + # kinks are far larger. + ap_topology_edge_split_tolerance_ft: 0.001 + # Max edges in an alternate path. The length budget prunes the search on its own; this + # only guarantees termination. Real cases seen need up to 3. + ap_topology_edge_split_max_hops: 6 + # QA (models/etl_dev_qa/district_boundaries): floors for qa__boundary__*_outliers, and + # the vertex-count product above which GEOS Hausdorff (O(n*m), uninterruptible) is + # skipped - 5e7 is a few seconds; whole fire divisions are ~1e10. + boundary_qa_outlier_min_hausdorff_ft: 5 + boundary_qa_outlier_min_symdiff_pct: 0.01 + boundary_qa_hausdorff_max_vertex_product: 50000000 + flags: fail-fast: true diff --git a/products/cscl/macros/clip_to_shoreline.sql b/products/cscl/macros/clip_to_shoreline.sql deleted file mode 100644 index 115e928996..0000000000 --- a/products/cscl/macros/clip_to_shoreline.sql +++ /dev/null @@ -1,100 +0,0 @@ -{% macro clip_to_shoreline(geom_column='d.geom') %} -{#- - Join clause pairing a district layer with the water overlapping each feature, for - the "clipped to shoreline" half of the district boundary gdb. Use with - `clipped_geom` in the select list, which does the actual subtraction. - - LEFT JOIN so features that touch no water survive with no water geometry, and - LATERAL so the union covers only the pieces the feature actually intersects - rather than the whole city. --#} -LEFT JOIN LATERAL ( - -- gridSize=0.001 (PostGIS's precision-aware overlay path, snaps during the - -- union's internal noding rather than only rounding inputs beforehand) - same - -- overlay-robustness fix as int__water_mask's own union, applied again here - -- since this is a second, independent union over whatever subset of mask - -- pieces intersect this specific feature. See int__water_mask.sql's comment - -- and chat log 2026-10-03 for the measurements behind this value. - SELECT st_union(m.geom, 0.001) AS geom - FROM {{ ref('int__water_mask') }} AS m - WHERE st_intersects({{ geom_column }}, m.geom) - ) AS water ON TRUE -{%- endmacro %} - - -{% macro clipped_geom(geom_column='d.geom', min_area=100) %} -{#- - The district geometry with water removed, discarding parts below `min_area` - and, - within each surviving part, discarding interior rings (holes) below the same floor. - - District edges and atomic polygon edges are nominally coincident but differ in the - last bits, so subtracting water leaves hairline slivers along them — without the - filter the difference produces ~109k sub-square-foot fragments across nymcea alone. - Part areas fall into two clearly separated groups, nothing between 0.83 and 167 - square feet, and a 100 sq ft floor reproduces prod's row counts exactly on every - affected layer (nycb2010/2020, nyct2010/2020, nyed, nypuma2020). - - The same slivering shows up *within* a single surviving part as sub-square-foot - interior rings, not just as separate top-level parts - e.g. PUMA 4105 (nypuma2010) - differences to one real part plus 4,926 "holes", every single one under 0.24 sq ft - (see CSCL-DISTRICTS-01, docs/prod_bugs). Untouched, GDAL's OpenFileGDB writer/reader - round-trip can't unambiguously reconstruct a polygon with that many rings (its - organizePolygons() heuristic, meant for formats with no explicit ring/hole - structure, kicks in above 100 parts and misreads some holes as separate exterior - parts) - inflating the exported feature's part count citywide even though the - underlying data was always correct. Stripping sub-`min_area` holes here, the same - way sub-`min_area` whole parts are already stripped below, removes the slivers - before they ever reach export - confirmed on PUMA 4105: 4,926 holes -> 0, area - changes by 6.9 sq ft (0.0000013%). - - Features left with no qualifying part come back empty rather than NULL, so callers - can drop them with `WHERE NOT st_isempty(geom)`. - - gridSize=0.001 on the difference itself (on top of the gridSize already applied to - `water.geom`'s own union in `clip_to_shoreline`) measurably reduces the hole count - further on top of that - confirmed on PUMA 4105: the union fix alone already took - 4,609 holes down to 271 differencing against the fixed mask, and adding gridSize - here too brought it to 218 (chat log 2026-10-03). Both fixes are needed; neither - alone gets all the way there - see that log for the still-unexplained residual. - - The precision-aware (gridSize) overlay path is stricter than the classic one about - its inputs: it throws "Overlay input is mixed-dimension" instead of just tolerating - it when `geom_column` isn't purely polygonal - which `st_intersection(...)` callers - (e.g. gdb_nysd's District 10 split, two polygons that only graze at a shared - boundary) can legitimately produce, as a GeometryCollection with stray points/lines - mixed into the polygon part. The classic difference silently handled this; the new - one errors immediately, before this macro's own st_collectionextract downstream ever - gets a chance to clean it up (confirmed broke the real build, chat log 2026-10-04). - Fixed by running geom_column through st_collectionextract(st_multi(...), 3) - keep - only the polygonal component - before it ever reaches the difference call. --#} -st_multi(coalesce( - ( - SELECT st_union(kept.geom) - FROM ( - SELECT (st_dump(st_collectionextract( - st_difference( - st_collectionextract(st_multi({{ geom_column }}), 3), - coalesce(water.geom, st_setsrid('POLYGON EMPTY'::geometry, st_srid({{ geom_column }}))), - 0.001 - ), 3 - ))).geom AS part_geom - ) AS parts - CROSS JOIN LATERAL ( - SELECT st_makepolygon( - st_exteriorring(parts.part_geom), - coalesce( - array_agg(hole.ring) FILTER (WHERE st_area(st_makepolygon(hole.ring)) >= {{ min_area }}), - ARRAY[]::geometry[] - ) - ) AS geom - FROM ( - SELECT st_interiorringn(parts.part_geom, gs) AS ring - FROM generate_series(1, st_numinteriorrings(parts.part_geom)) AS gs - ) AS hole - ) AS kept - WHERE st_area(parts.part_geom) >= {{ min_area }} - ), - st_setsrid('MULTIPOLYGON EMPTY'::geometry, st_srid({{ geom_column }})) - )) -{%- endmacro %} diff --git a/products/cscl/macros/create_pg_functions.sql b/products/cscl/macros/create_pg_functions.sql index 9a75cf808a..79a61118c7 100644 --- a/products/cscl/macros/create_pg_functions.sql +++ b/products/cscl/macros/create_pg_functions.sql @@ -209,4 +209,190 @@ END; $$ LANGUAGE plpgsql; +-------------------------------------------------------------------------------------------- +-- test_cases.geometry_diff_metrics: a robustness-focused geometry comparison for the +-- test_cases.examples TDD table (see products/cscl chat log, 2026-10-04), complementing +-- plain SHAPE_Length/SHAPE_Area diffing (qa__diffs_fgdb_* models). +-- +-- Deliberately schema-qualified to test_cases rather than created bare: test_cases is a +-- stable, shared schema in db-cscl, never touched by a branch's own build/recipe-reload +-- (which clears the *branch* schema, not this one - the mechanism that wiped the first, +-- ad hoc version of this work, see that chat log). Creating it here, inside the same +-- on-run-start hook every branch's `dbt build` already runs, means the function +-- redefines itself fresh on every build regardless of branch - no separate manual step, +-- and it can never again go stale or missing the way a one-off psql session's objects +-- did. +-- +-- Inputs may be in any SRID (stored examples use 4326) - both are transformed to 2263 +-- internally so every output is in real feet/sqft, not degrees. +-- +-- expected may be NULL (a pure build-side artifact with no prod counterpart, e.g. a +-- GEOS overlay-robustness sliver with nothing on the prod side to compare against) - the +-- three expected-comparison columns come back NULL in that case; max_ring_eccentricity +-- still computes from actual alone. +-- +-- Metrics, and why there are three instead of one: +-- symdiff_area_pct - % of actual's own area that disagrees with expected. +-- Catches real coverage gain/loss (a clipped-off pier, a +-- spurious extra chunk of land) - the obvious "do these +-- shapes basically match" check. +-- hausdorff_ft - max boundary deviation between actual and expected. +-- symdiff_boundary_len_ft - perimeter of the symmetric-difference region. +-- Both catch jagged/thin boundary defects that barely move +-- enclosed area but wildly inflate perimeter - confirmed +-- empirically on nycb2010 block 30666000002 (chat log +-- 2026-10-04): symdiff_area_pct was only 0.15% (invisible) +-- despite a 15x real SHAPE_Length inflation, while +-- hausdorff_ft (337.57) and symdiff_boundary_len_ft (770.31) +-- both caught it clearly. symdiff_area_pct alone is NOT +-- sufficient - a thread-thin sliver barely changes enclosed +-- area no matter how long or jagged it is. +-- max_ring_eccentricity - length / sqrt(area) of the single worst ring (interior or +-- exterior, across all parts) in actual. Needs no expected +-- geometry at all - catches structurally degenerate output +-- (needle-shaped holes, GEOS overlay artifacts) on its own +-- terms. A real, compact polygon stays in the single digits +-- to low tens; the PUMA 4105 needle sliver measured 2885.76 +-- (chat log 2026-10-03). Note this is ST_Length, not +-- ST_Perimeter - ST_ExteriorRing/ST_InteriorRingN return a +-- LINESTRING, and ST_Perimeter is 0 by definition on +-- anything non-areal (this silently broke the first version +-- of this function - caught immediately because the needle +-- test case scored 0 instead of ~2886, exactly the kind of +-- bug this TDD setup exists to catch). +-- perimeter_pct_diff - 100 * (actual's ST_Perimeter - expected's) / expected's, +-- raw and unfiltered - directly mirrors the SHAPE_Length +-- comparison compare_gdb.py/qa_int__prod_fgdb_* already do at +-- the scalar-field level (CSCL-DISTRICTS-03), now available +-- inside this framework too. Needed as its own metric, not +-- redundant with the others: symdiff_area_pct/ +-- symdiff_boundary_len_ft only sum fragments >= +-- min_fragment_sqft (100 by default in production use via +-- geometry_diff_qa), and every needle-sliver fragment found +-- so far is under a few sqft - so in real use those two +-- metrics are structurally blind to exactly this defect +-- class. hausdorff_ft catches it sometimes but isn't +-- reliable either: confirmed on PUMA 3701 (chat log +-- 2026-10-05), the same underlying defect (a single needle +-- inflating SHAPE_Length 22%) swung hausdorff_ft from 2,330 +-- ft down to 0.05 ft under clip_to_shoreline_gridsize=0.005, +-- purely from grid-alignment luck, not a real change in how +-- wrong the perimeter was before that fix. perimeter_pct_diff +-- measures the one thing none of the others do directly: is +-- the actual boundary length itself way off, independent of +-- fragment-area filtering or Hausdorff's own luck. +-- +-- symdiff_area_pct/symdiff_boundary_len_ft only sum symmetric-difference fragments +-- >= min_fragment_sqft (default 100, matching clipped_geom()'s own min_area floor). +-- Caught on nyfb/FireBN 43 (chat log 2026-10-05): the raw, unfiltered symmetric +-- difference against prod decomposed into 5,049 fragments, median area 0.0014 sqft - +-- pervasive hairline noise along a long, independently-digitized coastline, not real +-- defects (same mechanism as CSCL-DISTRICTS-03 generally) - summed together these +-- pushed symdiff_boundary_len_ft past 300,000 ft, swamping the one fragment that +-- actually mattered (91,531 sqft, a real discrepancy). Filtering first means these +-- two metrics reflect substantial disagreement only. +-- +-- hausdorff_ft runs ST_Simplify (tolerance 0.05 ft, far below anything real this +-- project cares about) on both inputs first - ST_HausdorffDistance's cost scales with +-- vertex count, and a battalion-scale coastline carries thousands of near-collinear +-- vertices simplification can collapse for free without touching genuine corners/ +-- deviations. +-- +-- A first attempt restricted the Hausdorff call to a buffered region around the real +-- (area-filtered) symdifference fragments instead - much faster, but wrong twice +-- over, caught by the same two test cases that validated everything else here (chat +-- log 2026-10-05): it reported 0 for the nycb2010 block's actual ~338 ft, because +-- that block's own real defect is itself a near-zero-area needle (high extent, tiny +-- area - exactly what min_fragment_sqft is designed to treat as noise, wrongly, when +-- it's actually the thing a Hausdorff check exists to catch) and so got excluded from +-- the search region entirely; and it reported ~9,387 ft for FireBN 47's real ~1,988 +-- ft, because ST_Intersection-ing each geometry down to that region creates a brand +-- new boundary edge exactly at the clip line, and that synthetic edge - not any real +-- disagreement - is what Hausdorff then measured. Simplification doesn't introduce +-- new edges or depend on an area threshold, so it doesn't have either failure mode. +CREATE SCHEMA IF NOT EXISTS test_cases; + +-- CREATE OR REPLACE only replaces a function with the exact same parameter list - it +-- adds a new overload instead of replacing when the signature changes (as it did here, +-- gaining min_fragment_sqft), leaving the old 2-arg version callable alongside the new +-- one and making every 2-arg call ambiguous ("is not unique") until the stale overload +-- is dropped explicitly (broke the very next build after this param was added, chat +-- log 2026-10-05). Same gotcha applies to adding perimeter_pct_diff below - a new +-- output column is a return-type change too, so the previous 3-arg signature must be +-- dropped explicitly again, not just the old 2-arg one. +DROP FUNCTION IF EXISTS test_cases.geometry_diff_metrics(geometry, geometry); +DROP FUNCTION IF EXISTS test_cases.geometry_diff_metrics(geometry, geometry, double precision); + +CREATE OR REPLACE FUNCTION test_cases.geometry_diff_metrics( + actual geometry, expected geometry, min_fragment_sqft double precision DEFAULT 100 +) +RETURNS TABLE ( + actual_area_sqft double precision, + expected_area_sqft double precision, + symdiff_area_pct double precision, + hausdorff_ft double precision, + symdiff_boundary_len_ft double precision, + symdiff_fragment_count integer, + max_ring_eccentricity double precision, + perimeter_pct_diff double precision +) AS $$ + -- linearize() (defined above) first: prod's raw FGDB load can carry curved + -- MultiSurface/MultiCurve geometry, which ST_Simplify (used below for + -- hausdorff_ft) rejects outright - straighten it the same way every stg__ model + -- already does for its own source geometry, rather than requiring every caller + -- of this function to remember to. + -- + -- Turned out to fix more than a crash: nyfb (chat log 2026-10-05) had several + -- rows (e.g. FireBN 3/4) reporting a real-looking symdiff_boundary_len_ft purely + -- because prod's still-curved shape and our always-straight-line build output are + -- two different representations of arguably the same boundary - comparing a curve + -- against its chord approximation looks like disagreement even with zero real + -- difference. Linearizing both sides the same way before comparing collapsed + -- those to a clean 0 while leaving genuine discrepancies on other MultiSurface + -- rows (e.g. FireBN 20) exactly where they were - confirmed directly via + -- ST_GeometryType on production_outputs.fgdb_nyfb.shape, not just inferred. + WITH a AS (SELECT st_transform(linearize(actual), 2263) AS geom), + e AS (SELECT st_transform(linearize(expected), 2263) AS geom), + parts AS (SELECT (st_dump(a.geom)).geom AS part_geom FROM a), + rings AS ( + SELECT st_exteriorring(part_geom) AS ring FROM parts + UNION ALL + SELECT st_interiorringn(part_geom, gs) AS ring + FROM parts, generate_series(1, st_numinteriorrings(part_geom)) AS gs + ), + ring_eccentricity AS ( + SELECT max( + st_length(ring) / greatest(sqrt(abs(st_area(st_makepolygon(ring)))), 0.0001) + ) AS value + FROM rings + ), + real_fragments AS ( + SELECT (st_dump(st_symdifference(a.geom, e.geom))).geom AS frag + FROM a, e + WHERE e.geom IS NOT NULL + ), + real_fragments_filtered AS ( + SELECT frag FROM real_fragments WHERE st_area(frag) >= min_fragment_sqft + ) + SELECT + st_area(a.geom), + st_area(e.geom), + CASE WHEN e.geom IS NOT NULL THEN + (SELECT coalesce(sum(st_area(frag)), 0) FROM real_fragments_filtered) + / nullif(st_area(a.geom), 0) * 100 + END, + CASE WHEN e.geom IS NOT NULL THEN + st_hausdorffdistance(st_simplify(a.geom, 0.05), st_simplify(e.geom, 0.05)) + END, + CASE WHEN e.geom IS NOT NULL THEN + (SELECT coalesce(sum(st_perimeter(frag)), 0) FROM real_fragments_filtered) + END, + CASE WHEN e.geom IS NOT NULL THEN (SELECT count(*)::int FROM real_fragments_filtered) END, + (SELECT value FROM ring_eccentricity), + CASE WHEN e.geom IS NOT NULL THEN + (st_perimeter(a.geom) - st_perimeter(e.geom)) / nullif(st_perimeter(e.geom), 0) * 100 + END + FROM a, e +$$ LANGUAGE sql IMMUTABLE; + {% endmacro %} diff --git a/products/cscl/macros/district_boundary.sql b/products/cscl/macros/district_boundary.sql new file mode 100644 index 0000000000..c1f40ab099 --- /dev/null +++ b/products/cscl/macros/district_boundary.sql @@ -0,0 +1,482 @@ +{#- + District boundaries built from the AtomicPolygon topology. Every district type is the + same five steps, parameterized only by which int__topology__ap_entities column holds + the district id: + + district_boundary_classify_edges -> int__boundary___edges + district_boundary_build_raw -> int__boundary___raw (face membership) + district_boundary_strip_noise_rings -> int__boundary__ + district_boundary_validity -> qa__boundary___validity + district_boundary_vs_prod -> qa__boundary___vs_prod + + Every step outputs `entity_id` as its key. No step touches the topology itself + (int__topology__edges is shared), so adding a district type costs only these steps. + No polygon overlay on source polygons is on the construction path: boundaries are + assembled from classified, noded topology edges. +-#} + + +{% macro district_boundary_classify_edges(entity_column, include_assigned_water=false) %} +{#- + Classifies every edge in int__topology__edges as 'interior' or 'exterior' to each + district that owns it, for building a land boundary: + + - land-water edge (one owner water, the other land): always 'exterior' - it's the + shoreline, whichever district the water is tagged with. Emitted once per land + owner, never also through the land-land path (a boundary line present twice + breaks ST_BuildArea's parity-based polygon building). + - water-water edge: excluded. + - land-land edge: 'interior' only when ALL its owners belong to the same district + and there are at least 2 of them; otherwise 'exterior'. + + include_assigned_water: when true, a water AP that carries a district id counts as + land for that district; only unassigned water acts as shoreline. Off for census + geographies (land-only); on for fire divisions, whose published layer includes the + water APs that have an administering fire company. + + APs with a NULL district still count as owners (so their neighbours' shared edges + read as 'exterior') but produce no rows of their own. +-#} +WITH edges AS ( + SELECT + e.node_lo, e.node_hi, e.is_arc, e.geom, e.atomicid, m.{{ entity_column }} AS entity_id, + {%- if include_assigned_water %} + e.water_flag = '1' AND m.{{ entity_column }} IS NULL AS is_water + {%- else %} + e.water_flag = '1' AS is_water + {%- endif %} + FROM {{ ref('int__topology__edges') }} AS e + INNER JOIN {{ ref('int__topology__ap_entities') }} AS m ON e.atomicid = m.atomicid +), + +has_water_neighbor AS ( + SELECT DISTINCT node_lo, node_hi, is_arc + FROM edges + WHERE is_water +), + +land_land_edges AS ( + SELECT e.node_lo, e.node_hi, e.is_arc, e.geom, e.atomicid, e.entity_id + FROM edges AS e + LEFT JOIN has_water_neighbor AS hw + ON e.node_lo = hw.node_lo AND e.node_hi = hw.node_hi AND e.is_arc = hw.is_arc + WHERE NOT e.is_water AND hw.node_lo IS NULL +), + +edge_owner_counts AS ( + SELECT node_lo, node_hi, is_arc, count(DISTINCT atomicid) AS total_owners + FROM land_land_edges + GROUP BY node_lo, node_hi, is_arc +), + +edge_entity_counts AS ( + SELECT + node_lo, node_hi, is_arc, entity_id, + count(DISTINCT atomicid) AS owners_in_entity, + (array_agg(geom))[1] AS geom + FROM land_land_edges + GROUP BY node_lo, node_hi, is_arc, entity_id +), + +land_land_classified AS ( + SELECT + ec.entity_id, ec.node_lo, ec.node_hi, ec.is_arc, ec.geom, + CASE + WHEN ec.owners_in_entity = eoc.total_owners AND eoc.total_owners >= 2 THEN 'interior' + ELSE 'exterior' + END AS edge_type + FROM edge_entity_counts AS ec + INNER JOIN edge_owner_counts AS eoc + ON ec.node_lo = eoc.node_lo AND ec.node_hi = eoc.node_hi AND ec.is_arc = eoc.is_arc +), + +-- Emitted once per LAND-side row, using THAT row's own entity - "is this land parcel's +-- edge exterior here because it borders water" doesn't depend on which entity the +-- water itself is tagged with, so this intentionally does NOT require the land and +-- water owner to share an entity. +shoreline_edges AS ( + SELECT + e.entity_id, + e.node_lo, + e.node_hi, + e.is_arc, + e.geom, + 'exterior' AS edge_type + FROM edges AS e + INNER JOIN has_water_neighbor AS hw + ON e.node_lo = hw.node_lo AND e.node_hi = hw.node_hi AND e.is_arc = hw.is_arc + WHERE NOT e.is_water +) + +-- An AP with no entity (e.g. no fire division) still counts as an owner above, so its +-- neighbors' shared edges correctly read as 'exterior' - it just builds no boundary. +SELECT * FROM land_land_classified WHERE entity_id IS NOT NULL +UNION ALL +SELECT * FROM shoreline_edges WHERE entity_id IS NOT NULL +{% endmacro %} + + +{% macro district_boundary_build_raw(edges_model, entity_column, include_assigned_water=false) %} +{#- + One row per district, built from its 'exterior' edges. "_raw": noise interior rings + are not yet removed (district_boundary_strip_noise_rings does that, and + district_boundary_validity inspects this pre-cleanup geometry). + + Face membership, not ring parity: the exterior edges are polygonized into faces, and + a face is kept only if it contains the interior point of one of the district's own + APs (same land / assigned-water rule as district_boundary_classify_edges). ST_BuildArea + decides inside vs outside by ring nesting, which fails where a district's outline + pinches - e.g. hurricane evacuation zone 5 wraps a zone-6 block that touches zone 5's + outer boundary at single corner points, and ST_BuildArea returned zone 5 with no hole, + swallowing the block. A face either is or isn't covered by the district's APs, + however its rings touch. Faces holding no member AP at all (slivers between two APs' + mismatched corners) drop out here too. + + Each district is built from its own edge classification rather than by unioning + smaller built districts - an edge between two tracts in the same PUMA is 'interior' + at PUMA level but 'exterior' at tract level. +-#} +WITH faces AS ( + SELECT + entity_id, + (st_dump(st_polygonize(geom))).geom AS face + FROM {{ ref(edges_model) }} + WHERE edge_type = 'exterior' + GROUP BY entity_id +), + +member_points AS ( + SELECT + m.{{ entity_column }} AS entity_id, + st_pointonsurface(a.geom) AS pt + FROM {{ ref('int__topology__ap_entities') }} AS m + INNER JOIN {{ ref('stg__atomicpolygons') }} AS a ON m.atomicid = a.atomicid + WHERE + m.{{ entity_column }} IS NOT NULL + {%- if not include_assigned_water %} + AND m.water_flag != '1' + {%- endif %} +), + +kept AS ( + SELECT + f.entity_id, + f.face + FROM faces AS f + WHERE EXISTS ( + SELECT 1 FROM member_points AS mp + WHERE mp.entity_id = f.entity_id AND f.face && mp.pt AND st_contains(f.face, mp.pt) + ) +) + +-- kept faces share exact, already-noded edges, so this union only dissolves them +SELECT + entity_id, + st_union(face) AS geom +FROM kept +GROUP BY entity_id +{% endmacro %} + + +{% macro district_boundary_strip_noise_rings(boundary_raw_model, edges_model) %} +{#- + Removes interior rings that are noise, keeps real ones. A ring is noise if any of its + edges is missing a left or right AtomicPolygon - a single owner across the whole + topology, not just within this district. Noise rings come from two AtomicPolygons + disagreeing about a shared corner (thin loops of one-owner edges, often near-zero area + but long). A real hole - an enclave district, or water - is bounded on both sides: + this district's AP on one, whatever fills the hole on the other. + + Ring edges are identified with ST_Covers, which is exact here because ST_BuildArea + reuses the input edges' coordinates. +-#} +WITH owner_counts AS ( + SELECT node_lo, node_hi, is_arc, count(DISTINCT atomicid) AS total_owners + FROM {{ ref('int__topology__edges') }} + GROUP BY node_lo, node_hi, is_arc +), + +one_owner_edges AS ( + SELECT e.entity_id, e.geom + FROM {{ ref(edges_model) }} AS e + INNER JOIN owner_counts AS oc + ON e.node_lo = oc.node_lo AND e.node_hi = oc.node_hi AND e.is_arc = oc.is_arc + WHERE e.edge_type = 'exterior' AND oc.total_owners = 1 +), + +parts AS ( + SELECT + r.entity_id, + -- ST_Dump of a single Polygon has an empty path - NULL would never join below + coalesce(d.path[1], 1) AS part_n, + d.geom AS part + FROM {{ ref(boundary_raw_model) }} AS r + CROSS JOIN LATERAL st_dump(r.geom) AS d +), + +interior_rings AS ( + SELECT + p.entity_id, + p.part_n, + n AS ring_n, + st_interiorringn(p.part, n) AS ring + FROM parts AS p + CROSS JOIN LATERAL generate_series(1, st_numinteriorrings(p.part)) AS n +), + +kept_rings AS ( + SELECT + ir.entity_id, + ir.part_n, + array_agg(ir.ring ORDER BY ir.ring_n) AS rings + FROM interior_rings AS ir + WHERE NOT EXISTS ( + SELECT 1 + FROM one_owner_edges AS e + WHERE e.entity_id = ir.entity_id AND e.geom && ir.ring AND st_covers(ir.ring, e.geom) + ) + GROUP BY ir.entity_id, ir.part_n +), + +rebuilt AS ( + SELECT + p.entity_id, + CASE + WHEN k.rings IS NULL THEN st_makepolygon(st_exteriorring(p.part)) + ELSE st_makepolygon(st_exteriorring(p.part), k.rings) + END AS geom + FROM parts AS p + LEFT JOIN kept_rings AS k ON p.entity_id = k.entity_id AND p.part_n = k.part_n +) + +SELECT + entity_id, + st_union(geom) AS geom +FROM rebuilt +GROUP BY entity_id +{% endmacro %} + + +{% macro district_boundary_validity(edges_model, boundary_raw_model, boundary_model) %} +{#- + Topology checks per district. Three gate is_valid: + even_degree_ok - every node on the district's 'exterior' edges has even degree + (closed rings; an odd node is a dangling chain). + buildarea_ok - ST_BuildArea returned a non-empty polygonal result. + final_geom_valid_ok - ST_IsValid on the final (noise-stripped) boundary. + One diagnostic only: + is_simple_ok - the raw 'exterior' edges, line-merged, are simple. False for + a district carrying a noise loop that the final geometry + correctly drops, so it doesn't gate; useful when + investigating a new failure. +-#} +WITH exterior AS ( + SELECT entity_id, node_lo, node_hi, geom + FROM {{ ref(edges_model) }} + WHERE edge_type = 'exterior' +), + +merged_check AS ( + SELECT + entity_id, + st_issimple(st_linemerge(st_collect(geom))) AS is_simple_ok + FROM exterior + GROUP BY entity_id +), + +node_degree AS ( + SELECT + entity_id, + node_id, + count(*) AS degree + FROM ( + SELECT entity_id, node_lo AS node_id FROM exterior + UNION ALL + SELECT entity_id, node_hi AS node_id FROM exterior + ) AS endpoints + GROUP BY entity_id, node_id +), + +degree_check AS ( + SELECT + entity_id, + count(*) FILTER (WHERE degree % 2 != 0) AS n_odd_degree_nodes, + count(*) AS n_nodes_total + FROM node_degree + GROUP BY entity_id +), + +buildarea_check AS ( + SELECT + entity_id, + geom IS NOT NULL + AND NOT st_isempty(geom) + AND st_geometrytype(geom) IN ('ST_Polygon', 'ST_MultiPolygon') AS buildarea_ok + FROM {{ ref(boundary_raw_model) }} +), + +final_geom_check AS ( + SELECT + entity_id, + geom IS NOT NULL AND st_isvalid(geom) AS final_geom_valid_ok + FROM {{ ref(boundary_model) }} +) + +SELECT + dc.entity_id, + dc.n_odd_degree_nodes = 0 AS even_degree_ok, + dc.n_odd_degree_nodes, + dc.n_nodes_total, + coalesce(mc.is_simple_ok, FALSE) AS is_simple_ok, + coalesce(bc.buildarea_ok, FALSE) AS buildarea_ok, + coalesce(fc.final_geom_valid_ok, FALSE) AS final_geom_valid_ok, + (dc.n_odd_degree_nodes = 0) + AND coalesce(bc.buildarea_ok, FALSE) + AND coalesce(fc.final_geom_valid_ok, FALSE) AS is_valid +FROM degree_check AS dc +LEFT JOIN merged_check AS mc ON dc.entity_id = mc.entity_id +LEFT JOIN buildarea_check AS bc ON dc.entity_id = bc.entity_id +LEFT JOIN final_geom_check AS fc ON dc.entity_id = fc.entity_id +{% endmacro %} + + +{% macro district_boundary_vs_prod(boundary_model, validity_model, prod_identifier, prod_key, entity_column, dissolve_prod=false) %} +{#- + Compares a built boundary against its published production_outputs layer: symdiff + area %, perimeter %, part and hole counts, Hausdorff (selectively), with the + district's validity verdict alongside. + + Full outer join, prod restricted to district ids present in scope, with match_status + 'matched' / 'built_only' / 'prod_only' - a district missing on one side is a result, + not something to drop silently. + + Hausdorff catches spurs and thin loops that area and perimeter miss, but GEOS's + implementation is O(n*m) in vertex count and can't be interrupted mid-call. It's + computed only when either side has an interior ring or the part counts differ, and + only below boundary_qa_hausdorff_max_vertex_product; NULL otherwise. + + dissolve_prod: for published layers that store one feature per disjoint piece rather + than one per district (e.g. fgdb_nymcea), union prod's features per key first. +-#} +WITH in_scope AS ( + SELECT DISTINCT {{ entity_column }} AS entity_id + FROM {{ ref('int__topology__ap_entities') }} +), + +prod AS ( + SELECT + p.{{ prod_key }} AS entity_id, + {% if dissolve_prod %}st_union(linearize(p.shape)){% else %}linearize(p.shape){% endif %} AS geom + FROM {{ adapter.get_relation(database="db-cscl", schema="production_outputs", identifier=prod_identifier) }} AS p + INNER JOIN in_scope AS s ON p.{{ prod_key }} = s.entity_id + {% if dissolve_prod %}GROUP BY p.{{ prod_key }}{% endif %} +), + +structure AS ( + SELECT + entity_id, + geom, + st_numgeometries(geom) AS n_parts, + (SELECT sum(st_numinteriorrings(d.geom)) FROM st_dump(geom) AS d) AS n_holes + FROM {{ ref(boundary_model) }} +), + +prod_structure AS ( + SELECT + entity_id, + geom, + st_numgeometries(geom) AS n_parts, + (SELECT sum(st_numinteriorrings(d.geom)) FROM st_dump(geom) AS d) AS n_holes + FROM prod +) + +SELECT + coalesce(p.entity_id, b.entity_id) AS entity_id, + CASE + WHEN b.entity_id IS NULL THEN 'prod_only' + WHEN p.entity_id IS NULL THEN 'built_only' + ELSE 'matched' + END AS match_status, + v.is_valid, + b.n_parts AS built_parts, + p.n_parts AS prod_parts, + b.n_holes AS built_holes, + p.n_holes AS prod_holes, + round((st_area(st_symdifference(b.geom, p.geom)) / nullif(st_area(p.geom), 0) * 100)::numeric, 6) AS symdiff_area_pct, + round(((st_perimeter(b.geom) - st_perimeter(p.geom)) / nullif(st_perimeter(p.geom), 0) * 100)::numeric, 4) AS perimeter_pct_diff, + CASE + WHEN + (b.n_holes > 0 OR p.n_holes > 0 OR b.n_parts != p.n_parts) + AND st_npoints(b.geom)::bigint * st_npoints(p.geom) <= {{ var('boundary_qa_hausdorff_max_vertex_product') }} + THEN round(st_hausdorffdistance(b.geom, p.geom)::numeric, 3) + END AS hausdorff_ft, + round(st_area(b.geom)::numeric, 1) AS built_area_sqft, + round(st_area(p.geom)::numeric, 1) AS prod_area_sqft +FROM prod_structure AS p +FULL OUTER JOIN structure AS b ON p.entity_id = b.entity_id +LEFT JOIN {{ ref(validity_model) }} AS v ON coalesce(p.entity_id, b.entity_id) = v.entity_id +{% endmacro %} + + +{% macro district_boundary_outliers(entity, entity_column, prod_identifier, prod_key) %} +{#- + For viewing (QGIS etc.): every district missing on one side, or whose Hausdorff (where + computed) or symdiff exceeds boundary_qa_outlier_min_hausdorff_ft / + boundary_qa_outlier_min_symdiff_pct, broken into layers by `layer`: + built / prod - the two boundaries + built_only - built minus prod + prod_only - prod minus built + ap - every AtomicPolygon in the district, with water_flag + ST_Difference is fine here: this is diagnostics, not the construction path. +-#} +WITH outliers AS ( + SELECT entity_id, hausdorff_ft, symdiff_area_pct + FROM {{ ref('qa__boundary__' ~ entity ~ '_vs_prod') }} + WHERE + match_status != 'matched' + OR hausdorff_ft > {{ var('boundary_qa_outlier_min_hausdorff_ft') }} + OR symdiff_area_pct > {{ var('boundary_qa_outlier_min_symdiff_pct') }} +), + +built AS ( + SELECT b.entity_id, b.geom + FROM {{ ref('int__boundary__' ~ entity) }} AS b + INNER JOIN outliers AS o ON b.entity_id = o.entity_id +), + +prod AS ( + SELECT p.{{ prod_key }} AS entity_id, linearize(p.shape) AS geom + FROM {{ adapter.get_relation(database="db-cscl", schema="production_outputs", identifier=prod_identifier) }} AS p + INNER JOIN outliers AS o ON p.{{ prod_key }} = o.entity_id +), + +layers AS ( + SELECT entity_id, 'built' AS layer, NULL AS atomicid, NULL AS water_flag, geom FROM built + UNION ALL + SELECT entity_id, 'prod', NULL, NULL, geom FROM prod + UNION ALL + SELECT b.entity_id, 'built_only', NULL, NULL, st_difference(b.geom, p.geom) + FROM built AS b INNER JOIN prod AS p ON b.entity_id = p.entity_id + UNION ALL + SELECT b.entity_id, 'prod_only', NULL, NULL, st_difference(p.geom, b.geom) + FROM built AS b INNER JOIN prod AS p ON b.entity_id = p.entity_id + UNION ALL + SELECT m.{{ entity_column }}, 'ap', a.atomicid, a.water_flag, a.geom + FROM {{ ref('int__topology__ap_entities') }} AS m + INNER JOIN outliers AS o ON m.{{ entity_column }} = o.entity_id + INNER JOIN {{ ref('stg__atomicpolygons') }} AS a ON m.atomicid = a.atomicid +) + +SELECT + l.entity_id, + l.layer, + l.atomicid, + l.water_flag, + o.hausdorff_ft, + o.symdiff_area_pct, + round(st_area(l.geom)::numeric, 1) AS area_sqft, + st_multi(st_collectionextract(l.geom, 3))::geometry(MultiPolygon, 2263) AS geom +FROM layers AS l +INNER JOIN outliers AS o ON l.entity_id = o.entity_id +WHERE NOT st_isempty(l.geom) +{% endmacro %} diff --git a/products/cscl/macros/geometry_diff_qa.sql b/products/cscl/macros/geometry_diff_qa.sql new file mode 100644 index 0000000000..fbe969a15f --- /dev/null +++ b/products/cscl/macros/geometry_diff_qa.sql @@ -0,0 +1,201 @@ +{% macro geometry_diff_qa(build_relation, prod_relation, key_column, build_geom_column='geom', prod_geom_column='shape', prod_key_column=none) %} +{#- + Full-layer geometry QA: joins every build row to its prod counterpart by key, runs + test_cases.geometry_diff_metrics (macros/create_pg_functions.sql) on each pair, and + flags pass/fail against the same thresholds qa__test_cases_geometry uses. See that + model and the function's own docstring for why area/perimeter diffing alone isn't + enough (products/cscl chat log, 2026-10-04/05). + + key_column: build-side key, as it appears in the build relation (quote if it's a + case-sensitive gdb-style column, e.g. '"FireBN"'). Single-column only for now - + a multi-column declared key (lion_outputs.csv's "FireCoType|FireCoNum" style) + needs a concatenation expression passed as key_column instead. + prod_key_column: defaults to key_column.lower() with quoting stripped, matching how + ogr2ogr lowercases every prod column on load - override if that guess is wrong. + build_geom_column/prod_geom_column: default to how gdb_* models and ogr2ogr-loaded + fgdb_* tables name their geometry column respectively - override per layer if + different. + + Rows only on one side (only_in_build/only_in_prod) get no metrics at all - there's + no pair to compare, and that's its own, more basic QA concern (row-count/key + coverage), not a geometry-shape one. + + actual_geom_4326/expected_geom_4326 carry through to the final output for eventual + display in a Python mapping tool (folium, geopandas .explore(), etc. - all expect + lat/lon) - linearized (prod's FGDB load can be curved MultiSurface, which most + geometry tooling, shapely/geopandas included, rejects outright) and transformed to + EPSG:4326/WGS84, separately from the geometry the metrics below are computed from. + Deliberately NOT the same transform path test_cases.geometry_diff_metrics uses + internally (linearize + transform to this product's usual EPSG:2263 working SRID) - + reprojecting display geometry to 4326 and reprojecting measurement geometry to 2263 + are two different concerns, and round-tripping the same geometry through both + (native SRID -> 4326 -> back to 2263 inside the metrics function) would add its own + tiny reprojection noise on top of everything this investigation has already spent on + chasing sub-foot precision - chat log 2026-10-06. actual_geom/expected_geom (no + _4326 suffix, native SRID, unlinearized) still feed the metrics exactly as before. +-#} + {%- set prod_key_column = prod_key_column or (key_column | replace('"', '') | lower) -%} + +-- One row per key on each side: a layer that stores a district as several features +-- (e.g. nysd's SD 10, once per borough) is dissolved first, so the key join compares +-- whole districts rather than every piece against every other piece. Single-row keys +-- pass through untouched. +WITH build AS ( + SELECT + {{ key_column }}::text AS key_value, + CASE + WHEN count(*) = 1 THEN (array_agg({{ build_geom_column }}))[1] + ELSE st_union(st_makevalid(linearize({{ build_geom_column }}))) + END AS actual_geom + FROM {{ build_relation }} + GROUP BY 1 +), +prod AS ( + SELECT + {{ prod_key_column }}::text AS key_value, + CASE + WHEN count(*) = 1 THEN (array_agg({{ prod_geom_column }}))[1] + ELSE st_union(st_makevalid(linearize({{ prod_geom_column }}))) + END AS expected_geom + FROM {{ prod_relation }} + GROUP BY 1 +), +joined AS ( + SELECT + coalesce(b.key_value, p.key_value) AS key_value, + b.actual_geom, + p.expected_geom, + (b.key_value IS NULL) AS only_in_prod, + (p.key_value IS NULL) AS only_in_build + FROM build AS b + FULL OUTER JOIN prod AS p ON b.key_value = p.key_value +), +metrics AS ( + SELECT + j.key_value, + j.only_in_prod, + j.only_in_build, + st_transform(linearize(j.actual_geom), 4326) AS actual_geom_4326, + st_transform(linearize(j.expected_geom), 4326) AS expected_geom_4326, + m.* + FROM joined AS j + LEFT JOIN LATERAL test_cases.geometry_diff_metrics(j.actual_geom, j.expected_geom) AS m + ON j.actual_geom IS NOT NULL +), +flagged AS ( + SELECT + *, + array_remove(ARRAY[ + CASE WHEN only_in_build THEN 'only_in_build' END, + CASE WHEN only_in_prod THEN 'only_in_prod' END, + CASE WHEN symdiff_area_pct > {{ var('geom_check_symdiff_area_pct', 1) }} + THEN 'symdiff_area_pct' END, + CASE WHEN hausdorff_ft > {{ var('geom_check_hausdorff_ft', 50) }} + THEN 'hausdorff_ft' END, + CASE WHEN symdiff_boundary_len_ft > {{ var('geom_check_symdiff_boundary_len_ft', 100) }} + THEN 'symdiff_boundary_len_ft' END, + CASE WHEN max_ring_eccentricity > {{ var('geom_check_max_ring_eccentricity', 50) }} + THEN 'max_ring_eccentricity' END, + CASE WHEN abs(perimeter_pct_diff) > {{ var('geom_check_perimeter_pct', 1) }} + THEN 'perimeter_pct_diff' END + ], NULL) AS failing_checks + FROM metrics +) + +SELECT + *, + (array_length(failing_checks, 1) IS NULL) AS passes +FROM flagged +{% endmacro %} + + +{% macro geometry_diff_qa_cheap(build_relation, prod_relation, key_column, build_geom_column='geom', prod_geom_column='shape', prod_key_column=none) %} +{#- + Cheap variant of geometry_diff_qa for layers too large to run full-table + (test_cases.geometry_diff_metrics' ST_SymDifference + simplified ST_HausdorffDistance + cost ~0.5-1.2s/row empirically - fine at dozens to low thousands of rows, not at + nyap's 69,786 or nycb2010/2010wi/2020/2020wi's ~38-39k each, which would add 5-20+ + hours per layer per build - chat log 2026-10-05). + + Computes only perimeter_pct_diff and area_pct_diff - a direct ST_Perimeter/ST_Area + comparison on each side independently, no overlay operation at all (no + ST_SymDifference, no ST_HausdorffDistance) - cheap enough to run on every row of even + the largest layers. Catches the exact class of bug that motivated adding + perimeter_pct_diff to the full framework in the first place (nearest-N rounding- + boundary false positives in the older qa_int__prod_fgdb_*/​*_by_field comparisons, + chat log 2026-10-05), just without symdiff_area_pct/hausdorff_ft/symdiff_fragment_ + count/max_ring_eccentricity's deeper (but expensive) shape-defect detection. + + Same key_column/prod_key_column/build_geom_column/prod_geom_column semantics as + geometry_diff_qa - see that macro's docstring. +-#} + {%- set prod_key_column = prod_key_column or (key_column | replace('"', '') | lower) -%} + +-- One row per key on each side (multi-feature districts dissolved) - see +-- geometry_diff_qa. +WITH build AS ( + SELECT + {{ key_column }}::text AS key_value, + CASE + WHEN count(*) = 1 THEN (array_agg(st_transform(linearize({{ build_geom_column }}), 2263)))[1] + ELSE st_union(st_makevalid(st_transform(linearize({{ build_geom_column }}), 2263))) + END AS actual_geom + FROM {{ build_relation }} + GROUP BY 1 +), +prod AS ( + SELECT + {{ prod_key_column }}::text AS key_value, + CASE + WHEN count(*) = 1 THEN (array_agg(st_transform(linearize({{ prod_geom_column }}), 2263)))[1] + ELSE st_union(st_makevalid(st_transform(linearize({{ prod_geom_column }}), 2263))) + END AS expected_geom + FROM {{ prod_relation }} + GROUP BY 1 +), +joined AS ( + SELECT + coalesce(b.key_value, p.key_value) AS key_value, + b.actual_geom, + p.expected_geom, + (b.key_value IS NULL) AS only_in_prod, + (p.key_value IS NULL) AS only_in_build + FROM build AS b + FULL OUTER JOIN prod AS p ON b.key_value = p.key_value +), +metrics AS ( + SELECT + key_value, + only_in_prod, + only_in_build, + st_area(actual_geom) AS actual_area_sqft, + st_area(expected_geom) AS expected_area_sqft, + CASE WHEN expected_geom IS NOT NULL THEN + (st_perimeter(actual_geom) - st_perimeter(expected_geom)) + / nullif(st_perimeter(expected_geom), 0) * 100 + END AS perimeter_pct_diff, + CASE WHEN expected_geom IS NOT NULL THEN + (st_area(actual_geom) - st_area(expected_geom)) + / nullif(st_area(expected_geom), 0) * 100 + END AS area_pct_diff + FROM joined +), +flagged AS ( + SELECT + *, + array_remove(ARRAY[ + CASE WHEN only_in_build THEN 'only_in_build' END, + CASE WHEN only_in_prod THEN 'only_in_prod' END, + CASE WHEN abs(perimeter_pct_diff) > {{ var('geom_check_perimeter_pct', 1) }} + THEN 'perimeter_pct_diff' END, + CASE WHEN abs(area_pct_diff) > {{ var('geom_check_area_pct_diff', 1) }} + THEN 'area_pct_diff' END + ], NULL) AS failing_checks + FROM metrics +) + +SELECT + *, + (array_length(failing_checks, 1) IS NULL) AS passes +FROM flagged +{% endmacro %} diff --git a/products/cscl/models/etl_dev_qa/_etl_dev_qa.yml b/products/cscl/models/etl_dev_qa/_etl_dev_qa.yml index b181905f31..53ac3cb911 100644 --- a/products/cscl/models/etl_dev_qa/_etl_dev_qa.yml +++ b/products/cscl/models/etl_dev_qa/_etl_dev_qa.yml @@ -1,6 +1,33 @@ version: 2 models: +# ── Geometry test cases (TDD fixtures) ────────────────────────────────────────── + +- name: qa__test_cases_geometry + description: > + Applies test_cases.geometry_diff_metrics to every row in test_cases.examples - + our TDD fixture table of known-weird geometries. Expected to go red when a new + weird case is added or a fix regresses, green once the underlying issue is + actually fixed. + data_tests: + # TODO: disabled 2026-10-05 - all 6 test_cases.examples rows are currently + # failing (see inv-tv1.2). Needs investigation before re-enabling: likely the + # qa thresholds/metrics need revisiting now that gridSize fixes have changed + # the fixtures' actual vs expected geoms, not necessarily a real regression. + - dbt_utils.expression_is_true: + arguments: + column_name: passes + expression: "= true" + config: + enabled: false + columns: + - name: slug + tests: [ not_null, unique ] + - name: passes + description: True if every computed metric is within its threshold (see model SQL). + - name: failing_metrics + description: Array of metric names that exceeded their threshold for this row. + # ── LION DAT QA ────────────────────────────────────────────────────────────── - name: qa__lion_dat_by_row diff --git a/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_fgdb_nycb2010.sql b/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_fgdb_nycb2010.sql new file mode 100644 index 0000000000..c94802ffbf --- /dev/null +++ b/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_fgdb_nycb2010.sql @@ -0,0 +1,54 @@ +{{ + config( + materialized='table', + tags=['on_demand', 'diffs', 'diffs_nycb2010'] + ) +}} + +-- Compressed diff view showing one row per bctcb2010 with changes in jsonb. +-- Includes status: modified, only_in_legacy, only_in_build. +-- accounted_for is always false for now - nycb2010's known issue (CSCL-DISTRICTS-03 +-- shoreline-clip geometry noise, not yet borough-bound) isn't fingerprinted to +-- specific rows yet, unlike e.g. qa__diffs_lion_dat's hardcoded bug cases. + +WITH base_diffs AS ( + {{ generate_diff_summary( + old_relation=ref('qa_int__prod_fgdb_nycb2010'), + new_relation=ref('gdb_nycb2010_by_field'), + primary_key='bctcb2010', + output_file_id='gdb_nycb2010', + build_table_name='gdb_nycb2010_by_field', + production_table_name='qa_int__prod_fgdb_nycb2010' + ) }} +), +categorized AS ( + SELECT + bctcb2010, + status, + changes, + output_file_id, + -- Get the keys from the changes jsonb + (SELECT array_agg(key) FROM jsonb_object_keys(changes) AS key) AS change_keys, + comparison_column, + build_table_name, + production_table_name + FROM base_diffs +) +SELECT + bctcb2010 AS comparison_id, + status, + changes, + output_file_id, + -- Categorize based on which fields changed + CASE + -- If only one field changed, use that as the group name + WHEN status = 'modified' AND array_length(change_keys, 1) = 1 + THEN change_keys[1] + ELSE '' + END AS diff_group, + '' AS subgroup, + comparison_column, + build_table_name, + production_table_name, + false AS accounted_for +FROM categorized diff --git a/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_fgdb_nycd.sql b/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_fgdb_nycd.sql new file mode 100644 index 0000000000..d384894f49 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_fgdb_nycd.sql @@ -0,0 +1,36 @@ +{{ + config( + materialized='table', + tags=['on_demand', 'diffs', 'diffs_nycd'] + ) +}} + +-- Standard diff-summary shape over qa__geometry_fgdb_nycd - see that model, +-- macros/geometry_diff_qa.sql, and qa__diffs_fgdb_nyfb for the pattern/caveats +-- (chat log 2026-10-04/05). Only non-passing rows are included. +SELECT + key_value AS comparison_id, + CASE + WHEN only_in_build THEN 'only_in_build' + WHEN only_in_prod THEN 'only_in_prod' + ELSE 'modified' + END AS status, + jsonb_build_object( + 'symdiff_area_pct', symdiff_area_pct, + 'hausdorff_ft', hausdorff_ft, + 'symdiff_boundary_len_ft', symdiff_boundary_len_ft, + 'symdiff_fragment_count', symdiff_fragment_count, + 'max_ring_eccentricity', max_ring_eccentricity + ) AS changes, + 'gdb_nycd' AS output_file_id, + CASE + WHEN array_length(failing_checks, 1) = 1 THEN failing_checks[1] + ELSE '' + END AS diff_group, + '' AS subgroup, + 'key_value' AS comparison_column, + 'gdb_nycd' AS build_table_name, + 'fgdb_nycd' AS production_table_name, + false AS accounted_for +FROM {{ ref('qa__geometry_fgdb_nycd') }} +WHERE NOT passes diff --git a/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_fgdb_nyfb.sql b/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_fgdb_nyfb.sql new file mode 100644 index 0000000000..d35eab6160 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_fgdb_nyfb.sql @@ -0,0 +1,48 @@ +{{ + config( + materialized='table', + tags=['on_demand', 'diffs', 'diffs_nyfb'] + ) +}} + +-- Standard diff-summary shape over qa__geometry_fgdb_nyfb (the detailed per-metric +-- model - see that model and macros/geometry_diff_qa.sql for how each row is +-- computed), so nyfb's geometry QA rolls up into qa__diffs_all like every other +-- layer. Only non-passing rows are included, matching qa__diffs_fgdb_altnames' +-- convention of emitting discrepancies only, not full row-for-row parity. +-- +-- Not a strict superset of the old SHAPE_Length/Area-tolerance check +-- (poc_validation/compare_gdb.py): confirmed on nyfb itself (chat log 2026-10-05) - +-- 11 of compare_gdb.py's 12 flagged rows overlap with this model's 13, but +-- FireBN 17 (28.2 ft off on a 44,599 ft perimeter - 0.063%, just past +-- compare_gdb.py's 0.05% relative tolerance) doesn't trip any of this model's +-- absolute thresholds, while FireBN 21/23 trip this model's shape-based checks +-- without moving SHAPE_Length/Area enough to trip the old one. Different, +-- overlapping methods, not one strictly containing the other - expect some rows to +-- show up in only one until both are tuned against a lot more real examples. +SELECT + key_value AS comparison_id, + CASE + WHEN only_in_build THEN 'only_in_build' + WHEN only_in_prod THEN 'only_in_prod' + ELSE 'modified' + END AS status, + jsonb_build_object( + 'symdiff_area_pct', symdiff_area_pct, + 'hausdorff_ft', hausdorff_ft, + 'symdiff_boundary_len_ft', symdiff_boundary_len_ft, + 'symdiff_fragment_count', symdiff_fragment_count, + 'max_ring_eccentricity', max_ring_eccentricity + ) AS changes, + 'gdb_nyfb' AS output_file_id, + CASE + WHEN array_length(failing_checks, 1) = 1 THEN failing_checks[1] + ELSE '' + END AS diff_group, + '' AS subgroup, + 'key_value' AS comparison_column, + 'gdb_nyfb' AS build_table_name, + 'fgdb_nyfb' AS production_table_name, + false AS accounted_for +FROM {{ ref('qa__geometry_fgdb_nyfb') }} +WHERE NOT passes diff --git a/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_fgdb_nypuma2010.sql b/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_fgdb_nypuma2010.sql new file mode 100644 index 0000000000..18991cf30c --- /dev/null +++ b/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_fgdb_nypuma2010.sql @@ -0,0 +1,37 @@ +{{ + config( + materialized='table', + tags=['on_demand', 'diffs', 'diffs_nypuma2010'] + ) +}} + +-- Standard diff-summary shape over qa__geometry_fgdb_nypuma2010 - see that model, +-- macros/geometry_diff_qa.sql, and qa__diffs_fgdb_nyfb for the pattern/caveats +-- (chat log 2026-10-04/05). Only non-passing rows are included. +SELECT + key_value AS comparison_id, + CASE + WHEN only_in_build THEN 'only_in_build' + WHEN only_in_prod THEN 'only_in_prod' + ELSE 'modified' + END AS status, + jsonb_build_object( + 'symdiff_area_pct', symdiff_area_pct, + 'hausdorff_ft', hausdorff_ft, + 'symdiff_boundary_len_ft', symdiff_boundary_len_ft, + 'symdiff_fragment_count', symdiff_fragment_count, + 'max_ring_eccentricity', max_ring_eccentricity, + 'perimeter_pct_diff', perimeter_pct_diff + ) AS changes, + 'gdb_nypuma2010' AS output_file_id, + CASE + WHEN array_length(failing_checks, 1) = 1 THEN failing_checks[1] + ELSE '' + END AS diff_group, + '' AS subgroup, + 'key_value' AS comparison_column, + 'gdb_nypuma2010' AS build_table_name, + 'fgdb_nypuma2010' AS production_table_name, + false AS accounted_for +FROM {{ ref('qa__geometry_fgdb_nypuma2010') }} +WHERE NOT passes diff --git a/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_fgdb_nypuma2020.sql b/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_fgdb_nypuma2020.sql new file mode 100644 index 0000000000..69d7330120 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_fgdb_nypuma2020.sql @@ -0,0 +1,37 @@ +{{ + config( + materialized='table', + tags=['on_demand', 'diffs', 'diffs_nypuma2020'] + ) +}} + +-- Standard diff-summary shape over qa__geometry_fgdb_nypuma2020 - see that model, +-- macros/geometry_diff_qa.sql, and qa__diffs_fgdb_nyfb for the pattern/caveats +-- (chat log 2026-10-04/05). Only non-passing rows are included. +SELECT + key_value AS comparison_id, + CASE + WHEN only_in_build THEN 'only_in_build' + WHEN only_in_prod THEN 'only_in_prod' + ELSE 'modified' + END AS status, + jsonb_build_object( + 'symdiff_area_pct', symdiff_area_pct, + 'hausdorff_ft', hausdorff_ft, + 'symdiff_boundary_len_ft', symdiff_boundary_len_ft, + 'symdiff_fragment_count', symdiff_fragment_count, + 'max_ring_eccentricity', max_ring_eccentricity, + 'perimeter_pct_diff', perimeter_pct_diff + ) AS changes, + 'gdb_nypuma2020' AS output_file_id, + CASE + WHEN array_length(failing_checks, 1) = 1 THEN failing_checks[1] + ELSE '' + END AS diff_group, + '' AS subgroup, + 'key_value' AS comparison_column, + 'gdb_nypuma2020' AS build_table_name, + 'fgdb_nypuma2020' AS production_table_name, + false AS accounted_for +FROM {{ ref('qa__geometry_fgdb_nypuma2020') }} +WHERE NOT passes diff --git a/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_geometry_fgdb_district_layers.sql b/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_geometry_fgdb_district_layers.sql new file mode 100644 index 0000000000..92cb3ce497 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_geometry_fgdb_district_layers.sql @@ -0,0 +1,39 @@ +{{ + config( + materialized='table', + tags=['on_demand', 'diffs'] + ) +}} + +-- Standard diff-summary shape over qa__geometry_fgdb_district_layers - see that model +-- and macros/geometry_diff_qa.sql for the pattern/caveats (chat log 2026-10-04/05). +-- One model covering all 34 layers in qa__geometry_fgdb_district_layers, output_file_id/ +-- build_table_name/production_table_name derived from source_layer per row rather than +-- one wrapper file per layer. Only non-passing rows are included. +SELECT + key_value AS comparison_id, + CASE + WHEN only_in_build THEN 'only_in_build' + WHEN only_in_prod THEN 'only_in_prod' + ELSE 'modified' + END AS status, + jsonb_build_object( + 'symdiff_area_pct', symdiff_area_pct, + 'hausdorff_ft', hausdorff_ft, + 'symdiff_boundary_len_ft', symdiff_boundary_len_ft, + 'symdiff_fragment_count', symdiff_fragment_count, + 'max_ring_eccentricity', max_ring_eccentricity, + 'perimeter_pct_diff', perimeter_pct_diff + ) AS changes, + 'gdb_' || source_layer AS output_file_id, + CASE + WHEN array_length(failing_checks, 1) = 1 THEN failing_checks[1] + ELSE '' + END AS diff_group, + '' AS subgroup, + 'key_value' AS comparison_column, + 'gdb_' || source_layer AS build_table_name, + 'fgdb_' || source_layer AS production_table_name, + false AS accounted_for +FROM {{ ref('qa__geometry_fgdb_district_layers') }} +WHERE NOT passes diff --git a/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_geometry_fgdb_district_layers_cheap.sql b/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_geometry_fgdb_district_layers_cheap.sql new file mode 100644 index 0000000000..1abe5021cc --- /dev/null +++ b/products/cscl/models/etl_dev_qa/diffs/districts/qa__diffs_geometry_fgdb_district_layers_cheap.sql @@ -0,0 +1,34 @@ +{{ + config( + materialized='table', + tags=['on_demand', 'diffs'] + ) +}} + +-- Standard diff-summary shape over qa__geometry_fgdb_district_layers_cheap - see that +-- model, qa__diffs_geometry_fgdb_district_layers.sql (the full-check equivalent), and +-- macros/geometry_diff_qa.sql's geometry_diff_qa_cheap for the pattern/caveats (chat +-- log 2026-10-05). Only non-passing rows are included. +SELECT + key_value AS comparison_id, + CASE + WHEN only_in_build THEN 'only_in_build' + WHEN only_in_prod THEN 'only_in_prod' + ELSE 'modified' + END AS status, + jsonb_build_object( + 'perimeter_pct_diff', perimeter_pct_diff, + 'area_pct_diff', area_pct_diff + ) AS changes, + 'gdb_' || source_layer AS output_file_id, + CASE + WHEN array_length(failing_checks, 1) = 1 THEN failing_checks[1] + ELSE '' + END AS diff_group, + '' AS subgroup, + 'key_value' AS comparison_column, + 'gdb_' || source_layer AS build_table_name, + 'fgdb_' || source_layer AS production_table_name, + false AS accounted_for +FROM {{ ref('qa__geometry_fgdb_district_layers_cheap') }} +WHERE NOT passes diff --git a/products/cscl/models/etl_dev_qa/diffs/qa__diffs_all.sql b/products/cscl/models/etl_dev_qa/diffs/qa__diffs_all.sql index ba22c2cadd..417f117e97 100644 --- a/products/cscl/models/etl_dev_qa/diffs/qa__diffs_all.sql +++ b/products/cscl/models/etl_dev_qa/diffs/qa__diffs_all.sql @@ -56,4 +56,18 @@ FROM ( SELECT * FROM {{ ref('qa__diffs_exception') }} UNION ALL SELECT * FROM {{ ref('qa__diffs_fgdb_altnames') }} + UNION ALL + SELECT * FROM {{ ref('qa__diffs_fgdb_nycb2010') }} + UNION ALL + SELECT * FROM {{ ref('qa__diffs_fgdb_nyfb') }} + UNION ALL + SELECT * FROM {{ ref('qa__diffs_fgdb_nycd') }} + UNION ALL + SELECT * FROM {{ ref('qa__diffs_fgdb_nypuma2010') }} + UNION ALL + SELECT * FROM {{ ref('qa__diffs_fgdb_nypuma2020') }} + UNION ALL + SELECT * FROM {{ ref('qa__diffs_geometry_fgdb_district_layers') }} + UNION ALL + SELECT * FROM {{ ref('qa__diffs_geometry_fgdb_district_layers_cheap') }} ) AS unioned_diffs diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/_district_boundaries_qa.yml b/products/cscl/models/etl_dev_qa/district_boundaries/_district_boundaries_qa.yml new file mode 100644 index 0000000000..b08bbbf751 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/_district_boundaries_qa.yml @@ -0,0 +1,336 @@ +version: 2 + +models: +- name: qa__boundary__ct2010_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__cb2010_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__puma2010_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__fd_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__ct2020_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__nta2020_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__cdta2020_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__fb_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__fc_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__puma2020_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__nta2010_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__mcea_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__ha_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__hc_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__ad_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__ed_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__sd_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__cd_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__bb_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__cb2020_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__hez_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__cc_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__cg_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__ss_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__mc_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__ct2010wi_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__ct2020wi_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__cb2010wi_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__cb2020wi_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__bbwi_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__cdwi_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__adwi_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__edwi_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__ccwi_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__cgwi_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__sswi_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn +- name: qa__boundary__mcwi_validity + columns: + - name: is_valid + tests: + - accepted_values: + arguments: + values: [true] + config: + severity: warn diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ad_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ad_validity.sql new file mode 100644 index 0000000000..98e0f56315 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ad_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per assembly district (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__ad_edges', 'int__boundary__ad_raw', 'int__boundary__ad') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ad_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ad_vs_prod.sql new file mode 100644 index 0000000000..a93a548ab5 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ad_vs_prod.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- int__boundary__ad vs production_outputs.fgdb_nyad. + +{{ district_boundary_vs_prod('int__boundary__ad', 'qa__boundary__ad_validity', 'fgdb_nyad', 'assemdist', 'ad') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__adwi_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__adwi_validity.sql new file mode 100644 index 0000000000..c205e40635 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__adwi_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per assembly district (water included). + +{{ district_boundary_validity('int__boundary__adwi_edges', 'int__boundary__adwi_raw', 'int__boundary__adwi') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__adwi_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__adwi_vs_prod.sql new file mode 100644 index 0000000000..610837246d --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__adwi_vs_prod.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- int__boundary__adwi vs production_outputs.fgdb_nyadwi. + +{{ district_boundary_vs_prod('int__boundary__adwi', 'qa__boundary__adwi_validity', 'fgdb_nyadwi', 'assemdist', 'ad') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__bb_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__bb_validity.sql new file mode 100644 index 0000000000..d26d012e95 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__bb_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per borough (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__bb_edges', 'int__boundary__bb_raw', 'int__boundary__bb') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__bb_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__bb_vs_prod.sql new file mode 100644 index 0000000000..0d635986e8 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__bb_vs_prod.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- int__boundary__bb vs production_outputs.fgdb_nybb. + +{{ district_boundary_vs_prod('int__boundary__bb', 'qa__boundary__bb_validity', 'fgdb_nybb', 'borocode', 'bb') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__bbwi_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__bbwi_validity.sql new file mode 100644 index 0000000000..0c5f740239 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__bbwi_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per borough (water included). + +{{ district_boundary_validity('int__boundary__bbwi_edges', 'int__boundary__bbwi_raw', 'int__boundary__bbwi') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__bbwi_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__bbwi_vs_prod.sql new file mode 100644 index 0000000000..ae00b876de --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__bbwi_vs_prod.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- int__boundary__bbwi vs production_outputs.fgdb_nybbwi. + +{{ district_boundary_vs_prod('int__boundary__bbwi', 'qa__boundary__bbwi_validity', 'fgdb_nybbwi', 'borocode', 'bb') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2010_212_pair.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2010_212_pair.sql new file mode 100644 index 0000000000..77c63c2ebc --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2010_212_pair.sql @@ -0,0 +1,69 @@ +{{ config(materialized='view') }} + +-- Census blocks 10212003000 / 10212004002 (Manhattan), where the published +-- fgdb_nycb2010 is not the union of the AtomicPolygons tagged with each block: two APs +-- each cross prod's dividing line into the other block. Kept as a reference case for +-- the prod-side block/AP divergence. Layers, by `layer`: +-- prod_cb - prod's census block polygon +-- ap - each AtomicPolygon, with the block its own attribute says +-- ap_spillover - the part of an AP lying inside a DIFFERENT prod block than its +-- attribute says (prod_cb = the block it actually falls in) + +WITH pair AS ( + SELECT unnest(ARRAY['10212003000', '10212004002']) AS bctcb2010 +), + +prod AS ( + SELECT + p.bctcb2010, + linearize(p.shape) AS geom + FROM {{ adapter.get_relation(database="db-cscl", schema="production_outputs", identifier="fgdb_nycb2010") }} AS p + INNER JOIN pair ON p.bctcb2010 = pair.bctcb2010 +), + +aps AS ( + SELECT + m.atomicid, + m.bctcb2010, + a.geom + FROM {{ ref('int__topology__ap_entities') }} AS m + INNER JOIN pair ON m.bctcb2010 = pair.bctcb2010 + INNER JOIN {{ ref('stg__atomicpolygons') }} AS a ON m.atomicid = a.atomicid +), + +layers AS ( + SELECT + 'prod_cb' AS layer, + NULL AS atomicid, + NULL AS ap_cb, + bctcb2010 AS prod_cb, + geom + FROM prod + UNION ALL + SELECT + 'ap' AS layer, + atomicid, + bctcb2010 AS ap_cb, + NULL AS prod_cb, + geom + FROM aps + UNION ALL + SELECT + 'ap_spillover' AS layer, + a.atomicid, + a.bctcb2010 AS ap_cb, + p.bctcb2010 AS prod_cb, + st_intersection(a.geom, p.geom) AS geom + FROM aps AS a + INNER JOIN prod AS p ON a.bctcb2010 != p.bctcb2010 AND st_intersects(a.geom, p.geom) +) + +SELECT + layer, + atomicid, + ap_cb, + prod_cb, + round(st_area(geom)::numeric, 1) AS area_sqft, + st_multi(st_collectionextract(geom, 3))::GEOMETRY (MULTIPOLYGON, 2263) AS geom +FROM layers +WHERE st_area(geom) > 1 diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2010_outliers.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2010_outliers.sql new file mode 100644 index 0000000000..4a2b150adc --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2010_outliers.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- 2010 census blocks that differ from prod, broken into viewing layers. + +{{ district_boundary_outliers('cb2010', 'bctcb2010', 'fgdb_nycb2010', 'bctcb2010') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2010_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2010_validity.sql new file mode 100644 index 0000000000..a7d4710002 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2010_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per 2010 census block (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__cb2010_edges', 'int__boundary__cb2010_raw', 'int__boundary__cb2010') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2010_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2010_vs_prod.sql new file mode 100644 index 0000000000..9c36fdc3f6 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2010_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__cb2010 vs production_outputs.fgdb_nycb2010. + +{{ district_boundary_vs_prod( + 'int__boundary__cb2010', + 'qa__boundary__cb2010_validity', + 'fgdb_nycb2010', + 'bctcb2010', + 'bctcb2010' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2010wi_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2010wi_validity.sql new file mode 100644 index 0000000000..a8c44099a9 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2010wi_validity.sql @@ -0,0 +1,9 @@ +{{ config(materialized='table') }} + +-- Topology checks per 2010 census block (water included). + +{{ district_boundary_validity( + 'int__boundary__cb2010wi_edges', + 'int__boundary__cb2010wi_raw', + 'int__boundary__cb2010wi' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2010wi_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2010wi_vs_prod.sql new file mode 100644 index 0000000000..c89d556bf6 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2010wi_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__cb2010wi vs production_outputs.fgdb_nycb2010wi. + +{{ district_boundary_vs_prod( + 'int__boundary__cb2010wi', + 'qa__boundary__cb2010wi_validity', + 'fgdb_nycb2010wi', + 'bctcb2010', + 'bctcb2010' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2020_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2020_validity.sql new file mode 100644 index 0000000000..c2934b5087 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2020_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per 2020 census block (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__cb2020_edges', 'int__boundary__cb2020_raw', 'int__boundary__cb2020') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2020_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2020_vs_prod.sql new file mode 100644 index 0000000000..dcd98a41c3 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2020_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__cb2020 vs production_outputs.fgdb_nycb2020. + +{{ district_boundary_vs_prod( + 'int__boundary__cb2020', + 'qa__boundary__cb2020_validity', + 'fgdb_nycb2020', + 'bctcb2020', + 'cb2020' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2020wi_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2020wi_validity.sql new file mode 100644 index 0000000000..0b21a55b95 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2020wi_validity.sql @@ -0,0 +1,9 @@ +{{ config(materialized='table') }} + +-- Topology checks per 2020 census block (water included). + +{{ district_boundary_validity( + 'int__boundary__cb2020wi_edges', + 'int__boundary__cb2020wi_raw', + 'int__boundary__cb2020wi' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2020wi_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2020wi_vs_prod.sql new file mode 100644 index 0000000000..14958e9ef4 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cb2020wi_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__cb2020wi vs production_outputs.fgdb_nycb2020wi. + +{{ district_boundary_vs_prod( + 'int__boundary__cb2020wi', + 'qa__boundary__cb2020wi_validity', + 'fgdb_nycb2020wi', + 'bctcb2020', + 'cb2020' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cc_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cc_validity.sql new file mode 100644 index 0000000000..b1ec03d647 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cc_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per city council district (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__cc_edges', 'int__boundary__cc_raw', 'int__boundary__cc') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cc_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cc_vs_prod.sql new file mode 100644 index 0000000000..c2456d47e6 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cc_vs_prod.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- int__boundary__cc vs production_outputs.fgdb_nycc. + +{{ district_boundary_vs_prod('int__boundary__cc', 'qa__boundary__cc_validity', 'fgdb_nycc', 'coundist', 'cc') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ccwi_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ccwi_validity.sql new file mode 100644 index 0000000000..e68d43ad9e --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ccwi_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per city council district (water included). + +{{ district_boundary_validity('int__boundary__ccwi_edges', 'int__boundary__ccwi_raw', 'int__boundary__ccwi') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ccwi_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ccwi_vs_prod.sql new file mode 100644 index 0000000000..0c1e16c266 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ccwi_vs_prod.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- int__boundary__ccwi vs production_outputs.fgdb_nyccwi. + +{{ district_boundary_vs_prod('int__boundary__ccwi', 'qa__boundary__ccwi_validity', 'fgdb_nyccwi', 'coundist', 'cc') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cd_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cd_validity.sql new file mode 100644 index 0000000000..70d9437e2c --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cd_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per community district (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__cd_edges', 'int__boundary__cd_raw', 'int__boundary__cd') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cd_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cd_vs_prod.sql new file mode 100644 index 0000000000..31f2436bd5 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cd_vs_prod.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- int__boundary__cd vs production_outputs.fgdb_nycd. + +{{ district_boundary_vs_prod('int__boundary__cd', 'qa__boundary__cd_validity', 'fgdb_nycd', 'borocd', 'cd') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cdta2020_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cdta2020_validity.sql new file mode 100644 index 0000000000..4ccb911cab --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cdta2020_validity.sql @@ -0,0 +1,9 @@ +{{ config(materialized='table') }} + +-- Topology checks per 2020 CDTA (is_valid must be true for every row). + +{{ district_boundary_validity( + 'int__boundary__cdta2020_edges', + 'int__boundary__cdta2020_raw', + 'int__boundary__cdta2020' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cdta2020_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cdta2020_vs_prod.sql new file mode 100644 index 0000000000..1943987e09 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cdta2020_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__cdta2020 vs production_outputs.fgdb_nycdta2020. + +{{ district_boundary_vs_prod( + 'int__boundary__cdta2020', + 'qa__boundary__cdta2020_validity', + 'fgdb_nycdta2020', + 'cdta2020', + 'cdta2020' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cdwi_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cdwi_validity.sql new file mode 100644 index 0000000000..b08ec3d26d --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cdwi_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per community district (water included). + +{{ district_boundary_validity('int__boundary__cdwi_edges', 'int__boundary__cdwi_raw', 'int__boundary__cdwi') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cdwi_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cdwi_vs_prod.sql new file mode 100644 index 0000000000..8ffac653b9 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cdwi_vs_prod.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- int__boundary__cdwi vs production_outputs.fgdb_nycdwi. + +{{ district_boundary_vs_prod('int__boundary__cdwi', 'qa__boundary__cdwi_validity', 'fgdb_nycdwi', 'borocd', 'cd') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cg_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cg_validity.sql new file mode 100644 index 0000000000..5ed44506b5 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cg_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per congressional district (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__cg_edges', 'int__boundary__cg_raw', 'int__boundary__cg') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cg_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cg_vs_prod.sql new file mode 100644 index 0000000000..ccbf2326d8 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cg_vs_prod.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- int__boundary__cg vs production_outputs.fgdb_nycg. + +{{ district_boundary_vs_prod('int__boundary__cg', 'qa__boundary__cg_validity', 'fgdb_nycg', 'congdist', 'cg') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cgwi_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cgwi_validity.sql new file mode 100644 index 0000000000..c31bd4d18f --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cgwi_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per congressional district (water included). + +{{ district_boundary_validity('int__boundary__cgwi_edges', 'int__boundary__cgwi_raw', 'int__boundary__cgwi') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cgwi_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cgwi_vs_prod.sql new file mode 100644 index 0000000000..546cf101e0 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__cgwi_vs_prod.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- int__boundary__cgwi vs production_outputs.fgdb_nycgwi. + +{{ district_boundary_vs_prod('int__boundary__cgwi', 'qa__boundary__cgwi_validity', 'fgdb_nycgwi', 'congdist', 'cg') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2010_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2010_validity.sql new file mode 100644 index 0000000000..7c6267abaa --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2010_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per 2010 census tract (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__ct2010_edges', 'int__boundary__ct2010_raw', 'int__boundary__ct2010') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2010_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2010_vs_prod.sql new file mode 100644 index 0000000000..7b5aab7568 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2010_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__ct2010 vs production_outputs.fgdb_nyct2010. + +{{ district_boundary_vs_prod( + 'int__boundary__ct2010', + 'qa__boundary__ct2010_validity', + 'fgdb_nyct2010', + 'boroct2010', + 'boroct' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2010wi_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2010wi_validity.sql new file mode 100644 index 0000000000..4ad594a456 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2010wi_validity.sql @@ -0,0 +1,9 @@ +{{ config(materialized='table') }} + +-- Topology checks per 2010 census tract (water included). + +{{ district_boundary_validity( + 'int__boundary__ct2010wi_edges', + 'int__boundary__ct2010wi_raw', + 'int__boundary__ct2010wi' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2010wi_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2010wi_vs_prod.sql new file mode 100644 index 0000000000..0b4c7257d1 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2010wi_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__ct2010wi vs production_outputs.fgdb_nyct2010wi. + +{{ district_boundary_vs_prod( + 'int__boundary__ct2010wi', + 'qa__boundary__ct2010wi_validity', + 'fgdb_nyct2010wi', + 'boroct2010', + 'boroct' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2020_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2020_validity.sql new file mode 100644 index 0000000000..28e6dd8b11 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2020_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per 2020 census tract (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__ct2020_edges', 'int__boundary__ct2020_raw', 'int__boundary__ct2020') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2020_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2020_vs_prod.sql new file mode 100644 index 0000000000..a0806132b5 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2020_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__ct2020 vs production_outputs.fgdb_nyct2020. + +{{ district_boundary_vs_prod( + 'int__boundary__ct2020', + 'qa__boundary__ct2020_validity', + 'fgdb_nyct2020', + 'boroct2020', + 'boroct2020' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2020wi_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2020wi_validity.sql new file mode 100644 index 0000000000..e564dc0c3a --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2020wi_validity.sql @@ -0,0 +1,9 @@ +{{ config(materialized='table') }} + +-- Topology checks per 2020 census tract (water included). + +{{ district_boundary_validity( + 'int__boundary__ct2020wi_edges', + 'int__boundary__ct2020wi_raw', + 'int__boundary__ct2020wi' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2020wi_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2020wi_vs_prod.sql new file mode 100644 index 0000000000..9572602b58 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ct2020wi_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__ct2020wi vs production_outputs.fgdb_nyct2020wi. + +{{ district_boundary_vs_prod( + 'int__boundary__ct2020wi', + 'qa__boundary__ct2020wi_validity', + 'fgdb_nyct2020wi', + 'boroct2020', + 'boroct2020' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ed_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ed_validity.sql new file mode 100644 index 0000000000..f2baa0f286 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ed_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per election district (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__ed_edges', 'int__boundary__ed_raw', 'int__boundary__ed') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ed_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ed_vs_prod.sql new file mode 100644 index 0000000000..5a0bdd19e2 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ed_vs_prod.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- int__boundary__ed vs production_outputs.fgdb_nyed. + +{{ district_boundary_vs_prod('int__boundary__ed', 'qa__boundary__ed_validity', 'fgdb_nyed', 'electdist', 'ed') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__edwi_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__edwi_validity.sql new file mode 100644 index 0000000000..7dc2a9e363 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__edwi_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per election district (water included). + +{{ district_boundary_validity('int__boundary__edwi_edges', 'int__boundary__edwi_raw', 'int__boundary__edwi') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__edwi_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__edwi_vs_prod.sql new file mode 100644 index 0000000000..e82979bcc9 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__edwi_vs_prod.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- int__boundary__edwi vs production_outputs.fgdb_nyedwi. + +{{ district_boundary_vs_prod('int__boundary__edwi', 'qa__boundary__edwi_validity', 'fgdb_nyedwi', 'electdist', 'ed') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__fb_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__fb_validity.sql new file mode 100644 index 0000000000..5d2943cbd9 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__fb_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per fire battalion (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__fb_edges', 'int__boundary__fb_raw', 'int__boundary__fb') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__fb_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__fb_vs_prod.sql new file mode 100644 index 0000000000..b6f4e08966 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__fb_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__fb vs production_outputs.fgdb_nyfb. + +{{ district_boundary_vs_prod( + 'int__boundary__fb', + 'qa__boundary__fb_validity', + 'fgdb_nyfb', + "firebn::text", + 'fire_battalion' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__fc_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__fc_validity.sql new file mode 100644 index 0000000000..bfd8a7c26f --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__fc_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per fire company (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__fc_edges', 'int__boundary__fc_raw', 'int__boundary__fc') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__fc_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__fc_vs_prod.sql new file mode 100644 index 0000000000..464b7167cd --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__fc_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__fc vs production_outputs.fgdb_nyfc. + +{{ district_boundary_vs_prod( + 'int__boundary__fc', + 'qa__boundary__fc_validity', + 'fgdb_nyfc', + "firecotype || ' ' || fireconum::text", + 'fire_company' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__fd_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__fd_validity.sql new file mode 100644 index 0000000000..8f681c016f --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__fd_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per fire division (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__fd_edges', 'int__boundary__fd_raw', 'int__boundary__fd') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__fd_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__fd_vs_prod.sql new file mode 100644 index 0000000000..e3fe601919 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__fd_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__fd vs production_outputs.fgdb_nyfd. + +{{ district_boundary_vs_prod( + 'int__boundary__fd', + 'qa__boundary__fd_validity', + 'fgdb_nyfd', + 'firediv::text', + 'fire_division' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ha_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ha_validity.sql new file mode 100644 index 0000000000..32b86dd49f --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ha_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per health area (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__ha_edges', 'int__boundary__ha_raw', 'int__boundary__ha') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ha_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ha_vs_prod.sql new file mode 100644 index 0000000000..09050d627f --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ha_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__ha vs production_outputs.fgdb_nyha. + +{{ district_boundary_vs_prod( + 'int__boundary__ha', + 'qa__boundary__ha_validity', + 'fgdb_nyha', + "borocode::text || lpad(healtharea::text, 4, '0')", + 'health_area' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__hc_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__hc_validity.sql new file mode 100644 index 0000000000..c4b205a23c --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__hc_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per health center district (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__hc_edges', 'int__boundary__hc_raw', 'int__boundary__hc') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__hc_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__hc_vs_prod.sql new file mode 100644 index 0000000000..ab8dcca350 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__hc_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__hc vs production_outputs.fgdb_nyhc. + +{{ district_boundary_vs_prod( + 'int__boundary__hc', + 'qa__boundary__hc_validity', + 'fgdb_nyhc', + "hcentdist::text", + 'health_center' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__hez_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__hez_validity.sql new file mode 100644 index 0000000000..13717dcbfe --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__hez_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per hurricane evacuation zone (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__hez_edges', 'int__boundary__hez_raw', 'int__boundary__hez') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__hez_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__hez_vs_prod.sql new file mode 100644 index 0000000000..c474c10da8 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__hez_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__hez vs production_outputs.fgdb_nyhez. + +{{ district_boundary_vs_prod( + 'int__boundary__hez', + 'qa__boundary__hez_validity', + 'fgdb_nyhez', + 'hurricane_evacuation_zone', + 'hez' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__mc_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__mc_validity.sql new file mode 100644 index 0000000000..0ba86b9635 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__mc_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per municipal court district (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__mc_edges', 'int__boundary__mc_raw', 'int__boundary__mc') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__mc_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__mc_vs_prod.sql new file mode 100644 index 0000000000..9fc91925be --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__mc_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__mc vs production_outputs.fgdb_nymc. + +{{ district_boundary_vs_prod( + 'int__boundary__mc', + 'qa__boundary__mc_validity', + 'fgdb_nymc', + 'borocode::text || p.municourt', + 'mc' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__mcea_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__mcea_validity.sql new file mode 100644 index 0000000000..13912340d0 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__mcea_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per MCEA (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__mcea_edges', 'int__boundary__mcea_raw', 'int__boundary__mcea') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__mcea_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__mcea_vs_prod.sql new file mode 100644 index 0000000000..8e642972a4 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__mcea_vs_prod.sql @@ -0,0 +1,13 @@ +{{ config(materialized='table') }} + +-- int__boundary__mcea vs production_outputs.fgdb_nymcea, which stores one feature per +-- disjoint piece (122 features for 115 areas) - dissolved per key before comparing. + +{{ district_boundary_vs_prod( + 'int__boundary__mcea', + 'qa__boundary__mcea_validity', + 'fgdb_nymcea', + 'mcea', + 'mcea', + dissolve_prod=true +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__mcwi_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__mcwi_validity.sql new file mode 100644 index 0000000000..bc6b091346 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__mcwi_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per municipal court district (water included). + +{{ district_boundary_validity('int__boundary__mcwi_edges', 'int__boundary__mcwi_raw', 'int__boundary__mcwi') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__mcwi_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__mcwi_vs_prod.sql new file mode 100644 index 0000000000..4019a2a465 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__mcwi_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__mcwi vs production_outputs.fgdb_nymcwi. + +{{ district_boundary_vs_prod( + 'int__boundary__mcwi', + 'qa__boundary__mcwi_validity', + 'fgdb_nymcwi', + 'borocode::text || p.municourt', + 'mc' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__nta2010_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__nta2010_validity.sql new file mode 100644 index 0000000000..2e7febc463 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__nta2010_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per 2010 NTA (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__nta2010_edges', 'int__boundary__nta2010_raw', 'int__boundary__nta2010') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__nta2010_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__nta2010_vs_prod.sql new file mode 100644 index 0000000000..b43170c48d --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__nta2010_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__nta2010 vs production_outputs.fgdb_nynta2010. + +{{ district_boundary_vs_prod( + 'int__boundary__nta2010', + 'qa__boundary__nta2010_validity', + 'fgdb_nynta2010', + "ntacode", + 'nta2010' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__nta2020_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__nta2020_validity.sql new file mode 100644 index 0000000000..936ccba785 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__nta2020_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per 2020 NTA (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__nta2020_edges', 'int__boundary__nta2020_raw', 'int__boundary__nta2020') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__nta2020_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__nta2020_vs_prod.sql new file mode 100644 index 0000000000..d3f427edee --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__nta2020_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__nta2020 vs production_outputs.fgdb_nynta2020. + +{{ district_boundary_vs_prod( + 'int__boundary__nta2020', + 'qa__boundary__nta2020_validity', + 'fgdb_nynta2020', + 'nta2020', + 'nta2020' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__puma2010_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__puma2010_validity.sql new file mode 100644 index 0000000000..7147eed32d --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__puma2010_validity.sql @@ -0,0 +1,9 @@ +{{ config(materialized='table') }} + +-- Topology checks per 2010 PUMA (is_valid must be true for every row). + +{{ district_boundary_validity( + 'int__boundary__puma2010_edges', + 'int__boundary__puma2010_raw', + 'int__boundary__puma2010' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__puma2010_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__puma2010_vs_prod.sql new file mode 100644 index 0000000000..aa9312bafb --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__puma2010_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__puma2010 vs production_outputs.fgdb_nypuma2010. + +{{ district_boundary_vs_prod( + 'int__boundary__puma2010', + 'qa__boundary__puma2010_validity', + 'fgdb_nypuma2010', + 'puma', + 'puma' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__puma2020_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__puma2020_validity.sql new file mode 100644 index 0000000000..194b7073ee --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__puma2020_validity.sql @@ -0,0 +1,9 @@ +{{ config(materialized='table') }} + +-- Topology checks per 2020 PUMA (is_valid must be true for every row). + +{{ district_boundary_validity( + 'int__boundary__puma2020_edges', + 'int__boundary__puma2020_raw', + 'int__boundary__puma2020' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__puma2020_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__puma2020_vs_prod.sql new file mode 100644 index 0000000000..800d1b70b2 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__puma2020_vs_prod.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +-- int__boundary__puma2020 vs production_outputs.fgdb_nypuma2020. + +{{ district_boundary_vs_prod( + 'int__boundary__puma2020', + 'qa__boundary__puma2020_validity', + 'fgdb_nypuma2020', + "puma", + 'puma2020' +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__sd_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__sd_validity.sql new file mode 100644 index 0000000000..6b7f9f04e5 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__sd_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per school district (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__sd_edges', 'int__boundary__sd_raw', 'int__boundary__sd') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__sd_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__sd_vs_prod.sql new file mode 100644 index 0000000000..c4d2891387 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__sd_vs_prod.sql @@ -0,0 +1,13 @@ +{{ config(materialized='table') }} + +-- int__boundary__sd vs production_outputs.fgdb_nysd, which stores SD 10 once per +-- borough (33 rows, 32 districts) - dissolved per key before comparing. + +{{ district_boundary_vs_prod( + 'int__boundary__sd', + 'qa__boundary__sd_validity', + 'fgdb_nysd', + 'schooldist', + 'sd', + dissolve_prod=true +) }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ss_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ss_validity.sql new file mode 100644 index 0000000000..41a8dae2db --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ss_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per state senate district (is_valid must be true for every row). + +{{ district_boundary_validity('int__boundary__ss_edges', 'int__boundary__ss_raw', 'int__boundary__ss') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ss_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ss_vs_prod.sql new file mode 100644 index 0000000000..ea8170caf6 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__ss_vs_prod.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- int__boundary__ss vs production_outputs.fgdb_nyss. + +{{ district_boundary_vs_prod('int__boundary__ss', 'qa__boundary__ss_validity', 'fgdb_nyss', 'stsendist', 'ss') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__sswi_validity.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__sswi_validity.sql new file mode 100644 index 0000000000..aac2599df1 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__sswi_validity.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- Topology checks per state senate district (water included). + +{{ district_boundary_validity('int__boundary__sswi_edges', 'int__boundary__sswi_raw', 'int__boundary__sswi') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__sswi_vs_prod.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__sswi_vs_prod.sql new file mode 100644 index 0000000000..45dd13853a --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__boundary__sswi_vs_prod.sql @@ -0,0 +1,5 @@ +{{ config(materialized='table') }} + +-- int__boundary__sswi vs production_outputs.fgdb_nysswi. + +{{ district_boundary_vs_prod('int__boundary__sswi', 'qa__boundary__sswi_validity', 'fgdb_nysswi', 'stsendist', 'ss') }} diff --git a/products/cscl/models/etl_dev_qa/district_boundaries/qa__topology__arc_disagreements.sql b/products/cscl/models/etl_dev_qa/district_boundaries/qa__topology__arc_disagreements.sql new file mode 100644 index 0000000000..b9aaf19252 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/district_boundaries/qa__topology__arc_disagreements.sql @@ -0,0 +1,67 @@ +{{ config(materialized='table') }} + +-- Checks int__topology__edges' arc keying. An arc is keyed only on its endpoints (+ +-- is_arc) and all owners share one canonical curve - right when owners trace the same +-- arc with different mid control points, wrong if two different arcs share both +-- endpoints. Per (arc key, owner): distance from the owner's own mid point to the +-- canonical curve. ~0 (quantization + linearization tolerance, < 0.001 ft) means same +-- arc; anything material means two curves collided on one key. + +WITH pt_nodes AS ( + SELECT + p.atomicid, + p.part, + p.ring, + p.seq, + p.is_arc_mid, + p.geom, + ptn.node_id + FROM {{ ref('int__topology__vertices') }} AS p + LEFT JOIN {{ ref('int__topology__exact_points') }} AS ec + ON p.geom = ec.geom AND NOT p.is_arc_mid + LEFT JOIN {{ ref('int__topology__point_to_node') }} AS ptn ON ec.point_id = ptn.exact_point_id +), + +mids AS ( + SELECT + atomicid, + geom AS mid_geom, + lag(node_id) OVER w AS node_a, + lead(node_id) OVER w AS node_b, + is_arc_mid + FROM pt_nodes + WINDOW w AS (PARTITION BY atomicid, part, ring ORDER BY seq) +), + +canonical AS ( + SELECT DISTINCT ON (node_lo, node_hi) + node_lo, + node_hi, + geom + FROM {{ ref('int__topology__edges') }} + WHERE is_arc + ORDER BY node_lo, node_hi, atomicid +), + +owners AS ( + SELECT + node_lo, + node_hi, + count(DISTINCT atomicid) AS n_owners + FROM {{ ref('int__topology__edges') }} + WHERE is_arc + GROUP BY node_lo, node_hi +) + +SELECT + c.node_lo, + c.node_hi, + o.n_owners, + m.atomicid, + round(st_distance(m.mid_geom, c.geom)::numeric, 6) AS mid_offset_ft, + round(st_length(c.geom)::numeric, 3) AS arc_length_ft +FROM mids AS m +INNER JOIN canonical AS c + ON least(m.node_a, m.node_b) = c.node_lo AND greatest(m.node_a, m.node_b) = c.node_hi +INNER JOIN owners AS o ON c.node_lo = o.node_lo AND c.node_hi = o.node_hi +WHERE m.is_arc_mid diff --git a/products/cscl/models/etl_dev_qa/qa__context_geoms.sql b/products/cscl/models/etl_dev_qa/qa__context_geoms.sql new file mode 100644 index 0000000000..d0faded9bd --- /dev/null +++ b/products/cscl/models/etl_dev_qa/qa__context_geoms.sql @@ -0,0 +1,316 @@ +{{ config(materialized='table', tags=['qa']) }} + +-- Every displayable geometry for a flagged row, one normalized table - the PRIMARY +-- pair (our own build output, and the legacy/prod geometry it's compared against) +-- plus any RELATED-but-distinct geometry that helps explain why the row was flagged +-- (e.g. a PUMA's member census tracts, when investigating a tract-dissolve +-- needle-sliver). The primary pair also lives as actual_geom_4326/expected_geom_4326 +-- columns directly on qa__geometry_fgdb_* - duplicated here too (not instead of) so a +-- future map UI (or anything else consuming this) can query ONE table +-- (layer, key_value, role) for everything about a flagged feature, rather than +-- switching schemas between "the row itself" and "things related to it" +-- (chat log 2026-10-06). NOT part of qa__diffs_all's lightweight summary rollup. +-- +-- role values so far: 'build' (our own output), 'legacy' (the prod/legacy geometry +-- it's compared against), 'constituent_tract' (a PUMA's member tracts), and four +-- roles splitting ST_Difference of build vs legacy into REAL defects vs noise (chat +-- log 2026-10-06, see below for why that split exists and how it's computed): +-- 'build_spur' - real area in build, not in legacy (e.g. a self-touching-vertex +-- artifact exposed by clip_to_shoreline) +-- 'legacy_spur' - real area in legacy, not in build +-- 'build_sliver' - noise: area in build not in legacy, but within +-- context_geom_noise_buffer_ft of the true boundary (tract-dissolve +-- needle retrace, not a real defect) +-- 'legacy_sliver' - noise, same idea, area in legacy not in build +-- build_spur + build_sliver together reconstruct the full raw ST_Difference(build, +-- legacy) (same for legacy_spur + legacy_sliver) - the split is exhaustive, not a +-- partial filter that drops anything silently. +-- +-- What counts as "related" differs per layer - a clip-based layer's water-mask pieces +-- or a different layer's neighboring features, later - `role` is meant to be filtered +-- on directly by a consuming front end, so name any new one the same clear, +-- self-explanatory way rather than an abbreviation that needs a lookup. +-- One row per individual related geometry, not one row per role with everything +-- collapsed into a single multi-geometry - keeps each feature individually +-- addressable (its own label, its own style/color) for a future map UI. build_spur/ +-- legacy_spur take this further and ST_Dump into one row per connected fragment (chat +-- log 2026-10-06 - "it's nice to know how many of them there are") rather than one +-- multi-geometry row per PUMA; build_sliver/legacy_sliver stay as a single combined +-- geometry per PUMA since there can be hundreds of noise fragments and the point of +-- showing them is "here's where the noise lives", not enumerating each one. +-- +-- Joins back to qa__geometry_fgdb_*/qa__diffs_all on (layer, key_value) - +-- key_value here matches those tables' key_value/comparison_id for the same layer +-- and feature. Geometry in EPSG:4326/WGS84, matching geometry_diff_qa's +-- actual_geom_4326/expected_geom_4326 convention - not this product's usual +-- EPSG:2263 internal working SRID. 'build'/'legacy' are stored at full fidelity in +-- 4326; the four spur/sliver roles are a separate, deliberately LOSSY computation - +-- NOT the same path test_cases.geometry_diff_metrics uses for the real +-- symdiff_area_pct etc. metrics (those stay on EPSG:2263, unaffected by anything here). +-- +-- build_spur/legacy_spur buffer the SUBTRACTED side by context_geom_noise_buffer_ft +-- (dbt_project.yml var, feet, computed on a 2263 round-trip since the buffer must be +-- in real-world units, not the 4326 degrees these geometries are otherwise stored in) +-- before differencing; build_sliver/legacy_sliver are what that buffer removes +-- (ST_Difference of the raw, unbuffered difference against the buffered one). Needed +-- because almost every symdiff fragment between build and legacy is tract-dissolve- +-- retrace needle noise that hugs the true boundary within a fraction of a foot NO +-- MATTER HOW LONG the needle is (confirmed on PUMA 3808, chat log 2026-10-06: max gap +-- 0.004 ft on a needle 2,838 ft long) - without the buffer, this table would be 99%+ +-- noise, defeating its purpose of helping a human spot the real defect. A genuine +-- spur/gap (same PUMA has one - a self-touching-vertex artifact reaching 104.5 ft from +-- the true boundary, on a SMALLER 501 ft perimeter than most of the noise needles) +-- survives the buffer because its extent, not its length, is what's real. +-- Perimeter/length was tested and rejected as the filter signal for exactly this +-- reason - it doesn't separate the two populations. +-- +-- label distinguishes individual rows within a role that can have more than one per +-- feature: constituent_tract's boroct id, and build_spur/legacy_spur's +-- "_" (fragment 1 is the largest by area, within that +-- PUMA/role). 'build'/'legacy'/'build_sliver'/'legacy_sliver' are always exactly one +-- row per feature, so label is just key_value there, not a meaningful sub-identity. +-- +-- PUMA (nypuma2010/nypuma2020) is the first populated case. +-- +-- Scoped to flagged PUMAs only (qa__geometry_fgdb_nypuma2010/2020 WHERE NOT passes) - +-- confirmed 2026-10-06: unscoped, constituent_tract alone stores all 55/57 PUMAs +-- citywide (4,495 rows) when only 6/4 are ever actually flagged. This table exists to +-- help explain WHY a flagged row differs, not to mirror every PUMA regardless of +-- whether anyone needs to look at it. + +WITH flagged_2010 AS ( + SELECT + key_value, + actual_geom_4326, + expected_geom_4326, + st_transform(actual_geom_4326, 2263) AS actual_geom_2263, + st_transform(expected_geom_4326, 2263) AS expected_geom_2263 + FROM {{ ref('qa__geometry_fgdb_nypuma2010') }} + WHERE NOT passes +), + +diffs_2010 AS ( + SELECT + key_value, + actual_geom_4326, + expected_geom_4326, + st_difference(actual_geom_2263, expected_geom_2263) AS raw_build_diff, + st_difference(expected_geom_2263, actual_geom_2263) AS raw_legacy_diff, + st_difference( + actual_geom_2263, + st_buffer(expected_geom_2263, {{ var('context_geom_noise_buffer_ft', 1.0) }}) + ) AS build_spur_geom, + st_difference( + expected_geom_2263, + st_buffer(actual_geom_2263, {{ var('context_geom_noise_buffer_ft', 1.0) }}) + ) AS legacy_spur_geom + FROM flagged_2010 + -- GEOS overlay ops on an invalid input produce meaningless garbage fragments, not a + -- real finding (confirmed: PUMA 4414/nypuma2020, inv-tv1.5 - an invalid legacy geom + -- alone produced 147 zero-area "legacy_spur" fragments). 'build'/'legacy' roles + -- still show the raw geometry regardless, so the invalidity itself stays visible. + WHERE st_isvalid(actual_geom_2263) AND st_isvalid(expected_geom_2263) +), + +flagged_2020 AS ( + SELECT + key_value, + actual_geom_4326, + expected_geom_4326, + st_transform(actual_geom_4326, 2263) AS actual_geom_2263, + st_transform(expected_geom_4326, 2263) AS expected_geom_2263 + FROM {{ ref('qa__geometry_fgdb_nypuma2020') }} + WHERE NOT passes +), + +diffs_2020 AS ( + SELECT + key_value, + actual_geom_4326, + expected_geom_4326, + st_difference(actual_geom_2263, expected_geom_2263) AS raw_build_diff, + st_difference(expected_geom_2263, actual_geom_2263) AS raw_legacy_diff, + st_difference( + actual_geom_2263, + st_buffer(expected_geom_2263, {{ var('context_geom_noise_buffer_ft', 1.0) }}) + ) AS build_spur_geom, + st_difference( + expected_geom_2263, + st_buffer(actual_geom_2263, {{ var('context_geom_noise_buffer_ft', 1.0) }}) + ) AS legacy_spur_geom + FROM flagged_2020 + -- see diffs_2010's identical comment - same guard, same PUMA 4414/inv-tv1.5 reason. + WHERE st_isvalid(actual_geom_2263) AND st_isvalid(expected_geom_2263) +), + +build_spur_frags_2010 AS ( + SELECT + key_value, + (st_dump(build_spur_geom)).geom AS fragment + FROM diffs_2010 +), + +legacy_spur_frags_2010 AS ( + SELECT + key_value, + (st_dump(legacy_spur_geom)).geom AS fragment + FROM diffs_2010 +), + +build_spur_frags_2020 AS ( + SELECT + key_value, + (st_dump(build_spur_geom)).geom AS fragment + FROM diffs_2020 +), + +legacy_spur_frags_2020 AS ( + SELECT + key_value, + (st_dump(legacy_spur_geom)).geom AS fragment + FROM diffs_2020 +) + +SELECT + 'nypuma2010' AS layer, + key_value, + 'build' AS role, + key_value AS label, + actual_geom_4326 AS geom +FROM flagged_2010 + +UNION ALL + +SELECT + 'nypuma2010' AS layer, + key_value, + 'legacy' AS role, + key_value AS label, + expected_geom_4326 AS geom +FROM flagged_2010 + +UNION ALL + +SELECT + 'nypuma2010' AS layer, + key_value, + 'build_spur' AS role, + key_value || '_' || row_number() OVER (PARTITION BY key_value ORDER BY st_area(fragment) DESC) AS label, + st_transform(fragment, 4326) AS geom +FROM build_spur_frags_2010 + +UNION ALL + +SELECT + 'nypuma2010' AS layer, + key_value, + 'legacy_spur' AS role, + key_value || '_' || row_number() OVER (PARTITION BY key_value ORDER BY st_area(fragment) DESC) AS label, + st_transform(fragment, 4326) AS geom +FROM legacy_spur_frags_2010 + +UNION ALL + +SELECT + 'nypuma2010' AS layer, + key_value, + 'build_sliver' AS role, + key_value AS label, + st_transform(st_difference(raw_build_diff, build_spur_geom), 4326) AS geom +FROM diffs_2010 +WHERE NOT st_isempty(st_difference(raw_build_diff, build_spur_geom)) + +UNION ALL + +SELECT + 'nypuma2010' AS layer, + key_value, + 'legacy_sliver' AS role, + key_value AS label, + st_transform(st_difference(raw_legacy_diff, legacy_spur_geom), 4326) AS geom +FROM diffs_2010 +WHERE NOT st_isempty(st_difference(raw_legacy_diff, legacy_spur_geom)) + +UNION ALL + +SELECT + 'nypuma2020' AS layer, + key_value, + 'build' AS role, + key_value AS label, + actual_geom_4326 AS geom +FROM flagged_2020 + +UNION ALL + +SELECT + 'nypuma2020' AS layer, + key_value, + 'legacy' AS role, + key_value AS label, + expected_geom_4326 AS geom +FROM flagged_2020 + +UNION ALL + +SELECT + 'nypuma2020' AS layer, + key_value, + 'build_spur' AS role, + key_value || '_' || row_number() OVER (PARTITION BY key_value ORDER BY st_area(fragment) DESC) AS label, + st_transform(fragment, 4326) AS geom +FROM build_spur_frags_2020 + +UNION ALL + +SELECT + 'nypuma2020' AS layer, + key_value, + 'legacy_spur' AS role, + key_value || '_' || row_number() OVER (PARTITION BY key_value ORDER BY st_area(fragment) DESC) AS label, + st_transform(fragment, 4326) AS geom +FROM legacy_spur_frags_2020 + +UNION ALL + +SELECT + 'nypuma2020' AS layer, + key_value, + 'build_sliver' AS role, + key_value AS label, + st_transform(st_difference(raw_build_diff, build_spur_geom), 4326) AS geom +FROM diffs_2020 +WHERE NOT st_isempty(st_difference(raw_build_diff, build_spur_geom)) + +UNION ALL + +SELECT + 'nypuma2020' AS layer, + key_value, + 'legacy_sliver' AS role, + key_value AS label, + st_transform(st_difference(raw_legacy_diff, legacy_spur_geom), 4326) AS geom +FROM diffs_2020 +WHERE NOT st_isempty(st_difference(raw_legacy_diff, legacy_spur_geom)) + +UNION ALL + +SELECT + 'nypuma2010' AS layer, + t.puma AS key_value, + 'constituent_tract' AS role, + t.boroct AS label, + st_transform(t.geom, 4326) AS geom +FROM {{ ref('stg__censustract2010') }} AS t +WHERE t.puma IN (SELECT key_value FROM flagged_2010) + +UNION ALL + +SELECT + 'nypuma2020' AS layer, + t.puma AS key_value, + 'constituent_tract' AS role, + t.boroct AS label, + st_transform(t.geom, 4326) AS geom +FROM {{ ref('stg__censustract2020') }} AS t +WHERE t.puma IN (SELECT key_value FROM flagged_2020) diff --git a/products/cscl/models/etl_dev_qa/qa__geometry_fgdb_district_layers.sql b/products/cscl/models/etl_dev_qa/qa__geometry_fgdb_district_layers.sql new file mode 100644 index 0000000000..a3c2ad2e49 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/qa__geometry_fgdb_district_layers.sql @@ -0,0 +1,114 @@ +{{ config(materialized='table', tags=['qa']) }} + +-- Combined full geometry_diff_qa (macros/geometry_diff_qa.sql) run across every +-- district_gdb layer small enough to run full-table (the per-row cost - ST_SymDifference +-- plus a simplified ST_HausdorffDistance - runs ~0.5-1.2s/row empirically, fine up to a +-- few thousand rows, not at nyap/nycb2010/nycb2010wi/nycb2020/nycb2020wi's 38k-70k rows, +-- which get the cheaper geometry_diff_qa_cheap instead - see +-- qa__geometry_fgdb_district_layers_cheap.sql). nycd/nyfb/nypuma2010/nypuma2020 already +-- had their own bespoke models before this one existed (chat log 2026-10-04/05) and are +-- left as-is rather than folded in here, to avoid touching working models without need. +-- +-- One combined table with a source_layer column, not 34 separate model files - same +-- underlying per-layer SQL (geometry_diff_qa expands to one full query per layer, +-- UNION ALL'd together), just far less file/boilerplate churn for a set this large +-- (chat log 2026-10-05). +-- +-- key_column follows each layer's declared key in seeds/lion_outputs.csv. A composite +-- (pipe-separated) declared key gets an explicit concat expression for both key_column +-- and prod_key_column - geometry_diff_qa's default prod_key_column inference (lower, +-- strip quotes) only works for a single bare column name. +-- +-- nyura deliberately excluded from the catalog below: prod's fgdb_nyura carries a +-- stale copy of nybid's schema (bidid/bid/borough, no uraid column at all) - a known +-- structural diff, not a fixable key mismatch (compare_gdb.py's KNOWN_STRUCTURAL_DIFFS, +-- Bug 013/inv-9k8). Prod's table is also empty (0 rows) - nothing meaningful to compare. +-- +-- DORMANT BY DEFAULT (chat log 2026-10-05): running the full catalog below took over +-- 10 minutes (33 layers, several thousand rows each at the slower mid-size layers - +-- nyct2010/wi, nyct2020/wi, nyed/wi), too slow for a routine build. `enabled` controls +-- which layers actually run - add a name to it to dive into that layer's geometry +-- shape QA, remove it when done. The full catalog stays here as reference so you don't +-- have to re-derive key columns/types from scratch each time. +{% set enabled = ['nycdta2020', 'nynta2010', 'nynta2020', 'nysd'] %} + +{% set all_layers = [ + {'name': 'nyad', 'key': '"AssemDist"'}, + {'name': 'nyadwi', 'key': '"AssemDist"'}, + {'name': 'nybb', 'key': '"BoroCode"'}, + {'name': 'nybbwi', 'key': '"BoroCode"'}, + {'name': 'nybid', 'key': '"BIDID"'}, + {'name': 'nycc', 'key': '"CounDist"'}, + {'name': 'nyccwi', 'key': '"CounDist"'}, + {'name': 'nycdta2020', 'key': '"CDTA2020"'}, + {'name': 'nycdwi', 'key': '"BoroCD"'}, + {'name': 'nycg', 'key': '"CongDist"'}, + {'name': 'nycgwi', 'key': '"CongDist"'}, + {'name': 'nyct2010', 'key': '"BoroCT2010"'}, + {'name': 'nyct2010wi', 'key': '"BoroCT2010"'}, + {'name': 'nyct2020', 'key': '"BoroCT2020"'}, + {'name': 'nyct2020wi', 'key': '"BoroCT2020"'}, + {'name': 'nyed', 'key': '"ElectDist"'}, + {'name': 'nyedwi', 'key': '"ElectDist"'}, + {'name': 'nyfc', 'key': '("FireCoType"::text || \'|\' || "FireCoNum"::text)', + 'prod_key': "(firecotype::text || '|' || fireconum::text)"}, + {'name': 'nyfd', 'key': '"FireDiv"'}, + {'name': 'nyha', 'key': '("BoroCode"::text || \'|\' || "HealthArea"::text)', + 'prod_key': "(borocode::text || '|' || healtharea::text)"}, + {'name': 'nyhc', 'key': '"HCentDist"'}, + {'name': 'nyhd', 'key': '"NAME"'}, + {'name': 'nyhez', 'key': '"HURRICANE_EVACUATION_ZONE"'}, + {'name': 'nymc', 'key': '("BoroCode"::text || \'|\' || "MuniCourt"::text)', + 'prod_key': "(borocode::text || '|' || municourt::text)"}, + {'name': 'nymcea', 'key': '("BOROCODE"::text || \'|\' || "MCEA"::text)', + 'prod_key': "(borocode::text || '|' || mcea::text)"}, + {'name': 'nymcwi', 'key': '("BoroCode"::text || \'|\' || "MuniCourt"::text)', + 'prod_key': "(borocode::text || '|' || municourt::text)"}, + {'name': 'nynta2010', 'key': '"NTACode"'}, + {'name': 'nynta2020', 'key': '"NTA2020"'}, + {'name': 'nypp', 'key': '"Precinct"'}, + {'name': 'nysd', 'key': '"SchoolDist"'}, + {'name': 'nyss', 'key': '"StSenDist"'}, + {'name': 'nysswi', 'key': '"StSenDist"'}, + {'name': 'nyzip', 'key': '"ZIP_CODE"'}, +] %} + +{% set layers = all_layers | selectattr('name', 'in', enabled) | list %} + +{% if layers | length > 0 %} + {% for l in layers %} + SELECT + '{{ l.name }}' AS source_layer, + * + FROM ( + {{ geometry_diff_qa( + build_relation=ref('gdb_' ~ l.name), + prod_relation=adapter.get_relation(database="db-cscl", schema="production_outputs", identifier="fgdb_" ~ l.name), + key_column=l.key, + prod_key_column=l.get('prod_key') + ) }} + ) AS t + {{ "UNION ALL" if not loop.last else "" }} + {% endfor %} +{% else %} +-- Nothing enabled - empty but correctly-shaped result so downstream models/tests +-- still build cleanly. Add a layer name to `enabled` above to activate it. +SELECT + NULL::text AS source_layer, + NULL::text AS key_value, + NULL::boolean AS only_in_prod, + NULL::boolean AS only_in_build, + NULL::geometry AS actual_geom_4326, + NULL::geometry AS expected_geom_4326, + NULL::double precision AS actual_area_sqft, + NULL::double precision AS expected_area_sqft, + NULL::double precision AS symdiff_area_pct, + NULL::double precision AS hausdorff_ft, + NULL::double precision AS symdiff_boundary_len_ft, + NULL::integer AS symdiff_fragment_count, + NULL::double precision AS max_ring_eccentricity, + NULL::double precision AS perimeter_pct_diff, + NULL::text[] AS failing_checks, + NULL::boolean AS passes +WHERE FALSE +{% endif %} diff --git a/products/cscl/models/etl_dev_qa/qa__geometry_fgdb_district_layers_cheap.sql b/products/cscl/models/etl_dev_qa/qa__geometry_fgdb_district_layers_cheap.sql new file mode 100644 index 0000000000..d47b553e24 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/qa__geometry_fgdb_district_layers_cheap.sql @@ -0,0 +1,32 @@ +{{ config(materialized='table', tags=['qa']) }} + +-- Combined geometry_diff_qa_cheap (macros/geometry_diff_qa.sql) run across the district +-- gdb layers too large for the full check's per-row ST_SymDifference/ST_HausdorffDistance +-- cost (~0.5-1.2s/row empirically -> 5-20+ hours each at these row counts): nyap +-- (69,786 rows), nycb2010/nycb2010wi (38,797/39,141), nycb2020/nycb2020wi +-- (37,588/37,984) - chat log 2026-10-05. See qa__geometry_fgdb_district_layers.sql for +-- every other district_gdb layer (full check) and geometry_diff_qa_cheap's own +-- docstring for exactly what this cheaper check does and doesn't catch. +-- nycb2010 is not listed: it has its own dedicated model (qa__diffs_fgdb_nycb2010), and +-- listing it here too double-reports every differing block in qa__diffs_all. +{% set layers = [ + {'name': 'nyap', 'key': '"ATOMICID"'}, + {'name': 'nycb2010wi', 'key': '"BCTCB2010"'}, + {'name': 'nycb2020', 'key': '"BCTCB2020"'}, + {'name': 'nycb2020wi', 'key': '"BCTCB2020"'}, +] %} + +{% for l in layers %} + SELECT + '{{ l.name }}' AS source_layer, + * + FROM ( + {{ geometry_diff_qa_cheap( + build_relation=ref('gdb_' ~ l.name), + prod_relation=adapter.get_relation(database="db-cscl", schema="production_outputs", identifier="fgdb_" ~ l.name), + key_column=l.key, + prod_key_column=l.get('prod_key') + ) }} + ) AS t + {{ "UNION ALL" if not loop.last else "" }} +{% endfor %} diff --git a/products/cscl/models/etl_dev_qa/qa__geometry_fgdb_nycd.sql b/products/cscl/models/etl_dev_qa/qa__geometry_fgdb_nycd.sql new file mode 100644 index 0000000000..4524de4175 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/qa__geometry_fgdb_nycd.sql @@ -0,0 +1,13 @@ +{{ config(materialized='table', tags=['qa']) }} + +-- Full-layer application of geometry_diff_qa (macros/geometry_diff_qa.sql) to nycd +-- (Community Districts) - the layer that originally motivated this whole geometry QA +-- effort (SI01's CDTA bug, chat log 2026-10-04, was Community District 1's 2020 +-- tabulation-area equivalent). An unfrozen, still-open layer (seeds/lion_outputs.csv: +-- "Shoreline-clip geometry noise (CSCL-DISTRICTS-03), not yet borough-bound"). + +{{ geometry_diff_qa( + build_relation=ref('gdb_nycd'), + prod_relation=adapter.get_relation(database="db-cscl", schema="production_outputs", identifier="fgdb_nycd"), + key_column='"BoroCD"' +) }} diff --git a/products/cscl/models/etl_dev_qa/qa__geometry_fgdb_nyfb.sql b/products/cscl/models/etl_dev_qa/qa__geometry_fgdb_nyfb.sql new file mode 100644 index 0000000000..1f5f9dea00 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/qa__geometry_fgdb_nyfb.sql @@ -0,0 +1,13 @@ +{{ config(materialized='table', tags=['qa']) }} + +-- Full-layer application of geometry_diff_qa (macros/geometry_diff_qa.sql) to nyfb +-- (Fire Battalions) - an unfrozen, still-open layer (seeds/lion_outputs.csv: +-- "Shoreline-clip geometry noise (CSCL-DISTRICTS-03), not yet borough-bound"), +-- picked to see how the new multi-metric check performs against a real layer beyond +-- the curated test_cases.examples fixtures. + +{{ geometry_diff_qa( + build_relation=ref('gdb_nyfb'), + prod_relation=adapter.get_relation(database="db-cscl", schema="production_outputs", identifier="fgdb_nyfb"), + key_column='"FireBN"' +) }} diff --git a/products/cscl/models/etl_dev_qa/qa__geometry_fgdb_nypuma2010.sql b/products/cscl/models/etl_dev_qa/qa__geometry_fgdb_nypuma2010.sql new file mode 100644 index 0000000000..6db1337f6d --- /dev/null +++ b/products/cscl/models/etl_dev_qa/qa__geometry_fgdb_nypuma2010.sql @@ -0,0 +1,12 @@ +{{ config(materialized='table', tags=['qa']) }} + +-- Full-layer application of geometry_diff_qa (macros/geometry_diff_qa.sql) to +-- nypuma2010. Motivated this layer's own needle-sliver and clip_to_shoreline_gridsize +-- investigation (inv-tv1.2, chat log 2026-10-05) - was previously only checked by +-- poc_validation/compare_gdb.py's SHAPE_Length/Area tolerance, with no in-db QA at all. + +{{ geometry_diff_qa( + build_relation=ref('gdb_nypuma2010'), + prod_relation=adapter.get_relation(database="db-cscl", schema="production_outputs", identifier="fgdb_nypuma2010"), + key_column='"PUMA"' +) }} diff --git a/products/cscl/models/etl_dev_qa/qa__geometry_fgdb_nypuma2020.sql b/products/cscl/models/etl_dev_qa/qa__geometry_fgdb_nypuma2020.sql new file mode 100644 index 0000000000..c5a1d49b77 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/qa__geometry_fgdb_nypuma2020.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table', tags=['qa']) }} + +-- Full-layer application of geometry_diff_qa (macros/geometry_diff_qa.sql) to +-- nypuma2020 - same motivation/history as qa__geometry_fgdb_nypuma2010 (inv-tv1.2, +-- chat log 2026-10-05). + +{{ geometry_diff_qa( + build_relation=ref('gdb_nypuma2020'), + prod_relation=adapter.get_relation(database="db-cscl", schema="production_outputs", identifier="fgdb_nypuma2020"), + key_column='"PUMA"' +) }} diff --git a/products/cscl/models/etl_dev_qa/qa__test_cases_geometry.sql b/products/cscl/models/etl_dev_qa/qa__test_cases_geometry.sql new file mode 100644 index 0000000000..2828815f52 --- /dev/null +++ b/products/cscl/models/etl_dev_qa/qa__test_cases_geometry.sql @@ -0,0 +1,95 @@ +{{ config(materialized='table', tags=['qa']) }} + +-- Runs test_cases.geometry_diff_metrics (macros/create_pg_functions.sql) against every +-- row in test_cases.examples - our TDD fixture table for known-weird geometries (see +-- products/cscl chat log, 2026-10-04). A row "passes" when none of its metrics exceed +-- their threshold; data_tests in _etl_dev_qa.yml asserts every row passes, same as any +-- other test suite - this model is expected to go red when a new weird case is added +-- (or an existing fix regresses) and green once the underlying geometry issue is +-- actually fixed, not a one-time snapshot. +-- +-- Thresholds are dbt vars (override with --vars rather than editing this file), picked +-- from exactly two data points so far (chat log 2026-10-04) - expect to retune as more +-- examples land across layers of different scale: +-- symdiff_area_pct 1% - generous; real hairline/overlay noise should be +-- far below this, a genuinely missing/extra chunk of +-- land should be far above it. +-- hausdorff_ft / symdiff_boundary_len_ft 50 / 100 ft - the one real example measured +-- (337.57 / 770.31) is far past either; small enough +-- that minor coastline noise on a huge borough-scale +-- feature could plausibly need a looser, layer-aware +-- threshold later. +-- max_ring_eccentricity 50 - compact real polygons measured in the single digits +-- to low tens so far; the needle-sliver test case +-- measured 2885.76, two orders of magnitude past this. +-- perimeter_pct_diff 1% - raw, unfiltered SHAPE_Length comparison, added +-- 2026-10-05 because symdiff_area_pct/ +-- symdiff_boundary_len_ft only sum fragments >= +-- min_fragment_sqft (100 sqft default) and every +-- needle-sliver fragment found so far is under that - +-- those two metrics are structurally blind to this +-- defect class at production thresholds. Confirmed on +-- PUMA 3701 (chat log 2026-10-05): symdiff_area_pct=0, +-- symdiff_fragment_count=0, but perimeter_pct_diff=22% +-- (matches the real SHAPE_Length gap vs prod exactly). +-- A metric that's NULL (no expected_geom to compare against) is never treated as a +-- failure on its own - see geometry_diff_metrics' docstring for when that happens. + +WITH examples AS ( + SELECT * FROM {{ adapter.get_relation( + database="db-cscl", schema="test_cases", identifier="examples" + ) }} +), + +metrics AS ( + SELECT + e.slug, + e.label, + e.category, + e.layer, + e.key_column, + e.key_value, + e.status, + m.actual_area_sqft, + m.expected_area_sqft, + m.symdiff_area_pct, + m.hausdorff_ft, + m.symdiff_boundary_len_ft, + m.max_ring_eccentricity, + m.perimeter_pct_diff + FROM examples AS e + CROSS JOIN LATERAL test_cases.geometry_diff_metrics(e.actual_geom, e.expected_geom) AS m +), + +flagged AS ( + SELECT + *, + array_remove(ARRAY[ + CASE + WHEN symdiff_area_pct > {{ var('geom_check_symdiff_area_pct', 1) }} + THEN 'symdiff_area_pct' + END, + CASE + WHEN hausdorff_ft > {{ var('geom_check_hausdorff_ft', 50) }} + THEN 'hausdorff_ft' + END, + CASE + WHEN symdiff_boundary_len_ft > {{ var('geom_check_symdiff_boundary_len_ft', 100) }} + THEN 'symdiff_boundary_len_ft' + END, + CASE + WHEN max_ring_eccentricity > {{ var('geom_check_max_ring_eccentricity', 50) }} + THEN 'max_ring_eccentricity' + END, + CASE + WHEN abs(perimeter_pct_diff) > {{ var('geom_check_perimeter_pct', 1) }} + THEN 'perimeter_pct_diff' + END + ], NULL) AS failing_metrics + FROM metrics +) + +SELECT + *, + (array_length(failing_metrics, 1) IS NULL) AS passes +FROM flagged diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/_atomicpolygon_topology.yml b/products/cscl/models/intermediate/atomicpolygon_topology/_atomicpolygon_topology.yml new file mode 100644 index 0000000000..3fe6b64097 --- /dev/null +++ b/products/cscl/models/intermediate/atomicpolygon_topology/_atomicpolygon_topology.yml @@ -0,0 +1,41 @@ +version: 2 + +models: +- name: int__topology__ap_entities + description: One row per in-scope AtomicPolygon with one membership column per district type. + columns: + - name: atomicid + tests: [ unique, not_null ] +- name: int__topology__ap_police + description: Police precinct, patrol borough and sector per AtomicPolygon (polygon at the centroid, or majority of area where the centroid falls outside the AP), shared by ThinLION. + columns: + - name: atomicid + tests: [ unique, not_null ] +- name: int__topology__vertices + description: Ring vertices of every in-scope AtomicPolygon in ring order, from the raw curved geometry. +- name: int__topology__exact_points + description: Distinct vertex coordinates (arc mid control points excluded). + columns: + - name: point_id + tests: [ unique, not_null ] +- name: int__topology__grid_cells + description: Occupied grid cells at the node merge tolerance. +- name: int__topology__point_to_node + description: Exact point to canonical node id. + columns: + - name: exact_point_id + tests: [ unique, not_null ] +- name: int__topology__nodes + description: Canonical topology nodes. + columns: + - name: node_id + tests: [ unique, not_null ] +- name: int__topology__edges_unnoded + description: Topology edges before splitting at crossings. +- name: int__topology__edge_crossings + description: Points where two straight topology edges properly cross (source AP overlaps), with new node ids. +- name: int__topology__edges + description: One row per (edge, owning AtomicPolygon), keyed by (node_lo, node_hi, is_arc); fully noded. + columns: + - name: atomicid + tests: [ not_null ] diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/_int_atomicpolygon_topology.yml b/products/cscl/models/intermediate/atomicpolygon_topology/_int_atomicpolygon_topology.yml deleted file mode 100644 index d9fd66d103..0000000000 --- a/products/cscl/models/intermediate/atomicpolygon_topology/_int_atomicpolygon_topology.yml +++ /dev/null @@ -1,110 +0,0 @@ -##### Atomic Polygon Topology POC ##### -# Points-based topology experiment: instead of atomic polygons carrying their own literal -# boundary coordinates (which drift apart by hairline amounts between adjacent polygons - -# see CSCL-DISTRICTS-01 in data_issues.md), explode every ring vertex, snap it to a shared -# "node" table, and rejoin. Adjacent polygons whose boundary vertices snap to the same node -# are then structurally guaranteed to share that coordinate, not just be close to it. - -models: -- name: int__atomicpolygon_vertices - description: Every ring vertex of every stg__atomicpolygons geometry, exploded via ST_DumpPoints. - columns: - - name: atomicid - tests: [ not_null ] - - name: path - - name: part_index - - name: ring_index - - name: point_ordinal - - name: geom - -- name: int__atomicpolygon_nodes - description: > - Canonical topology nodes - atomic polygon vertices grid-snapped and collapsed to one - node (centroid) per grid cell. See model SQL for how the grid size was picked. - columns: - - name: node_id - tests: [ unique, not_null ] - - name: grid_key - tests: [ unique, not_null ] - - name: geom - - name: vertex_count - - name: polygon_count - -- name: int__atomicpolygon_vertices_snapped - description: Each original vertex joined to its canonical node_id, for ring/polygon reconstruction. - columns: - - name: atomicid - tests: [ not_null ] - - name: node_id - tests: [ not_null ] - -- name: int__atomicpolygon_rings - description: Each ring rebuilt as a closed linestring of shared node coordinates. - columns: - - name: atomicid - tests: [ not_null ] - - name: part_index - - name: ring_index - - name: ring_geom - -- name: int__atomicpolygon_rebuilt - description: > - Atomic polygons reassembled from node-snapped rings. Adjacent polygons that snapped to - the same node now literally share that boundary coordinate; compare against - stg__atomicpolygons.geom, which this pipeline never touches. - columns: - - name: atomicid - tests: [ unique, not_null ] - - name: geom - -- name: int__atomicpolygon_edges - description: Every ring edge (consecutive node pair), keyed so the same physical edge traced by two adjacent polygons lands on the same key regardless of winding direction. - columns: - - name: atomicid - tests: [ not_null ] - - name: node_lo - tests: [ not_null ] - - name: node_hi - tests: [ not_null ] - -- name: int__atomicpolygon_shoreline_edges - description: > - Every distinct edge classified by ownership/water_flag - shoreline, internal_seam, - orphan_water_edge, orphan_land_edge, multi_owner. A GROUP BY, not a polygon overlay - - the alternative to finding gaps via ST_Union/ST_Difference sliver holes. - columns: - - name: node_lo - tests: [ not_null ] - - name: node_hi - tests: [ not_null ] - - name: edge_type - tests: - - not_null - - accepted_values: - arguments: - values: [ shoreline, internal_seam, orphan_water_edge, orphan_land_edge, multi_owner ] - - name: geom - -- name: int__atomicpolygon_edge_matches - description: > - Every orphan edge resolved against its nearest edge of ANY type/atomicid (including - other orphans - excluding those was the East River false-positive bug, see chat log - 2026-09-20). is_resolved distinguishes real bookkeeping artifacts (something sits - right on top of this edge, just unmatched) from genuine gap candidates. Classification - only - does not yet splice a host's missing vertex into its ring. - columns: - - name: node_lo - tests: [ not_null ] - - name: node_hi - tests: [ not_null ] - - name: is_resolved - tests: [ not_null ] - -- name: int__atomicpolygon_water_mask - description: > - POC comparison for int__water_mask - the water atomic polygons unioned from - int__atomicpolygon_rebuilt (node-snapped) instead of raw stg__atomicpolygons, no - buffer-closing trick, to test whether node-snapping alone closes the - CSCL-DISTRICTS-03 spur gaps. - columns: - - name: geom diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_edge_matches.sql b/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_edge_matches.sql deleted file mode 100644 index d6cfcb5f21..0000000000 --- a/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_edge_matches.sql +++ /dev/null @@ -1,62 +0,0 @@ -{{ config( - enabled = false, - materialized = 'table', - indexes=[ - {'columns': ['node_lo', 'node_hi']}, - {'columns': ['is_resolved']}, - ] -) }} - --- DISABLED (2026-10-02): the orphan-vs-any-edge nearest-neighbor search below ran for --- 20+ hours citywide and never finished (see products/cscl chat log, 2026-10-02/03) - --- the "East River false positive" fix (searching orphans against other orphans too, --- not just confirmed matches) means the `WHERE NOT (e.atomicids && o.atomicids)` filter --- sits outside what the GiST KNN index on `geom <-> geom` can use: for an orphan whose --- truly-nearest candidates are also excluded by the filter, the index scan has to keep --- pulling candidates further and further out before one survives. Needs a real fix --- (pre-filter candidates spatially before the KNN, or batch orphan clusters) before --- re-enabling, not just a bigger timeout. - --- Resolves orphan edges (from int__atomicpolygon_shoreline_edges) against the nearest --- edge belonging to a genuinely different atomicid, of ANY edge_type - including other --- orphans. Excluding other orphans from this search was the bug behind the East River --- false positives (see chat log, 2026-09-20): two adjacent water pieces that split a --- river don't node-match each other, so each side's edge is "orphan," but they sit right --- on top of each other - the true nearest neighbor for an orphan edge is very often --- another orphan, not a "confirmed match" edge_type. --- --- is_resolved (dist < {{ var('atomicpolygon_edge_match_tolerance', 5) }}ft) means: this --- edge is provably not a real gap - something else (of any kind) sits essentially on top --- of it, and the "orphan" label is just a vertex-bookkeeping artifact (mechanism --- B/C - vertex-density mismatch or a cross-atomicid seam), not a coverage hole. --- Unresolved edges are the genuine candidates worth investigating. --- --- This does NOT yet insert the host's missing vertex into the other ring - it only --- classifies. Turning a resolved match into an actual shared node (so graph traversal / --- ST_LineMerge can walk through it) is a further step: project this edge's endpoints --- onto the host edge and splice them into the host ring at the right position. -WITH orphans AS ( - SELECT node_lo, node_hi, atomicids, edge_type, geom - FROM {{ ref('int__atomicpolygon_shoreline_edges') }} - WHERE edge_type IN ('orphan_water_edge', 'orphan_land_edge') -) - -SELECT - o.node_lo, - o.node_hi, - o.atomicids, - o.edge_type, - nn.node_lo AS host_node_lo, - nn.node_hi AS host_node_hi, - nn.atomicids AS host_atomicids, - nn.edge_type AS host_edge_type, - nn.dist, - nn.dist < {{ var('atomicpolygon_edge_match_tolerance', 5) }} AS is_resolved -FROM orphans AS o -CROSS JOIN LATERAL ( - SELECT e.node_lo, e.node_hi, e.atomicids, e.edge_type, e.geom <-> o.geom AS dist - FROM {{ ref('int__atomicpolygon_shoreline_edges') }} AS e - WHERE NOT (e.atomicids && o.atomicids) - ORDER BY e.geom <-> o.geom - LIMIT 1 -) AS nn diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_edges.sql b/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_edges.sql deleted file mode 100644 index 1ed38cff9f..0000000000 --- a/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_edges.sql +++ /dev/null @@ -1,44 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['node_lo', 'node_hi']}, - {'columns': ['atomicid']}, - ] -) }} - --- Every ring edge (consecutive node pair) of every atomic polygon, keyed so the same --- physical edge traced by two adjacent polygons (in opposite winding order) lands on the --- same (node_lo, node_hi) key regardless of direction. This is the arc-node "edge" layer: --- classifying shoreline becomes a GROUP BY on this table (see --- int__atomicpolygon_shoreline_edges) instead of a polygon overlay (ST_Union/ST_Difference), --- which is what was producing hundreds of thousands of interior-ring holes on a plain --- land-minus-water dissolve (see products/cscl chat log, 2026-09-20). --- --- Rings are already closed by ST_DumpPoints (first point == last point), so LEAD() ordered --- by point_ordinal naturally produces exactly ring-length-minus-one edges with no --- wraparound case to special-case - the trailing NULL row (nothing after the closing point) --- is dropped. -WITH edges AS ( - SELECT - atomicid, - part_index, - ring_index, - point_ordinal, - node_id AS node_a, - lead(node_id) OVER ( - PARTITION BY atomicid, part_index, ring_index ORDER BY point_ordinal - ) AS node_b - FROM {{ ref('int__atomicpolygon_vertices_snapped') }} -) - -SELECT - atomicid, - part_index, - ring_index, - point_ordinal, - node_a, - node_b, - least(node_a, node_b) AS node_lo, - greatest(node_a, node_b) AS node_hi -FROM edges -WHERE node_b IS NOT NULL diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_nodes.sql b/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_nodes.sql deleted file mode 100644 index 7088064e6f..0000000000 --- a/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_nodes.sql +++ /dev/null @@ -1,40 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['grid_key']}, - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - --- Canonical "topology nodes": every atomic-polygon ring vertex, grid-snapped to --- {{ var('atomicpolygon_node_grid_size', 0.02) }}ft and collapsed to one node per grid --- cell, centered on the true centroid of whatever vertices landed in that cell. --- --- Deliberately a GROUP BY, not ST_ClusterDBSCAN: DBSCAN is a window function that has to --- hold its whole partition in memory and run GEOS clustering internals over it in one --- shot - not spatially partitionable, so at ~5M vertices citywide it's a real risk for --- the crashes/timeouts already seen with it elsewhere. A grid-snap + GROUP BY is a plain --- hash aggregate: deterministic, indexable, scales the boring way. --- --- Tolerance picked from int__atomicpolygon_vertices' actual nearest-neighbor distances --- (products/cscl chat log, 2026-09-20): ~37% of vertices already sit at literal --- floating-point-identical coordinates with their nearest different-polygon neighbor; of --- the rest, only ~3% fall between 0 and 0.03ft (the real "drift" band) before the --- distribution steepens sharply past 0.03-0.05ft into what's almost certainly distinct, --- unrelated vertices - not snapping candidates. 0.02ft sits inside that gap. -WITH grid_keys AS ( - SELECT - atomicid, - geom, - st_snaptogrid(geom, {{ var('atomicpolygon_node_grid_size', 0.02) }}) AS grid_key - FROM {{ ref('int__atomicpolygon_vertices') }} -) - -SELECT - row_number() OVER (ORDER BY grid_key) AS node_id, - grid_key, - st_centroid(st_collect(geom)) AS geom, - count(*) AS vertex_count, - count(DISTINCT atomicid) AS polygon_count -FROM grid_keys -GROUP BY grid_key diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_rebuilt.sql b/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_rebuilt.sql deleted file mode 100644 index 69cd08a1a7..0000000000 --- a/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_rebuilt.sql +++ /dev/null @@ -1,30 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['atomicid']}, - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - --- Reassembles each atomic polygon from its node-snapped rings (int__atomicpolygon_rings): --- exterior ring + any interior/hole rings per part, unioned back into a multipolygon per --- atomicid. This is the POC's payoff - geometries that share a canonical node at a --- boundary now literally touch there, not just sit close to it. Compare against --- stg__atomicpolygons.geom, which is untouched by any of this. -WITH parts AS ( - SELECT - atomicid, - part_index, - st_makepolygon( - (array_agg(ring_geom ORDER BY ring_index) FILTER (WHERE ring_index = 1))[1], - coalesce(array_agg(ring_geom ORDER BY ring_index) FILTER (WHERE ring_index > 1), ARRAY[]::geometry[]) - ) AS part_geom - FROM {{ ref('int__atomicpolygon_rings') }} - GROUP BY atomicid, part_index -) - -SELECT - atomicid, - st_makevalid(st_multi(st_collect(part_geom ORDER BY part_index))) AS geom -FROM parts -GROUP BY atomicid diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_rings.sql b/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_rings.sql deleted file mode 100644 index aa9ba87c54..0000000000 --- a/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_rings.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['atomicid']}, - ] -) }} - --- Rebuilds each ring (ring_index 1 = exterior, 2+ = interior/holes) as a closed --- linestring made of shared node coordinates instead of each polygon's own original --- vertices. Two atomic polygons that snapped to the same node at their shared boundary --- now produce rings that pass through the exact same coordinate there, not just a nearby --- one. Ring closure (first point == last point) falls out for free: ST_DumpPoints always --- emits the same original coordinate for a ring's first and last point, so they snap to --- the same node deterministically. -SELECT - atomicid, - part_index, - ring_index, - st_makeline(array_agg(node_geom ORDER BY point_ordinal)) AS ring_geom -FROM {{ ref('int__atomicpolygon_vertices_snapped') }} -GROUP BY atomicid, part_index, ring_index diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_shoreline_edges.sql b/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_shoreline_edges.sql deleted file mode 100644 index a3eb72ca43..0000000000 --- a/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_shoreline_edges.sql +++ /dev/null @@ -1,64 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['node_lo', 'node_hi']}, - {'columns': ['edge_type']}, - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - --- Classifies every unique edge (undirected node pair) by who owns it and on which side of --- water_flag. No polygon overlay - purely a GROUP BY over int__atomicpolygon_edges joined --- to water_flag, so it's cheap regardless of how big the underlying polygons are. --- --- shoreline - exactly 2 owning atomicids, one water_flag='1' and one not: this --- IS the coastline, read directly off the topology. --- internal_seam - exactly 2 owners, same side of water_flag: an ordinary interior --- boundary between two land parcels or two water polygons. --- orphan_water_edge - exactly 1 owner, water side: either a true open edge (coastline --- against nothing further, or the dataset boundary) or the --- vertex-density-mismatch case where the land side has no vertex --- at this node at all - the real "leftover water" candidates. --- orphan_land_edge - exactly 1 owner, land side: same idea, land-side orphan. --- multi_owner - more than 2 owners; unexpected for planar polygons, worth --- investigating on its own if it shows up at any real volume. -WITH edges AS ( - SELECT - e.node_lo, - e.node_hi, - e.atomicid, - a.water_flag - FROM {{ ref('int__atomicpolygon_edges') }} AS e - INNER JOIN {{ ref('stg__atomicpolygons') }} AS a ON e.atomicid = a.atomicid -), - -classified AS ( - SELECT - node_lo, - node_hi, - array_agg(DISTINCT atomicid) AS atomicids, - array_agg(DISTINCT water_flag) AS water_flags, - count(DISTINCT atomicid) AS owner_count, - bool_or(water_flag = '1') AS has_water_owner, - bool_or(water_flag <> '1') AS has_land_owner - FROM edges - GROUP BY node_lo, node_hi -) - -SELECT - c.node_lo, - c.node_hi, - c.atomicids, - c.water_flags, - c.owner_count, - CASE - WHEN c.owner_count = 2 AND c.has_water_owner AND c.has_land_owner THEN 'shoreline' - WHEN c.owner_count = 2 THEN 'internal_seam' - WHEN c.owner_count = 1 AND c.has_water_owner THEN 'orphan_water_edge' - WHEN c.owner_count = 1 THEN 'orphan_land_edge' - ELSE 'multi_owner' - END AS edge_type, - st_makeline(n_lo.geom, n_hi.geom) AS geom -FROM classified AS c -INNER JOIN {{ ref('int__atomicpolygon_nodes') }} AS n_lo ON c.node_lo = n_lo.node_id -INNER JOIN {{ ref('int__atomicpolygon_nodes') }} AS n_hi ON c.node_hi = n_hi.node_id diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_vertices.sql b/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_vertices.sql deleted file mode 100644 index bbec305722..0000000000 --- a/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_vertices.sql +++ /dev/null @@ -1,29 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['atomicid']}, - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - --- Every ring vertex of every atomic polygon, exploded out of stg__atomicpolygons.geom. --- ST_DumpPoints on a multipolygon returns path = [part_index, ring_index, point_ordinal]; --- ring_index 1 is the exterior ring, 2+ are interior rings (holes), and point_ordinal is --- the point's position within that ring (first == last, since rings are closed). --- POC for a points-based topology: see products/cscl chat log 2026-09-20. -WITH dumped AS ( - SELECT - atomicid, - (st_dumppoints(geom)).path AS path, - (st_dumppoints(geom)).geom AS geom - FROM {{ ref('stg__atomicpolygons') }} -) - -SELECT - atomicid, - path, - path[1] AS part_index, - path[2] AS ring_index, - path[3] AS point_ordinal, - geom -FROM dumped diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_vertices_snapped.sql b/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_vertices_snapped.sql deleted file mode 100644 index 97b47c3e59..0000000000 --- a/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_vertices_snapped.sql +++ /dev/null @@ -1,33 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['atomicid']}, - {'columns': ['node_id']}, - ] -) }} - --- Each original ring vertex resolved to its canonical topology node (int__atomicpolygon_nodes). --- This is the join that lets a ring/polygon be rebuilt from shared nodes instead of literal --- per-polygon coordinates - two atomic polygons whose boundary vertices snapped to the same --- node are now structurally guaranteed to share that exact coordinate, not just close to it. -WITH vertices AS ( - SELECT - atomicid, - part_index, - ring_index, - point_ordinal, - geom AS original_geom, - st_snaptogrid(geom, {{ var('atomicpolygon_node_grid_size', 0.02) }}) AS grid_key - FROM {{ ref('int__atomicpolygon_vertices') }} -) - -SELECT - v.atomicid, - v.part_index, - v.ring_index, - v.point_ordinal, - v.original_geom, - n.node_id, - n.geom AS node_geom -FROM vertices AS v -INNER JOIN {{ ref('int__atomicpolygon_nodes') }} AS n ON v.grid_key = n.grid_key diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_water_mask.sql b/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_water_mask.sql deleted file mode 100644 index ab5af581c8..0000000000 --- a/products/cscl/models/intermediate/atomicpolygon_topology/int__atomicpolygon_water_mask.sql +++ /dev/null @@ -1,25 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - --- POC comparison for int__water_mask.sql: same idea (union the water atomic polygons, --- subdivide for a selective GIST index), but sourced from int__atomicpolygon_rebuilt --- (node-snapped) instead of raw stg__atomicpolygons - no buffer-union-debuffer --- morphological-closing trick needed if node-snapping alone already closes the --- CSCL-DISTRICTS-03 hairline gaps (measured at ~0.00005-0.00017 ft between adjacent --- water atomic polygons - two to three orders of magnitude below the 0.02ft grid --- tolerance int__atomicpolygon_nodes was calibrated to, so well inside "definitely the --- same point" territory). See products/cscl chat log, 2026-09-20, for the MN31/SI12/ --- QN98/QN45 spur measurements this is being tested against. -WITH unioned AS ( - SELECT st_unaryunion(st_collect(r.geom)) AS geom - FROM {{ ref('int__atomicpolygon_rebuilt') }} AS r - INNER JOIN {{ ref('stg__atomicpolygons') }} AS a ON r.atomicid = a.atomicid - WHERE a.water_flag = '1' -) - -SELECT st_subdivide(geom, 256) AS geom -FROM unioned diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__ap_entities.sql b/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__ap_entities.sql new file mode 100644 index 0000000000..ea3e602759 --- /dev/null +++ b/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__ap_entities.sql @@ -0,0 +1,96 @@ +{{ config( + materialized='table', + indexes=[ + {'columns': ['atomicid'], 'unique': True}, + ] +) }} + +-- One row per in-scope AtomicPolygon, with one column per district type it belongs to. +-- This is the only place topology scope is decided and the only place membership +-- lives: the topology itself (vertices -> nodes -> edges) is keyed on atomicid and node +-- ids alone and is shared by every district type. Adding a district type means adding +-- a column here - by attribute (an AP column, or a lookup chain from one), never by +-- spatial join. NULL means the AP isn't assigned to any district of that type. +-- +-- Scope (ap_topology_scope_pumas OR ap_topology_scope_boros; default citywide) must be +-- a union of whole districts for every district type compared against prod - a +-- district cut by the scope edge builds as a fragment. + +{% set pumas = var('ap_topology_scope_pumas') %} +{% set boros = var('ap_topology_scope_boros') %} + +SELECT DISTINCT + a.atomicid, + a.water_flag, + -- for majority-by-area rollups (e.g. a district's borough) without geometry downstream + st_area(a.geom) AS area_sqft, + a.borocode, + -- direct AP attributes, typed to match the published layers' keys + a.borocode::int AS bb, + a.assemdist::int AS ad, + -- ED numbers repeat across assembly districts; AD * 1000 + ED is the published key + a.assemdist::int * 1000 + a.electdist::int AS ed, + nullif(a.schooldist, '')::int AS sd, + -- school district as published in fgdb_nysd: District 10 once per borough (Bronx + -- and Manhattan), every other district whole - including 9, 13, 14, 15 and 30, which + -- also have a few APs on the far side of a borough line + CASE + WHEN a.schooldist = '10' THEN '10-' || a.borocode + WHEN a.schooldist != '' THEN a.schooldist::int::text + END AS sd_boro, + nullif(a.commdist, '0')::int AS cd, + a.borocode || a.censustract_2020 || a.censusblock_2020_raw AS cb2020, + a.hurricane_evacuation_zone AS hez, + -- via the AP's election district (THINED roster, keyed AD + ED): each ED belongs to + -- exactly one council / congressional / state senate / municipal court district + thined.city_council_district::int AS cc, + thined.congress_district::int AS cg, + thined.state_sen_district::int AS ss, + -- municipal court numbers repeat per borough, so the id is borocode || court, as in + -- fgdb_nymc; '00' is the unassigned placeholder, excluded from the published layer + CASE WHEN thined.muni_court_district != '00' THEN thined.borough || thined.muni_court_district END AS mc, + ct.boroct, + -- same format as fgdb_nycb2010.bctcb2010: borocode(1) || ct2010(6) || cb2010(4) + a.borocode || a.censustract_2010 || a.censusblock_2010_raw AS bctcb2010, + ct.puma, + -- via the AP's administering fire company and the THINFIRE company roster; NULL + -- where admin_fire_company is '0' (unassigned islands/marshes, outside fgdb_nyfd) + fc.fire_division, + fc.fire_battalion, + fc.unit_short AS fire_company, + -- 2010 tract lookups + ct.neighborhood_code AS nta2010, + -- health areas follow 2010 tracts; codes repeat across boroughs, so the id is + -- borocode || 4-digit code. nullif: water-only tracts (990100) carry '' + a.borocode || lpad(nullif(ct.health_area, ''), 4, '0') AS health_area, + ha.health_ct_district AS health_center, + -- 2000 tract lookup + ct00.mcea, + -- 2020 hierarchy via the AP's 2020 tract: every tract has exactly one NTA and one + -- CDTA, NTAs nest in CDTAs, and the union of each NTA's / CDTA's tracts matches + -- the source NTA / CDTA polygons (verified 2026-10-09) + ct20.boroct AS boroct2020, + ct20.neighborhood_code AS nta2020, + ct20.cdta_code AS cdta2020, + ct20.puma AS puma2020 +FROM {{ ref('stg__atomicpolygons') }} AS a +INNER JOIN {{ ref('stg__censustract2010') }} AS ct + ON a.borocode = ct.borocode AND right(ct.boroct, 6) = a.censustract_2010 +LEFT JOIN {{ ref('stg__censustract2020') }} AS ct20 + ON a.borocode = ct20.borocode AND a.censustract_2020 = ct20.ct +LEFT JOIN {{ ref('stg__censustract2000') }} AS ct00 + ON a.borocode = ct00.borocode AND a.censustract_2000 = ct00.ct +LEFT JOIN {{ ref('stg__healtharea') }} AS ha + ON a.borocode = ha.borough AND ct.health_area = ha.healtharea +LEFT JOIN {{ ref('thined_by_field_unformatted') }} AS thined + ON a.electdist = thined.election_district AND a.assemdist = thined.assembly_district +LEFT JOIN {{ ref('thinfire_by_field_unformatted') }} AS fc + ON a.fire_company_type = fc.fire_company_type AND a.fire_company_number = fc.fire_company_number +WHERE + FALSE + {%- if pumas %} + OR ct.puma IN ({{ "'" ~ (pumas | join("','")) ~ "'" }}) + {%- endif %} + {%- if boros %} + OR a.borocode IN ({{ "'" ~ (boros | join("','")) ~ "'" }}) + {%- endif %} diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__ap_police.sql b/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__ap_police.sql new file mode 100644 index 0000000000..ca3b8d25f2 --- /dev/null +++ b/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__ap_police.sql @@ -0,0 +1,102 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['atomicid'], 'unique': True}] +) }} + +-- Police geography of every AtomicPolygon - precinct, patrol borough and sector (NYPD +-- beat) - computed once here and shared, so ThinLION and anything else reading an AP's +-- police attributes can't disagree. +-- +-- The rule reproduces prod's own per-AP assignment (verified against the published +-- ThinLION: 69,782 of 69,786 APs match on all three fields): +-- 1. If the AP's centroid lies inside the AP (68,204 APs), use the polygon containing +-- the centroid - an exact match with prod for every one of them. +-- 2. Otherwise - crescent, concave or multipart APs whose centroid falls outside them +-- (1,582 APs) - use the value holding the largest share of the AP's area (summed per +-- value: the beat layer has several polygons per sector). Here PostGIS's and the +-- legacy ETL's fallback points disagree, and majority area is what matches prod. +-- The 4 remaining mismatches are near-ties or slivers (e.g. 1024000006: 51% / 49%). +-- +-- NYPD's polygons don't follow AP edges (their lines cut through hundreds of APs), which +-- is why the choice of rule matters at all. method / *_share record how each AP was +-- assigned. NULL where an AP touches no polygon of that layer. + +{% set layers = [ + {'name': 'precinct', 'model': 'stg__nypdprecinct', 'id': 'precinct', 'out': 'police_precinct', 'globalid': 'nypdprecinct_globalid'}, + {'name': 'patrol_borough', 'model': 'stg__nypdpatrolborough', 'id': 'patrol_borough', 'out': 'patrol_borough', 'globalid': 'nypdpatrolborough_globalid'}, + {'name': 'sector', 'model': 'stg__nypdbeat', 'id': 'sector', 'out': 'police_sector', 'globalid': 'nypdbeat_globalid'}, +] %} + +WITH aps AS ( + SELECT + atomicid, + geom, + st_centroid(geom) AS centroid, + st_within(st_centroid(geom), geom) AS centroid_inside + FROM {{ ref('stg__atomicpolygons') }} +), + +{% for l in layers %} + {{ l.name }}_at_centroid AS ( + SELECT DISTINCT ON (a.atomicid) + a.atomicid, + p.{{ l.id }} AS value, + p.globalid + FROM aps AS a + INNER JOIN {{ ref(l.model) }} AS p ON st_intersects(p.geom, a.centroid) + WHERE a.centroid_inside + ORDER BY a.atomicid, p.globalid + ), + + {{ l.name }}_overlap AS ( + -- only APs whose centroid falls outside them; an AP wholly inside one polygon skips + -- the intersection + SELECT + a.atomicid, + p.{{ l.id }} AS value, + p.globalid, + CASE + WHEN st_covers(p.geom, a.geom) THEN st_area(a.geom) + ELSE st_area(st_intersection(a.geom, p.geom)) + END AS overlap_sqft + FROM aps AS a + INNER JOIN {{ ref(l.model) }} AS p ON st_intersects(a.geom, p.geom) + WHERE NOT a.centroid_inside + ), + + {{ l.name }}_by_value AS ( + SELECT + atomicid, + value, + sum(overlap_sqft) AS value_sqft, + (array_agg(globalid ORDER BY overlap_sqft DESC))[1] AS globalid + FROM {{ l.name }}_overlap + WHERE overlap_sqft > 0 + GROUP BY atomicid, value + ), + + {{ l.name }}_majority AS ( + SELECT DISTINCT ON (atomicid) + atomicid, + value, + globalid, + value_sqft / nullif(sum(value_sqft) OVER (PARTITION BY atomicid), 0) AS share + FROM {{ l.name }}_by_value + ORDER BY atomicid ASC, value_sqft DESC, value ASC + ){{ ',' if not loop.last }} + +{% endfor %} +SELECT + a.atomicid, + CASE WHEN a.centroid_inside THEN 'centroid' ELSE 'majority_area' END AS method, +{% for l in layers %} + coalesce({{ l.name }}_at_centroid.value, {{ l.name }}_majority.value) AS {{ l.out }}, + round({{ l.name }}_majority.share::numeric, 4) AS {{ l.out }}_share, + coalesce({{ l.name }}_at_centroid.globalid, {{ l.name }}_majority.globalid) AS {{ l.globalid }} + {{- ',' if not loop.last }} +{% endfor %} +FROM aps AS a +{% for l in layers %} + LEFT JOIN {{ l.name }}_at_centroid ON a.atomicid = {{ l.name }}_at_centroid.atomicid + LEFT JOIN {{ l.name }}_majority ON a.atomicid = {{ l.name }}_majority.atomicid +{% endfor %} diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__edge_crossings.sql b/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__edge_crossings.sql new file mode 100644 index 0000000000..8dc132953c --- /dev/null +++ b/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__edge_crossings.sql @@ -0,0 +1,107 @@ +{{ config(materialized='table') }} + +-- Every point where two straight topology edges properly cross (interiors intersect; +-- meeting at a shared node isn't a crossing), with a new node id for each point. A +-- crossing means two AtomicPolygons overlap slightly in the source - their shared +-- corner was digitized as two nodes further apart than the merge tolerance (e.g. APs +-- 4021400227 / 4021400220: 0.0023 ft, overlapping by 0.012 sq ft). Left unnoded, +-- ST_BuildArea can't polygonize the edges around it and the whole district builds +-- empty. int__topology__edges splits both edges at the point. +-- +-- Arcs aren't checked; no arc crossing has been observed. + +-- A crossing comes from two AtomicPolygons overlapping instead of sharing an edge, so +-- at least one edge of every crossing pair has a single owner. Probing from those +-- (~1k citywide) against all edges via the geom index keeps this cheap; probing every +-- edge against every neighbour is ~700k index lookups. +WITH one_owner AS ( + SELECT + node_lo, + node_hi, + (array_agg(geom))[1] AS geom + FROM {{ ref('int__topology__edges_unnoded') }} + WHERE NOT is_arc + GROUP BY node_lo, node_hi + HAVING count(DISTINCT atomicid) = 1 +), + +candidate_pairs AS ( + SELECT + a.node_lo AS a_lo, + a.node_hi AS a_hi, + b.node_lo AS b_lo, + b.node_hi AS b_hi, + a.geom AS a_geom, + b.geom AS b_geom + FROM one_owner AS a + INNER JOIN {{ ref('int__topology__edges_unnoded') }} AS b + ON a.geom && b.geom AND st_crosses(a.geom, b.geom) + WHERE NOT b.is_arc +), + +-- each crossing once, whichever side it was found from +pairs AS ( + SELECT DISTINCT ON (a_lo, a_hi, b_lo, b_hi) + a_lo, + a_hi, + b_lo, + b_hi, + a_geom, + b_geom + FROM ( + SELECT + a_lo, + a_hi, + b_lo, + b_hi, + a_geom, + b_geom + FROM candidate_pairs + WHERE (a_lo, a_hi) < (b_lo, b_hi) + UNION ALL + SELECT + b_lo, + b_hi, + a_lo, + a_hi, + b_geom, + a_geom + FROM candidate_pairs + WHERE (b_lo, b_hi) < (a_lo, a_hi) + ) AS ordered +), + +points AS ( + SELECT + p.a_lo, + p.a_hi, + p.b_lo, + p.b_hi, + d.geom + FROM pairs AS p + CROSS JOIN LATERAL st_dump(st_intersection(p.a_geom, p.b_geom)) AS d + WHERE st_geometrytype(d.geom) = 'ST_Point' +), + +numbered AS ( + SELECT + *, + (SELECT max(node_id) FROM {{ ref('int__topology__nodes') }}) + + dense_rank() OVER (ORDER BY st_x(geom), st_y(geom)) AS node_id + FROM points +) + +-- one row per (crossed edge, crossing point) +SELECT + a_lo AS node_lo, + a_hi AS node_hi, + node_id, + geom +FROM numbered +UNION ALL +SELECT + b_lo AS node_lo, + b_hi AS node_hi, + node_id, + geom +FROM numbered diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__edges.sql b/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__edges.sql new file mode 100644 index 0000000000..be7b2c9d62 --- /dev/null +++ b/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__edges.sql @@ -0,0 +1,97 @@ +{{ config( + materialized='table', + indexes=[ + {'columns': ['node_lo', 'node_hi', 'is_arc']}, + {'columns': ['atomicid']}, + ] +) }} + +-- The topology's edges, fully noded: int__topology__edges_unnoded with every straight +-- edge that crosses another (int__topology__edge_crossings) split at its crossing +-- points. Each piece keeps the parent edge's owners, so the two edges' shared stretch +-- becomes one matched edge and the sliver beyond the crossing is left bounded by +-- one-owner edges, which district_boundary_strip_noise_rings removes. Crossing nodes +-- are new node ids that don't appear in int__topology__nodes. +-- +-- Same shape as int__topology__edges_unnoded: one row per (edge, owning atomicid), +-- keyed by (node_lo, node_hi, is_arc). + +WITH unnoded AS ( + SELECT * FROM {{ ref('int__topology__edges_unnoded') }} +), + +crossed AS ( + SELECT + e.*, + row_number() OVER () AS edge_row + FROM unnoded AS e + WHERE + NOT e.is_arc + AND EXISTS ( + SELECT 1 FROM {{ ref('int__topology__edge_crossings') }} AS x + WHERE x.node_lo = e.node_lo AND x.node_hi = e.node_hi + ) +), + +-- every point along a crossed edge, in order: its two end nodes and its crossing points +edge_points AS ( + SELECT + c.edge_row, + 0.0 AS frac, + CASE WHEN st_equals(st_startpoint(c.geom), lo.geom) THEN c.node_lo ELSE c.node_hi END AS node_id, + st_startpoint(c.geom) AS geom + FROM crossed AS c + INNER JOIN {{ ref('int__topology__nodes') }} AS lo ON c.node_lo = lo.node_id + UNION ALL + SELECT + c.edge_row, + 1.0 AS frac, + CASE WHEN st_equals(st_startpoint(c.geom), lo.geom) THEN c.node_hi ELSE c.node_lo END AS node_id, + st_endpoint(c.geom) AS geom + FROM crossed AS c + INNER JOIN {{ ref('int__topology__nodes') }} AS lo ON c.node_lo = lo.node_id + UNION ALL + SELECT + c.edge_row, + st_linelocatepoint(c.geom, x.geom) AS frac, + x.node_id, + x.geom + FROM crossed AS c + INNER JOIN {{ ref('int__topology__edge_crossings') }} AS x + ON c.node_lo = x.node_lo AND c.node_hi = x.node_hi +), + +pieces AS ( + SELECT + edge_row, + node_id AS node_a, + lead(node_id) OVER w AS node_b, + st_makeline(geom, lead(geom) OVER w) AS geom + FROM edge_points + WINDOW w AS (PARTITION BY edge_row ORDER BY frac) +) + +SELECT + node_lo, + node_hi, + is_arc, + geom, + atomicid, + water_flag +FROM unnoded AS e +WHERE + NOT EXISTS ( + SELECT 1 FROM crossed AS c + WHERE c.node_lo = e.node_lo AND c.node_hi = e.node_hi AND NOT e.is_arc + ) +UNION ALL +SELECT + least(p.node_a, p.node_b) AS node_lo, + greatest(p.node_a, p.node_b) AS node_hi, + FALSE AS is_arc, + p.geom, + c.atomicid, + c.water_flag +FROM pieces AS p +INNER JOIN crossed AS c ON p.edge_row = c.edge_row +WHERE p.node_b IS NOT NULL AND p.node_a != p.node_b diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__edges_unnoded.sql b/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__edges_unnoded.sql new file mode 100644 index 0000000000..d4b71ab7af --- /dev/null +++ b/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__edges_unnoded.sql @@ -0,0 +1,294 @@ +{{ config( + materialized='table', + indexes=[ + {'columns': ['node_lo', 'node_hi', 'is_arc']}, + {'columns': ['atomicid']}, + {'columns': ['geom'], 'type': 'gist'}, + ] +) }} + +-- Before noding (int__topology__edges splits edges that cross - see that model). +-- +-- Every ring edge of every in-scope AtomicPolygon: one row per (edge, owning atomicid), +-- keyed by (node_lo, node_hi, is_arc) so the same physical edge traced by two adjacent +-- AtomicPolygons in opposite winding order lands on the same key. District-agnostic - +-- the district boundary layer (macros/district_boundary.sql) joins +-- int__topology__ap_entities on atomicid and classifies edges by who owns them. +-- +-- Arcs: an arc is ONE edge between its two endpoint nodes; its mid control point is not +-- a node (two AtomicPolygons tracing the same arc can pick different mids). is_arc is +-- part of the key, so a straight edge and an arc between the same two nodes stay +-- distinct with no tolerance involved; qa__topology__arc_disagreements flags the +-- (so far never observed) case of two different arcs sharing both endpoints. Each arc +-- key gets one canonical curve, shared bit-for-bit by all its owners. A full circle +-- (start == end) would collapse here; none exist in the source. +-- +-- Orphan edge splitting: a straight edge with only one owner is often its neighbour's +-- edge digitized with fewer vertices (the neighbour has extra, collinear intermediate +-- points). For each such orphan A-B, look for an alternate path of up to +-- ap_topology_edge_split_max_hops existing edges from A to B whose total length equals +-- the direct A-B distance within ap_topology_edge_split_tolerance_ft. By the triangle +-- inequality that's only possible if every intermediate node lies on the A-B line, so +-- this one scalar test is an exact collinearity check for any number of hops. A match +-- replaces the orphan's edge with the neighbour's sub-edges (same atomicid, existing +-- node keys - no inferred ownership), which is degree-neutral by construction. An orphan +-- with no such match (two AtomicPolygons genuinely disagreeing about a corner) stays an +-- orphan and is handled downstream (it reads as a boundary edge; any thin loop it +-- creates is stripped as noise). + +WITH RECURSIVE pts_with_nodes AS ( + SELECT + p.atomicid, + p.water_flag, + p.part, + p.ring, + p.seq, + p.is_arc_mid, + p.geom AS raw_geom, + n.node_id, + n.geom AS node_geom + FROM {{ ref('int__topology__vertices') }} AS p + LEFT JOIN {{ ref('int__topology__exact_points') }} AS ec + ON p.geom = ec.geom AND NOT p.is_arc_mid + LEFT JOIN {{ ref('int__topology__point_to_node') }} AS ptn ON ec.point_id = ptn.exact_point_id + LEFT JOIN {{ ref('int__topology__nodes') }} AS n ON ptn.node_id = n.node_id +), + +-- For each node point, whether the very next raw point is an arc mid (and its coords). +with_next AS ( + SELECT + *, + lead(is_arc_mid) OVER w AS next_is_mid, + lead(raw_geom) OVER w AS next_raw_geom + FROM pts_with_nodes + WINDOW w AS (PARTITION BY atomicid, part, ring ORDER BY seq) +), + +raw_edges AS ( + SELECT + atomicid, + water_flag, + node_id AS node_a, + lead(node_id) OVER w AS node_b, + node_geom AS geom_a, + lead(node_geom) OVER w AS geom_b, + coalesce(next_is_mid, FALSE) AS is_arc, + CASE WHEN next_is_mid THEN next_raw_geom END AS arc_mid + FROM with_next + WHERE NOT is_arc_mid + WINDOW w AS (PARTITION BY atomicid, part, ring ORDER BY seq) +), + +-- One canonical curve per arc key: the lowest-atomicid owner's mid control point, +-- through the canonical endpoint node coordinates, oriented lower -> higher node id, +-- linearized with the same parameters as public.linearize() (ST_CurveToLine, 0.00025 ft +-- max deviation) - i.e. the same way prod's layers are - with both ends snapped exactly +-- onto their node coordinates so ST_BuildArea sees them join. Every owner gets this +-- same geometry, bit for bit, regardless of winding or its own choice of mid. +arc_geoms AS ( + SELECT + node_lo, + node_hi, + st_setpoint(st_setpoint(line, 0, lo_geom), -1, hi_geom) AS geom + FROM ( + SELECT DISTINCT ON (node_lo, node_hi) + least(node_a, node_b) AS node_lo, + greatest(node_a, node_b) AS node_hi, + CASE WHEN node_a < node_b THEN geom_a ELSE geom_b END AS lo_geom, + CASE WHEN node_a < node_b THEN geom_b ELSE geom_a END AS hi_geom, + st_curvetoline( + st_setsrid(st_geomfromtext(format( + 'CIRCULARSTRING(%s %s, %s %s, %s %s)', + st_x(CASE WHEN node_a < node_b THEN geom_a ELSE geom_b END), + st_y(CASE WHEN node_a < node_b THEN geom_a ELSE geom_b END), + st_x(arc_mid), st_y(arc_mid), + st_x(CASE WHEN node_a < node_b THEN geom_b ELSE geom_a END), + st_y(CASE WHEN node_a < node_b THEN geom_b ELSE geom_a END) + )), st_srid(arc_mid)), + 0.00025, 1 + ) AS line + FROM raw_edges + WHERE is_arc AND node_b IS NOT NULL AND node_a != node_b + ORDER BY least(node_a, node_b), greatest(node_a, node_b), atomicid + ) AS canonical +), + +edges AS ( + SELECT + r.atomicid, + r.water_flag, + r.node_a, + r.node_b, + least(r.node_a, r.node_b) AS node_lo, + greatest(r.node_a, r.node_b) AS node_hi, + r.is_arc, + -- chord: what the orphan path search below measures (its triangle-inequality + -- test is about straight-line distance between the two nodes) + st_distance(r.geom_a, r.geom_b) AS chord_length, + coalesce(ag.geom, st_makeline(r.geom_a, r.geom_b)) AS geom + FROM raw_edges AS r + LEFT JOIN arc_geoms AS ag + ON r.is_arc AND least(r.node_a, r.node_b) = ag.node_lo AND greatest(r.node_a, r.node_b) = ag.node_hi + WHERE r.node_b IS NOT NULL AND r.node_a != r.node_b +), + +edge_owner_totals AS ( + SELECT + node_lo, + node_hi, + is_arc, + count(DISTINCT atomicid) AS total_owners + FROM edges + GROUP BY node_lo, node_hi, is_arc +), + +orphan_edges AS ( + SELECT + e.atomicid, + e.water_flag, + e.node_lo, + e.node_hi, + e.geom, + e.chord_length AS direct_length + FROM edges AS e + INNER JOIN edge_owner_totals AS t + ON e.node_lo = t.node_lo AND e.node_hi = t.node_hi AND e.is_arc = t.is_arc + -- straight edges only: the path-length match is a collinearity test, meaningless + -- for a curve + WHERE t.total_owners = 1 AND NOT e.is_arc +), + +-- Undirected `edges` as a directed adjacency list (both directions), so path-walking +-- below is a plain join. +directed_edges AS ( + SELECT + node_lo AS from_node, + node_hi AS to_node, + geom, + chord_length AS len + FROM edges + WHERE NOT is_arc + UNION ALL + SELECT + node_hi AS from_node, + node_lo AS to_node, + geom, + chord_length AS len + FROM edges + WHERE NOT is_arc +), + +-- Bounded path search: for each orphan A-B, walk other edges outward from A (never +-- revisiting a node, never taking the direct A-B edge as the very first hop - that +-- would just be re-walking the edge being tested) and stop extending any branch once +-- its cumulative length already exceeds the direct distance plus tolerance, since +-- length only accumulates hop over hop. This keeps the search tight regardless of the +-- hop cap: real estate boundaries don't have chains of near-zero-length edges, so a +-- detour that's genuinely the "same line, more vertices" burns through its length +-- budget in a handful of hops or not at all. +path_search ( + a, b, atomicid, water_flag, direct_length, cur_node, path_nodes, path_geoms, path_length, hops +) AS ( + SELECT + node_lo, + node_hi, + atomicid, + water_flag, + direct_length, + node_lo AS cur_node, + ARRAY[node_lo] AS path_nodes, + ARRAY[]::geometry[] AS path_geoms, + 0.0::double precision AS path_length, + 0 AS hops + FROM orphan_edges + + UNION ALL + + SELECT + ps.a, + ps.b, + ps.atomicid, + ps.water_flag, + ps.direct_length, + de.to_node, + ps.path_nodes || de.to_node, + ps.path_geoms || de.geom, + ps.path_length + de.len, + ps.hops + 1 + FROM path_search AS ps + INNER JOIN directed_edges AS de ON ps.cur_node = de.from_node + WHERE + ps.hops < {{ var('ap_topology_edge_split_max_hops') }} + AND NOT (de.to_node = any(ps.path_nodes)) + AND NOT (ps.hops = 0 AND de.to_node = ps.b) + AND ps.path_length + de.len <= ps.direct_length + {{ var('ap_topology_edge_split_tolerance_ft') }} +), + +-- A completed detour (back at B, at least 2 hops so it's genuinely an alternate route, +-- not the direct edge itself) whose total length matches the direct distance - by the +-- triangle inequality, only possible if every intermediate node lies (essentially) +-- exactly on the straight line between A and B. +split_matches AS ( + SELECT DISTINCT ON (a, b) + a, + b, + atomicid, + water_flag, + path_nodes, + path_geoms, + abs(path_length - direct_length) AS length_diff + FROM path_search + WHERE + cur_node = b + AND hops >= 2 + AND abs(path_length - direct_length) <= {{ var('ap_topology_edge_split_tolerance_ft') }} + ORDER BY a, b, abs(path_length - direct_length) +), + +split_sub_edges AS ( + SELECT + sm.atomicid, + sm.water_flag, + least(sm.path_nodes[i], sm.path_nodes[i + 1]) AS node_lo, + greatest(sm.path_nodes[i], sm.path_nodes[i + 1]) AS node_hi, + FALSE AS is_arc, + sm.path_geoms[i] AS geom + FROM split_matches AS sm + CROSS JOIN LATERAL generate_series(1, array_length(sm.path_nodes, 1) - 1) AS i +), + +-- The matched orphan's own direct edge is replaced by its split sub-edges (same +-- atomicid) - everything else passes through unchanged. +edges_densified AS ( + SELECT + e.atomicid, + e.water_flag, + e.node_lo, + e.node_hi, + e.is_arc, + e.geom + FROM edges AS e + LEFT JOIN split_matches AS sm + ON + e.atomicid = sm.atomicid AND NOT e.is_arc + AND e.node_lo = least(sm.a, sm.b) AND e.node_hi = greatest(sm.a, sm.b) + WHERE sm.a IS NULL + UNION ALL + SELECT + atomicid, + water_flag, + node_lo, + node_hi, + is_arc, + geom + FROM split_sub_edges +) + +SELECT + node_lo, + node_hi, + is_arc, + geom, + atomicid, + water_flag +FROM edges_densified diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__exact_points.sql b/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__exact_points.sql new file mode 100644 index 0000000000..3f0adbc1ea --- /dev/null +++ b/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__exact_points.sql @@ -0,0 +1,27 @@ +{{ config( + materialized='table', + indexes=[ + {'columns': ['geom'], 'type': 'gist'}, + ] +) }} + +-- Every distinct vertex coordinate (arc mids excluded), grouped by exact equality - the +-- base point set before tolerance-based merging. Exact grouping can never split two +-- bit-identical points, unlike a bare ST_SnapToGrid, where two points closer than the +-- grid size can still straddle a cell boundary. Not partitioned by any district: a +-- vertex shared across a district border is one point. + +SELECT + -- deterministic ids, so each node's representative coordinate (int__topology__nodes) + -- is the same on every build + row_number() OVER (ORDER BY st_x(geom), st_y(geom)) AS point_id, + geom, + vertex_count +FROM ( + SELECT + geom, + count(*) AS vertex_count + FROM {{ ref('int__topology__vertices') }} + WHERE NOT is_arc_mid + GROUP BY geom +) AS distinct_coords diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__grid_cells.sql b/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__grid_cells.sql new file mode 100644 index 0000000000..0e8b8ffcb3 --- /dev/null +++ b/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__grid_cells.sql @@ -0,0 +1,26 @@ +{{ config( + materialized='table', + indexes=[ + {'columns': ['x_cell', 'y_cell']}, + {'columns': ['geom'], 'type': 'gist'}, + ] +) }} + +-- One row per occupied grid cell, keyed by integer cell coordinates, with grid size = +-- ap_topology_node_merge_tolerance_ft. Since the grid is exactly the tolerance wide, two +-- points within tolerance of each other differ by at most one cell in x and in y, so a +-- cell's 8 neighbours are guaranteed to contain every true match +-- (int__topology__point_to_node checks them). This replaces windowed clustering +-- (ST_ClusterDBSCAN), which must hold a whole partition in memory: it can't run +-- citywide, and any partition just relocates the merge problem to the partition +-- boundary. + +SELECT + round(st_x(geom) / {{ var('ap_topology_node_merge_tolerance_ft') }})::bigint AS x_cell, + round(st_y(geom) / {{ var('ap_topology_node_merge_tolerance_ft') }})::bigint AS y_cell, + (array_agg(geom ORDER BY point_id))[1] AS geom, + array_agg(point_id) AS exact_point_ids +FROM {{ ref('int__topology__exact_points') }} +GROUP BY + round(st_x(geom) / {{ var('ap_topology_node_merge_tolerance_ft') }})::bigint, + round(st_y(geom) / {{ var('ap_topology_node_merge_tolerance_ft') }})::bigint diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__nodes.sql b/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__nodes.sql new file mode 100644 index 0000000000..2d4d888470 --- /dev/null +++ b/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__nodes.sql @@ -0,0 +1,18 @@ +{{ config( + materialized='table', + indexes=[ + {'columns': ['node_id'], 'unique': True}, + {'columns': ['geom'], 'type': 'gist'}, + ] +) }} + +-- Canonical topology nodes, one per int__topology__point_to_node cluster. The +-- representative coordinate (first exact point by id) is arbitrary - all that matters +-- is that every member of a cluster resolves to the same one. + +SELECT + p.node_id, + (array_agg(ec.geom ORDER BY p.exact_point_id))[1] AS geom +FROM {{ ref('int__topology__point_to_node') }} AS p +INNER JOIN {{ ref('int__topology__exact_points') }} AS ec ON p.exact_point_id = ec.point_id +GROUP BY p.node_id diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__point_to_node.sql b/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__point_to_node.sql new file mode 100644 index 0000000000..ba780ed87c --- /dev/null +++ b/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__point_to_node.sql @@ -0,0 +1,82 @@ +{{ config( + materialized='table' +) }} + +-- Maps every exact point (int__topology__exact_points) to its canonical node id. Three +-- steps, each scaling linearly: +-- 1. int__topology__grid_cells has already grid-snapped the points (a GROUP BY). +-- 2. cell_merges: for each cell, its 8 neighbours by integer offset (an equality +-- join, so hashable), confirmed with ST_DWithin at the merge tolerance. +-- 3. reachable/components: connected components over those merges, so merging is +-- transitive (A~B and B~C merge A, B, C even if A and C aren't within tolerance). +-- Merge clusters are tiny (a few near-coincident copies of one corner), so the +-- recursion is bounded by cluster size, not dataset size. +-- node_id is global - a node is routinely shared by AtomicPolygons in different +-- districts. + +WITH RECURSIVE cell_ids AS ( + SELECT + row_number() OVER (ORDER BY x_cell, y_cell) AS cell_id, + x_cell, + y_cell, + geom, + exact_point_ids + FROM {{ ref('int__topology__grid_cells') }} +), + +-- The 3x3 neighbourhood as 9 explicit (dx, dy) offsets joined by equality: a BETWEEN +-- range join can't be hashed and plans as a nested loop over all pairs of cells. +-- MATERIALIZED so the recursive step joins against a fixed table. +cell_merges AS MATERIALIZED ( + SELECT + a.cell_id AS cell_a, + b.cell_id AS cell_b + FROM cell_ids AS a + CROSS JOIN generate_series(-1, 1) AS dx + CROSS JOIN generate_series(-1, 1) AS dy + INNER JOIN cell_ids AS b + ON + b.x_cell = a.x_cell + dx + AND b.y_cell = a.y_cell + dy + WHERE + a.cell_id != b.cell_id + AND st_dwithin(a.geom, b.geom, {{ var('ap_topology_node_merge_tolerance_ft') }}) +), + +-- Only cells that merge with something need the recursive walk; every other cell is +-- its own component (the coalesce in `components`). +reachable (start_cell, reached_cell) AS ( + SELECT DISTINCT + cell_a, + cell_a -- noqa: AL08 + FROM cell_merges + UNION + SELECT + r.start_cell, + m.cell_b + FROM reachable AS r + INNER JOIN cell_merges AS m ON r.reached_cell = m.cell_a +), + +components AS ( + SELECT + ci.cell_id, + coalesce(min(r.reached_cell), ci.cell_id) AS component_id + FROM cell_ids AS ci + LEFT JOIN reachable AS r ON ci.cell_id = r.start_cell + GROUP BY ci.cell_id +), + +node_ids AS ( + SELECT + component_id, + row_number() OVER (ORDER BY component_id) AS node_id + FROM (SELECT DISTINCT component_id FROM components) AS distinct_components +) + +SELECT + unnest(ci.exact_point_ids) AS exact_point_id, + n.node_id +FROM cell_ids AS ci +INNER JOIN components AS c ON ci.cell_id = c.cell_id +INNER JOIN node_ids AS n ON c.component_id = n.component_id diff --git a/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__vertices.sql b/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__vertices.sql new file mode 100644 index 0000000000..48a430469e --- /dev/null +++ b/products/cscl/models/intermediate/atomicpolygon_topology/int__topology__vertices.sql @@ -0,0 +1,89 @@ +{{ config( + materialized='table' +) }} + +-- Every ring vertex of every in-scope AtomicPolygon, in ring order, from the RAW curved +-- source geometry (not the linearized stg__atomicpolygons.geom) - arcs are handled +-- exactly in int__topology__edges rather than approximated here. +-- +-- ST_DumpPoints on a curved MultiSurface returns [part, ring, curve_component, +-- point_in_component]; consecutive components repeat their shared transition point, +-- which is dropped so every ring is a clean ordered sequence. (part, ring) is the ring +-- key - rings of multi-part or holed AtomicPolygons must never be interleaved. +-- +-- is_arc_mid flags the middle control point of each circular arc (even-indexed points +-- of a CIRCULARSTRING). Mid points are not topology nodes: an arc's mid can sit +-- anywhere on the arc, so two AtomicPolygons tracing the same arc often disagree on it. + +WITH dumped AS ( + SELECT + a.atomicid, + a.water_flag, + st_hasarc(a.raw_geom) AS has_arc, + a.raw_geom, + d.path, + d.geom + FROM {{ ref('int__topology__ap_entities') }} AS s + INNER JOIN {{ ref('stg__atomicpolygons') }} AS a ON s.atomicid = a.atomicid + CROSS JOIN LATERAL st_dumppoints(a.raw_geom) AS d +), + +-- The ring a point sits on - only needed (and only computed) for AtomicPolygons with arcs. +with_ring AS ( + SELECT + atomicid, + water_flag, + path, + geom, + CASE + WHEN NOT has_arc THEN NULL + WHEN path[2] = 1 THEN st_exteriorring(st_geometryn(raw_geom, path[1])) + ELSE st_interiorringn(st_geometryn(raw_geom, path[1]), path[2] - 1) + END AS ring_geom + FROM dumped +), + +flat AS ( + SELECT + atomicid, + water_flag, + path[1] AS part, + path[2] AS ring, + row_number() OVER (PARTITION BY atomicid, path[1], path[2] ORDER BY path[3], path[4]) AS seq, + coalesce( + CASE + -- COMPOUNDCURVE ring: path = [part, ring, component, point_in_component] + WHEN array_length(path, 1) = 4 + THEN st_geometrytype(st_curven(ring_geom, path[3])) = 'ST_CircularString' AND path[4] % 2 = 0 + -- bare CIRCULARSTRING ring: path = [part, ring, point] + ELSE st_geometrytype(ring_geom) = 'ST_CircularString' AND path[3] % 2 = 0 + END, + FALSE + ) AS is_arc_mid, + geom + FROM with_ring +), + +deduped AS ( + SELECT + atomicid, + water_flag, + part, + ring, + seq, + is_arc_mid, + geom, + lag(geom) OVER (PARTITION BY atomicid, part, ring ORDER BY seq) AS prev_geom + FROM flat +) + +SELECT + atomicid, + water_flag, + part, + ring, + row_number() OVER (PARTITION BY atomicid, part, ring ORDER BY seq) AS seq, + is_arc_mid, + geom +FROM deduped +WHERE prev_geom IS NULL OR NOT st_equals(geom, prev_geom) diff --git a/products/cscl/models/intermediate/district_boundaries/_district_boundaries.yml b/products/cscl/models/intermediate/district_boundaries/_district_boundaries.yml new file mode 100644 index 0000000000..fc5c8f8dcd --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/_district_boundaries.yml @@ -0,0 +1,314 @@ +version: 2 + +models: +- name: int__boundary__ct2010_edges +- name: int__boundary__ct2010_raw +- name: int__boundary__ct2010 + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__cb2010_edges +- name: int__boundary__cb2010_raw +- name: int__boundary__cb2010 + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__puma2010_edges +- name: int__boundary__puma2010_raw +- name: int__boundary__puma2010 + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__fd_edges +- name: int__boundary__fd_raw +- name: int__boundary__fd + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__ct2020_edges +- name: int__boundary__ct2020_raw +- name: int__boundary__ct2020 + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__nta2020_edges +- name: int__boundary__nta2020_raw +- name: int__boundary__nta2020 + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__cdta2020_edges +- name: int__boundary__cdta2020_raw +- name: int__boundary__cdta2020 + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__fb_edges +- name: int__boundary__fb_raw +- name: int__boundary__fb + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__fc_edges +- name: int__boundary__fc_raw +- name: int__boundary__fc + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__puma2020_edges +- name: int__boundary__puma2020_raw +- name: int__boundary__puma2020 + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__nta2010_edges +- name: int__boundary__nta2010_raw +- name: int__boundary__nta2010 + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__mcea_edges +- name: int__boundary__mcea_raw +- name: int__boundary__mcea + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__ha_edges +- name: int__boundary__ha_raw +- name: int__boundary__ha + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__hc_edges +- name: int__boundary__hc_raw +- name: int__boundary__hc + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__ad_edges +- name: int__boundary__ad_raw +- name: int__boundary__ad + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__ed_edges +- name: int__boundary__ed_raw +- name: int__boundary__ed + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__sd_edges +- name: int__boundary__sd_raw +- name: int__boundary__sd + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__cd_edges +- name: int__boundary__cd_raw +- name: int__boundary__cd + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__bb_edges +- name: int__boundary__bb_raw +- name: int__boundary__bb + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__cb2020_edges +- name: int__boundary__cb2020_raw +- name: int__boundary__cb2020 + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__hez_edges +- name: int__boundary__hez_raw +- name: int__boundary__hez + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__cc_edges +- name: int__boundary__cc_raw +- name: int__boundary__cc + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__cg_edges +- name: int__boundary__cg_raw +- name: int__boundary__cg + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__ss_edges +- name: int__boundary__ss_raw +- name: int__boundary__ss + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__mc_edges +- name: int__boundary__mc_raw +- name: int__boundary__mc + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__sd_boro_edges +- name: int__boundary__sd_boro_raw +- name: int__boundary__sd_boro + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__ct2010wi_edges +- name: int__boundary__ct2010wi_raw +- name: int__boundary__ct2010wi + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__ct2020wi_edges +- name: int__boundary__ct2020wi_raw +- name: int__boundary__ct2020wi + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__cb2010wi_edges +- name: int__boundary__cb2010wi_raw +- name: int__boundary__cb2010wi + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__cb2020wi_edges +- name: int__boundary__cb2020wi_raw +- name: int__boundary__cb2020wi + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__bbwi_edges +- name: int__boundary__bbwi_raw +- name: int__boundary__bbwi + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__cdwi_edges +- name: int__boundary__cdwi_raw +- name: int__boundary__cdwi + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__adwi_edges +- name: int__boundary__adwi_raw +- name: int__boundary__adwi + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__edwi_edges +- name: int__boundary__edwi_raw +- name: int__boundary__edwi + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__ccwi_edges +- name: int__boundary__ccwi_raw +- name: int__boundary__ccwi + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__cgwi_edges +- name: int__boundary__cgwi_raw +- name: int__boundary__cgwi + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__sswi_edges +- name: int__boundary__sswi_raw +- name: int__boundary__sswi + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__mcwi_edges +- name: int__boundary__mcwi_raw +- name: int__boundary__mcwi + columns: + - name: entity_id + tests: [ unique, not_null ] + - name: geom + tests: [ not_null ] +- name: int__boundary__pp + description: Police precincts - NYPD source polygons clipped to topology-built land (not topology-built). + columns: + - name: entity_id + tests: [ unique, not_null ] +- name: int__boundary__mcea_rows + description: MCEA boundaries grouped into the published layer's rows (connected pieces of the dissolved 2000 tracts). diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ad.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ad.sql new file mode 100644 index 0000000000..efcdd07c08 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ad.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- assembly district boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__ad_raw', 'int__boundary__ad_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ad_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ad_edges.sql new file mode 100644 index 0000000000..b97b74cd15 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ad_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each assembly district that owns it. Land only. + +{{ district_boundary_classify_edges('ad') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ad_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ad_raw.sql new file mode 100644 index 0000000000..86da20cdf9 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ad_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per assembly district, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__ad_edges', 'ad') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__adwi.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__adwi.sql new file mode 100644 index 0000000000..334ebb2830 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__adwi.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- Assembly district boundaries, water included, built from the AtomicPolygon topology. + +{{ district_boundary_strip_noise_rings('int__boundary__adwi_raw', 'int__boundary__adwi_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__adwi_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__adwi_edges.sql new file mode 100644 index 0000000000..43e4a6d9e3 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__adwi_edges.sql @@ -0,0 +1,9 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each assembly district, water included: every +-- water AP carrying the district's id is part of it (the *wi layers cover water too). + +{{ district_boundary_classify_edges('ad', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__adwi_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__adwi_raw.sql new file mode 100644 index 0000000000..0bdc893c2e --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__adwi_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per assembly district (water included), before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__adwi_edges', 'ad', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__bb.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__bb.sql new file mode 100644 index 0000000000..4c149a9ecb --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__bb.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- borough boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__bb_raw', 'int__boundary__bb_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__bb_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__bb_edges.sql new file mode 100644 index 0000000000..f72acebc89 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__bb_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each borough that owns it. Land only. + +{{ district_boundary_classify_edges('bb') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__bb_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__bb_raw.sql new file mode 100644 index 0000000000..097d1cb466 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__bb_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per borough, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__bb_edges', 'bb') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__bbwi.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__bbwi.sql new file mode 100644 index 0000000000..0c672c85af --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__bbwi.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- Borough boundaries, water included, built from the AtomicPolygon topology. + +{{ district_boundary_strip_noise_rings('int__boundary__bbwi_raw', 'int__boundary__bbwi_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__bbwi_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__bbwi_edges.sql new file mode 100644 index 0000000000..fe5ab73afe --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__bbwi_edges.sql @@ -0,0 +1,9 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each borough, water included: every +-- water AP carrying the district's id is part of it (the *wi layers cover water too). + +{{ district_boundary_classify_edges('bb', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__bbwi_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__bbwi_raw.sql new file mode 100644 index 0000000000..8877a9cc10 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__bbwi_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per borough (water included), before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__bbwi_edges', 'bb', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2010.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2010.sql new file mode 100644 index 0000000000..dac56e0b08 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2010.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- 2010 census block boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__cb2010_raw', 'int__boundary__cb2010_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2010_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2010_edges.sql new file mode 100644 index 0000000000..320b775257 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2010_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each 2010 census block that owns it. Land only. + +{{ district_boundary_classify_edges('bctcb2010') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2010_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2010_raw.sql new file mode 100644 index 0000000000..191d51ca57 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2010_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per 2010 census block, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__cb2010_edges', 'bctcb2010') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2010wi.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2010wi.sql new file mode 100644 index 0000000000..8d2615dfb0 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2010wi.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- 2010 census block boundaries, water included, built from the AtomicPolygon topology. + +{{ district_boundary_strip_noise_rings('int__boundary__cb2010wi_raw', 'int__boundary__cb2010wi_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2010wi_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2010wi_edges.sql new file mode 100644 index 0000000000..d44dfaea21 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2010wi_edges.sql @@ -0,0 +1,9 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each 2010 census block, water included: every +-- water AP carrying the district's id is part of it (the *wi layers cover water too). + +{{ district_boundary_classify_edges('bctcb2010', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2010wi_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2010wi_raw.sql new file mode 100644 index 0000000000..1e59be19f4 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2010wi_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per 2010 census block (water included), before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__cb2010wi_edges', 'bctcb2010', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2020.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2020.sql new file mode 100644 index 0000000000..e89e1ec33d --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2020.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- 2020 census block boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__cb2020_raw', 'int__boundary__cb2020_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2020_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2020_edges.sql new file mode 100644 index 0000000000..b273c3a26d --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2020_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each 2020 census block that owns it. Land only. + +{{ district_boundary_classify_edges('cb2020') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2020_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2020_raw.sql new file mode 100644 index 0000000000..5784e03484 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2020_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per 2020 census block, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__cb2020_edges', 'cb2020') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2020wi.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2020wi.sql new file mode 100644 index 0000000000..8b0d21802f --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2020wi.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- 2020 census block boundaries, water included, built from the AtomicPolygon topology. + +{{ district_boundary_strip_noise_rings('int__boundary__cb2020wi_raw', 'int__boundary__cb2020wi_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2020wi_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2020wi_edges.sql new file mode 100644 index 0000000000..6328a48f0d --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2020wi_edges.sql @@ -0,0 +1,9 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each 2020 census block, water included: every +-- water AP carrying the district's id is part of it (the *wi layers cover water too). + +{{ district_boundary_classify_edges('cb2020', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2020wi_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2020wi_raw.sql new file mode 100644 index 0000000000..a24af4c088 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cb2020wi_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per 2020 census block (water included), before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__cb2020wi_edges', 'cb2020', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cc.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cc.sql new file mode 100644 index 0000000000..ad2b77251d --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cc.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- city council district boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__cc_raw', 'int__boundary__cc_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cc_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cc_edges.sql new file mode 100644 index 0000000000..3ab000f0aa --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cc_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each city council district that owns it. Land only. + +{{ district_boundary_classify_edges('cc') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cc_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cc_raw.sql new file mode 100644 index 0000000000..8d72664a13 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cc_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per city council district, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__cc_edges', 'cc') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ccwi.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ccwi.sql new file mode 100644 index 0000000000..470eb41a5f --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ccwi.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- City council district boundaries, water included, built from the AtomicPolygon topology. + +{{ district_boundary_strip_noise_rings('int__boundary__ccwi_raw', 'int__boundary__ccwi_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ccwi_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ccwi_edges.sql new file mode 100644 index 0000000000..40b59ac53c --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ccwi_edges.sql @@ -0,0 +1,9 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each city council district, water included: every +-- water AP carrying the district's id is part of it (the *wi layers cover water too). + +{{ district_boundary_classify_edges('cc', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ccwi_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ccwi_raw.sql new file mode 100644 index 0000000000..d0f21d3ec8 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ccwi_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per city council district (water included), before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__ccwi_edges', 'cc', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cd.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cd.sql new file mode 100644 index 0000000000..81b7dc3f71 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cd.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- community district boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__cd_raw', 'int__boundary__cd_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cd_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cd_edges.sql new file mode 100644 index 0000000000..f7620a56e5 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cd_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each community district that owns it. Land only. + +{{ district_boundary_classify_edges('cd') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cd_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cd_raw.sql new file mode 100644 index 0000000000..31f18a420d --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cd_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per community district, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__cd_edges', 'cd') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cdta2020.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cdta2020.sql new file mode 100644 index 0000000000..68d3898e6a --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cdta2020.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- 2020 CDTA boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__cdta2020_raw', 'int__boundary__cdta2020_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cdta2020_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cdta2020_edges.sql new file mode 100644 index 0000000000..e055b53b0a --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cdta2020_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each 2020 CDTA that owns it. Land only. + +{{ district_boundary_classify_edges('cdta2020') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cdta2020_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cdta2020_raw.sql new file mode 100644 index 0000000000..ec62d086a8 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cdta2020_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per 2020 CDTA, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__cdta2020_edges', 'cdta2020') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cdwi.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cdwi.sql new file mode 100644 index 0000000000..8d012dd438 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cdwi.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- Community district boundaries, water included, built from the AtomicPolygon topology. + +{{ district_boundary_strip_noise_rings('int__boundary__cdwi_raw', 'int__boundary__cdwi_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cdwi_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cdwi_edges.sql new file mode 100644 index 0000000000..a4b7794456 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cdwi_edges.sql @@ -0,0 +1,9 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each community district, water included: every +-- water AP carrying the district's id is part of it (the *wi layers cover water too). + +{{ district_boundary_classify_edges('cd', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cdwi_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cdwi_raw.sql new file mode 100644 index 0000000000..c1db99ae82 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cdwi_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per community district (water included), before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__cdwi_edges', 'cd', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cg.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cg.sql new file mode 100644 index 0000000000..200c6829e9 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cg.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- congressional district boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__cg_raw', 'int__boundary__cg_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cg_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cg_edges.sql new file mode 100644 index 0000000000..57ff628674 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cg_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each congressional district that owns it. Land only. + +{{ district_boundary_classify_edges('cg') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cg_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cg_raw.sql new file mode 100644 index 0000000000..5ee968d1f8 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cg_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per congressional district, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__cg_edges', 'cg') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cgwi.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cgwi.sql new file mode 100644 index 0000000000..0fa28f2281 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cgwi.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- Congressional district boundaries, water included, built from the AtomicPolygon topology. + +{{ district_boundary_strip_noise_rings('int__boundary__cgwi_raw', 'int__boundary__cgwi_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cgwi_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cgwi_edges.sql new file mode 100644 index 0000000000..8cb19f4c36 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cgwi_edges.sql @@ -0,0 +1,9 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each congressional district, water included: every +-- water AP carrying the district's id is part of it (the *wi layers cover water too). + +{{ district_boundary_classify_edges('cg', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__cgwi_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__cgwi_raw.sql new file mode 100644 index 0000000000..4623a1c0de --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__cgwi_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per congressional district (water included), before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__cgwi_edges', 'cg', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2010.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2010.sql new file mode 100644 index 0000000000..460a13e302 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2010.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- 2010 census tract boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__ct2010_raw', 'int__boundary__ct2010_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2010_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2010_edges.sql new file mode 100644 index 0000000000..a41b932280 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2010_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each 2010 census tract that owns it. Land only. + +{{ district_boundary_classify_edges('boroct') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2010_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2010_raw.sql new file mode 100644 index 0000000000..0237a50d3a --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2010_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per 2010 census tract, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__ct2010_edges', 'boroct') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2010wi.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2010wi.sql new file mode 100644 index 0000000000..40cc96a41b --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2010wi.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- 2010 census tract boundaries, water included, built from the AtomicPolygon topology. + +{{ district_boundary_strip_noise_rings('int__boundary__ct2010wi_raw', 'int__boundary__ct2010wi_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2010wi_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2010wi_edges.sql new file mode 100644 index 0000000000..69d63d45a3 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2010wi_edges.sql @@ -0,0 +1,9 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each 2010 census tract, water included: every +-- water AP carrying the district's id is part of it (the *wi layers cover water too). + +{{ district_boundary_classify_edges('boroct', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2010wi_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2010wi_raw.sql new file mode 100644 index 0000000000..7539e8617d --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2010wi_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per 2010 census tract (water included), before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__ct2010wi_edges', 'boroct', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2020.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2020.sql new file mode 100644 index 0000000000..6e911c66d5 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2020.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- 2020 census tract boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__ct2020_raw', 'int__boundary__ct2020_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2020_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2020_edges.sql new file mode 100644 index 0000000000..0c7f9d625c --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2020_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each 2020 census tract that owns it. Land only. + +{{ district_boundary_classify_edges('boroct2020') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2020_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2020_raw.sql new file mode 100644 index 0000000000..6c04b314f0 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2020_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per 2020 census tract, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__ct2020_edges', 'boroct2020') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2020wi.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2020wi.sql new file mode 100644 index 0000000000..b67ef5a5b1 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2020wi.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- 2020 census tract boundaries, water included, built from the AtomicPolygon topology. + +{{ district_boundary_strip_noise_rings('int__boundary__ct2020wi_raw', 'int__boundary__ct2020wi_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2020wi_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2020wi_edges.sql new file mode 100644 index 0000000000..8a82ab94c2 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2020wi_edges.sql @@ -0,0 +1,9 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each 2020 census tract, water included: every +-- water AP carrying the district's id is part of it (the *wi layers cover water too). + +{{ district_boundary_classify_edges('boroct2020', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2020wi_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2020wi_raw.sql new file mode 100644 index 0000000000..2ea930864c --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ct2020wi_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per 2020 census tract (water included), before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__ct2020wi_edges', 'boroct2020', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ed.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ed.sql new file mode 100644 index 0000000000..944b36c641 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ed.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- election district boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__ed_raw', 'int__boundary__ed_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ed_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ed_edges.sql new file mode 100644 index 0000000000..e4df489fd0 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ed_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each election district that owns it. Land only. + +{{ district_boundary_classify_edges('ed') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ed_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ed_raw.sql new file mode 100644 index 0000000000..85834303cb --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ed_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per election district, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__ed_edges', 'ed') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__edwi.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__edwi.sql new file mode 100644 index 0000000000..6ee2caae5a --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__edwi.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- Election district boundaries, water included, built from the AtomicPolygon topology. + +{{ district_boundary_strip_noise_rings('int__boundary__edwi_raw', 'int__boundary__edwi_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__edwi_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__edwi_edges.sql new file mode 100644 index 0000000000..f6ea4d2895 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__edwi_edges.sql @@ -0,0 +1,9 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each election district, water included: every +-- water AP carrying the district's id is part of it (the *wi layers cover water too). + +{{ district_boundary_classify_edges('ed', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__edwi_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__edwi_raw.sql new file mode 100644 index 0000000000..7b4669b36f --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__edwi_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per election district (water included), before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__edwi_edges', 'ed', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__fb.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__fb.sql new file mode 100644 index 0000000000..769ed10f48 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__fb.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- Fire battalion boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__fb_raw', 'int__boundary__fb_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__fb_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__fb_edges.sql new file mode 100644 index 0000000000..54d3367a1e --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__fb_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each fire battalion that owns it. Water APs assigned to a fire battalion are part of it, as in the published layer. + +{{ district_boundary_classify_edges('fire_battalion', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__fb_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__fb_raw.sql new file mode 100644 index 0000000000..1a245c01fb --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__fb_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per fire battalion, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__fb_edges', 'fire_battalion', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__fc.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__fc.sql new file mode 100644 index 0000000000..c81df86990 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__fc.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- Fire company boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__fc_raw', 'int__boundary__fc_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__fc_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__fc_edges.sql new file mode 100644 index 0000000000..2c12eb2611 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__fc_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each fire company that owns it. Water APs assigned to a fire company are part of it, as in the published layer. + +{{ district_boundary_classify_edges('fire_company', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__fc_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__fc_raw.sql new file mode 100644 index 0000000000..84838d6f36 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__fc_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per fire company, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__fc_edges', 'fire_company', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__fd.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__fd.sql new file mode 100644 index 0000000000..2fdb99fb61 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__fd.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- Fire division boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__fd_raw', 'int__boundary__fd_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__fd_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__fd_edges.sql new file mode 100644 index 0000000000..6890c7fcf6 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__fd_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each fire division that owns it. Water APs assigned to a fire division are part of it, as in the published layer. + +{{ district_boundary_classify_edges('fire_division', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__fd_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__fd_raw.sql new file mode 100644 index 0000000000..5197ae4492 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__fd_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per fire division, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__fd_edges', 'fire_division', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ha.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ha.sql new file mode 100644 index 0000000000..2c940f8b64 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ha.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- Health area boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__ha_raw', 'int__boundary__ha_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ha_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ha_edges.sql new file mode 100644 index 0000000000..3a793cd888 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ha_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each health area that owns it. Land only. + +{{ district_boundary_classify_edges('health_area') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ha_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ha_raw.sql new file mode 100644 index 0000000000..68ffcf598b --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ha_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per health area, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__ha_edges', 'health_area') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__hc.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__hc.sql new file mode 100644 index 0000000000..ec944ea671 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__hc.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- Health center district boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__hc_raw', 'int__boundary__hc_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__hc_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__hc_edges.sql new file mode 100644 index 0000000000..5635fefa93 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__hc_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each health center district that owns it. Land only. + +{{ district_boundary_classify_edges('health_center') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__hc_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__hc_raw.sql new file mode 100644 index 0000000000..2d05fcc41b --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__hc_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per health center district, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__hc_edges', 'health_center') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__hez.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__hez.sql new file mode 100644 index 0000000000..1d80ae5418 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__hez.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- hurricane evacuation zone boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__hez_raw', 'int__boundary__hez_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__hez_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__hez_edges.sql new file mode 100644 index 0000000000..8b8629970a --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__hez_edges.sql @@ -0,0 +1,9 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each hurricane evacuation zone that owns it. Water APs +-- carrying this district's id are part of it, as in the published layer. + +{{ district_boundary_classify_edges('hez', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__hez_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__hez_raw.sql new file mode 100644 index 0000000000..1593fa2e99 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__hez_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per hurricane evacuation zone, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__hez_edges', 'hez', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__mc.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__mc.sql new file mode 100644 index 0000000000..8763609488 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__mc.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- municipal court district boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__mc_raw', 'int__boundary__mc_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__mc_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__mc_edges.sql new file mode 100644 index 0000000000..a561a8ffc6 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__mc_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each municipal court district that owns it. Land only. + +{{ district_boundary_classify_edges('mc') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__mc_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__mc_raw.sql new file mode 100644 index 0000000000..37af2b3985 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__mc_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per municipal court district, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__mc_edges', 'mc') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__mcea.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__mcea.sql new file mode 100644 index 0000000000..c7c66e1880 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__mcea.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- MCEA boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__mcea_raw', 'int__boundary__mcea_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__mcea_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__mcea_edges.sql new file mode 100644 index 0000000000..c1462fe8ca --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__mcea_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each MCEA that owns it. Land only. + +{{ district_boundary_classify_edges('mcea') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__mcea_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__mcea_raw.sql new file mode 100644 index 0000000000..787f4de975 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__mcea_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per MCEA, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__mcea_edges', 'mcea') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__mcea_rows.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__mcea_rows.sql new file mode 100644 index 0000000000..850e173525 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__mcea_rows.sql @@ -0,0 +1,34 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- MCEA boundaries grouped into the published layer's rows: one row per connected piece of +-- the MCEA's dissolved 2000 census tracts. Tract polygons include their water, so land +-- parts across a waterway share a row (e.g. MCEA 0801: three rows holding 1, 1 and 16 +-- parts). The tract union only decides which parts go together; the geometry itself is +-- int__boundary__mcea's. + +WITH tract_components AS ( + SELECT + t.mcea, + (st_dump(st_union(t.geom))).geom AS geom + FROM {{ ref('stg__censustract2000') }} AS t + WHERE t.mcea IS NOT NULL + GROUP BY t.mcea +), + +parts AS ( + SELECT + b.entity_id, + (st_dump(b.geom)).geom AS geom + FROM {{ ref('int__boundary__mcea') }} AS b +) + +SELECT + c.mcea AS entity_id, + st_multi(st_collect(p.geom)) AS geom +FROM tract_components AS c +INNER JOIN parts AS p + ON c.mcea = p.entity_id AND st_intersects(c.geom, st_pointonsurface(p.geom)) +GROUP BY c.mcea, c.geom diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__mcwi.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__mcwi.sql new file mode 100644 index 0000000000..3d0dd4a3b0 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__mcwi.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- Municipal court district boundaries, water included, built from the AtomicPolygon topology. + +{{ district_boundary_strip_noise_rings('int__boundary__mcwi_raw', 'int__boundary__mcwi_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__mcwi_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__mcwi_edges.sql new file mode 100644 index 0000000000..6a8d59c1a2 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__mcwi_edges.sql @@ -0,0 +1,9 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each municipal court district, water included: every +-- water AP carrying the district's id is part of it (the *wi layers cover water too). + +{{ district_boundary_classify_edges('mc', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__mcwi_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__mcwi_raw.sql new file mode 100644 index 0000000000..939a8784ab --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__mcwi_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per municipal court district (water included), before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__mcwi_edges', 'mc', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__nta2010.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__nta2010.sql new file mode 100644 index 0000000000..f77c8667dd --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__nta2010.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- 2010 NTA boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__nta2010_raw', 'int__boundary__nta2010_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__nta2010_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__nta2010_edges.sql new file mode 100644 index 0000000000..be8b4ef617 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__nta2010_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each 2010 NTA that owns it. Land only. + +{{ district_boundary_classify_edges('nta2010') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__nta2010_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__nta2010_raw.sql new file mode 100644 index 0000000000..d1ea645fc6 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__nta2010_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per 2010 NTA, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__nta2010_edges', 'nta2010') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__nta2020.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__nta2020.sql new file mode 100644 index 0000000000..b9c4bf169f --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__nta2020.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- 2020 NTA boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__nta2020_raw', 'int__boundary__nta2020_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__nta2020_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__nta2020_edges.sql new file mode 100644 index 0000000000..43f568b8a6 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__nta2020_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each 2020 NTA that owns it. Land only. + +{{ district_boundary_classify_edges('nta2020') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__nta2020_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__nta2020_raw.sql new file mode 100644 index 0000000000..52937a274c --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__nta2020_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per 2020 NTA, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__nta2020_edges', 'nta2020') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__pp.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__pp.sql new file mode 100644 index 0000000000..f0d4b94b4a --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__pp.sql @@ -0,0 +1,49 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- Police precinct boundaries: the NYPD source precinct polygons clipped to land - not +-- built from the AtomicPolygon topology, since NYPD's lines don't follow AP edges (they +-- cut through hundreds of APs; see int__topology__ap_police). Land is the union of the +-- topology-built borough boundaries (int__boundary__bb), so the shoreline matches every +-- other layer. Clipping a line that runs almost along the shoreline leaves hairline +-- slivers; parts and holes under 100 sq ft are dropped (every sliver observed is under +-- 1 sq ft; real pieces are far larger). + +WITH land AS ( + SELECT st_union(geom) AS geom + FROM {{ ref('int__boundary__bb') }} +), + +clipped AS ( + SELECT + p.precinct::smallint AS precinct, + (st_dump(st_collectionextract(st_intersection(p.geom, land.geom), 3))).geom AS part + FROM {{ ref('stg__nypdprecinct') }} AS p + CROSS JOIN land +), + +cleaned AS ( + SELECT + c.precinct, + st_makepolygon( + st_exteriorring(c.part), + coalesce( + ( + SELECT array_agg(st_interiorringn(c.part, n)) + FROM generate_series(1, st_numinteriorrings(c.part)) AS n + WHERE st_area(st_makepolygon(st_interiorringn(c.part, n))) >= 100 + ), + ARRAY[]::geometry[] + ) + ) AS geom + FROM clipped AS c + WHERE st_area(c.part) >= 100 +) + +SELECT + precinct AS entity_id, + st_multi(st_union(geom)) AS geom +FROM cleaned +GROUP BY precinct diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__puma2010.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__puma2010.sql new file mode 100644 index 0000000000..61f7e265c4 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__puma2010.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- 2010 PUMA boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__puma2010_raw', 'int__boundary__puma2010_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__puma2010_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__puma2010_edges.sql new file mode 100644 index 0000000000..57ef0a0048 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__puma2010_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each 2010 PUMA that owns it. Land only. + +{{ district_boundary_classify_edges('puma') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__puma2010_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__puma2010_raw.sql new file mode 100644 index 0000000000..2de85c430f --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__puma2010_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per 2010 PUMA, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__puma2010_edges', 'puma') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__puma2020.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__puma2020.sql new file mode 100644 index 0000000000..9da4d9b1dc --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__puma2020.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- 2020 PUMA boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__puma2020_raw', 'int__boundary__puma2020_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__puma2020_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__puma2020_edges.sql new file mode 100644 index 0000000000..91abdcfe7d --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__puma2020_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each 2020 PUMA that owns it. Land only. + +{{ district_boundary_classify_edges('puma2020') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__puma2020_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__puma2020_raw.sql new file mode 100644 index 0000000000..dca1e7e490 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__puma2020_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per 2020 PUMA, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__puma2020_edges', 'puma2020') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__sd.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__sd.sql new file mode 100644 index 0000000000..8600c7e7bb --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__sd.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- school district boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__sd_raw', 'int__boundary__sd_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__sd_boro.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__sd_boro.sql new file mode 100644 index 0000000000..bb871d481a --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__sd_boro.sql @@ -0,0 +1,9 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- School district boundaries split by borough, built from the AtomicPolygon topology +-- (gdb_nysd's shape: District 10 is published once per borough). + +{{ district_boundary_strip_noise_rings('int__boundary__sd_boro_raw', 'int__boundary__sd_boro_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__sd_boro_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__sd_boro_edges.sql new file mode 100644 index 0000000000..86afcd11c8 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__sd_boro_edges.sql @@ -0,0 +1,9 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each school district as published +-- (District 10 split per borough as '10-', others whole). Water APs carrying the district are part of it, as for sd. + +{{ district_boundary_classify_edges('sd_boro', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__sd_boro_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__sd_boro_raw.sql new file mode 100644 index 0000000000..45d0420364 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__sd_boro_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per school district within a borough, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__sd_boro_edges', 'sd_boro', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__sd_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__sd_edges.sql new file mode 100644 index 0000000000..c163f8afc8 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__sd_edges.sql @@ -0,0 +1,9 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each school district that owns it. Water APs +-- carrying this district's id are part of it, as in the published layer. + +{{ district_boundary_classify_edges('sd', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__sd_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__sd_raw.sql new file mode 100644 index 0000000000..a429f4d66d --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__sd_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per school district, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__sd_edges', 'sd', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ss.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ss.sql new file mode 100644 index 0000000000..ed5304e6fc --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ss.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- state senate district boundaries built from the AtomicPolygon topology (macros/district_boundary.sql). + +{{ district_boundary_strip_noise_rings('int__boundary__ss_raw', 'int__boundary__ss_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ss_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ss_edges.sql new file mode 100644 index 0000000000..574412c0dc --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ss_edges.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each state senate district that owns it. Land only. + +{{ district_boundary_classify_edges('ss') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__ss_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__ss_raw.sql new file mode 100644 index 0000000000..71d481b1ce --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__ss_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per state senate district, from its exterior edges, before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__ss_edges', 'ss') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__sswi.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__sswi.sql new file mode 100644 index 0000000000..95ffdcd490 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__sswi.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id'], 'unique': True}, {'columns': ['geom'], 'type': 'gist'}] +) }} + +-- State senate district boundaries, water included, built from the AtomicPolygon topology. + +{{ district_boundary_strip_noise_rings('int__boundary__sswi_raw', 'int__boundary__sswi_edges') }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__sswi_edges.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__sswi_edges.sql new file mode 100644 index 0000000000..41256a78e5 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__sswi_edges.sql @@ -0,0 +1,9 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['entity_id']}] +) }} + +-- Every topology edge classified relative to each state senate district, water included: every +-- water AP carrying the district's id is part of it (the *wi layers cover water too). + +{{ district_boundary_classify_edges('ss', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/district_boundaries/int__boundary__sswi_raw.sql b/products/cscl/models/intermediate/district_boundaries/int__boundary__sswi_raw.sql new file mode 100644 index 0000000000..14eaa98ab4 --- /dev/null +++ b/products/cscl/models/intermediate/district_boundaries/int__boundary__sswi_raw.sql @@ -0,0 +1,8 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['geom'], 'type': 'gist'}] +) }} + +-- One row per state senate district (water included), before noise-ring removal. + +{{ district_boundary_build_raw('int__boundary__sswi_edges', 'ss', include_assigned_water=true) }} diff --git a/products/cscl/models/intermediate/districts/int__water_mask.sql b/products/cscl/models/intermediate/districts/int__water_mask.sql deleted file mode 100644 index 7ef0257224..0000000000 --- a/products/cscl/models/intermediate/districts/int__water_mask.sql +++ /dev/null @@ -1,77 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - --- Water polygons, subdivided and indexed, used to clip the district layers to the --- shoreline by subtraction. --- --- Clipping to land is done by subtracting water rather than intersecting land: the --- two are exact complements (4.632e9 + 8.423e9 = 13.055e9 sq ft) but water is 1,875 --- atomic polygons against land's 66,717, so it is far less geometry to process. --- Dissolving land into a single mask instead produced one 7.2M-vertex, 117MB --- geometry that took ~86 minutes to build and exhausted the build server's memory --- when every district feature was intersected against it. --- --- ST_Subdivide caps each row at 256 vertices so the GIST index stays selective and --- each per-feature union touches only nearby pieces. --- --- WATER_FLAG '1' is water; '2' (non-water), '3' (pier) and '4' are land. The ETL --- spec predates flag '4' — see the note in the README. --- --- Adjacent atomic polygons meant to share an edge exactly instead differ by --- ~0.0001-0.0002 ft after import (confirmed empirically - see CSCL-DISTRICTS-03), --- leaving hairline gaps between them. Left alone, wherever a district's own source --- boundary happens to run parallel to a chain of these gaps, they accumulate into --- a long, thin, un-clipped "spur" reaching into the water (up to ~4,000 ft on some --- coastline-adjacent NTAs) once differenced against this mask. A small --- buffer-union-debuffer ("morphological closing") merges water polygons across --- gaps below 0.02 ft before subdividing, closing them without perceptibly changing --- any real coastline shape - confirmed to match true ST_Snap's result on a known --- spur, at a fraction of the cost (~19s citywide vs. ~70s+). quad_segs=2 keeps the --- tiny buffer from bloating vertex count with circular-arc approximation it doesn't --- need at this scale. --- --- Buffer distance is 0.01 ft, not the larger 0.05 ft first tried: at 0.05 ft the --- closing merged some water polygons across a real (if narrow) non-water feature --- between them - confirmed on MN24, where it ate a genuine sliver of land prod --- keeps (0.1 sq ft over a 3,230 ft perimeter - a thin real feature, not noise). --- 0.01 ft is still ~50x the ~0.0001-0.0002 ft import-precision gap this is meant to --- close, with much less room to reach a real feature by mistake. --- --- Reverted off int__atomicpolygon_rebuilt's grid-snapped topology (see git history for --- that version) - on the real citywide comparison it left the worst spurs (SI12, --- QN98, QN45 - genuine coverage voids, not vertex misalignment - see --- CSCL-DISTRICTS-03) completely untouched, while introducing the same MN24-style --- regressions on other previously-exact NTAs. Back on stg__atomicpolygons directly --- pending further investigation. --- --- Separate from the hairline-seam closing above: st_union over many polygons that meet --- at exactly-coincident vertices can still produce spurious needle-shaped holes in the --- result - a GEOS overlay-robustness artifact, not a real gap (confirmed empirically on --- PUMA 4105, chat log 2026-10-03: every pairwise ST_Distance between adjacent source --- polygons there is exactly 0, yet st_union(stg__atomicpolygons...) alone, with no --- buffering/clipping involved at all, produced 8,893 interior-ring holes, one spanning --- ~1,900 ft while enclosing 0.237 sq ft - a zigzag, not a real loop). Passing a gridSize --- to st_union (PostGIS's precision-aware overlay path, which snaps during noding rather --- than only rounding inputs beforehand) cuts this dramatically: 0.001 ft took that same --- census-tract-dissolve case from 8,893 holes to 675 (92% fewer) for a 1.4 sq ft area --- change citywide - negligible, and comfortably smaller than this data's real positional --- accuracy. Coarser grids (0.01-0.05 ft) did better still (down to ~51-82 holes) but --- 0.001 ft was chosen to stay well under any tolerance a client might object to; the --- residual ~675 holes at this setting aren't yet root-caused (see follow-up). - -WITH closed AS ( - SELECT - st_buffer( - st_union(st_buffer(geom, 0.01, 'quad_segs=2'), 0.001), - -0.01, 'quad_segs=2' - ) AS geom - FROM {{ ref('stg__atomicpolygons') }} - WHERE water_flag = '1' -) - -SELECT st_subdivide(geom, 256) AS geom -FROM closed diff --git a/products/cscl/models/intermediate/fire/_fire.yml b/products/cscl/models/intermediate/fire/_fire.yml new file mode 100644 index 0000000000..b5e7576c57 --- /dev/null +++ b/products/cscl/models/intermediate/fire/_fire.yml @@ -0,0 +1,8 @@ +version: 2 + +models: +- name: int__fire_company_borough + description: Borough of each fire company (company centroid in borough polygons, with the ETL spec's exceptions). + columns: + - name: globalid + tests: [ unique, not_null ] diff --git a/products/cscl/models/intermediate/fire/int__fire_company_borough.sql b/products/cscl/models/intermediate/fire/int__fire_company_borough.sql new file mode 100644 index 0000000000..dc438a248c --- /dev/null +++ b/products/cscl/models/intermediate/fire/int__fire_company_borough.sql @@ -0,0 +1,54 @@ +{{ config( + materialized='table', + indexes=[{'columns': ['globalid'], 'unique': True}] +) }} + +-- Borough of each fire company: the company polygon's centroid (falling back to a point +-- on its surface) against the borough polygons, with the ETL spec's documented +-- exceptions. The one geometric step in THINFIRE, computed here so the product model is +-- plain joins. + +WITH fire_companies_normalized AS ( + SELECT + fc.globalid, + fc.geom, + replace(fc.unit_short, ' ', '') AS unit_short_normalized + FROM {{ ref('stg__firecompany') }} AS fc +) + +SELECT + fc.globalid, + CASE + -- Special case: E-81 belongs to Bronx even though it covers Marble Hill (Manhattan) + WHEN fc.unit_short_normalized = 'E81' THEN '2' + -- Special case: E-260 belongs to Queens even though it covers Roosevelt Island (Manhattan) + WHEN fc.unit_short_normalized = 'E260' THEN '4' + -- Special case: E-263 belongs to Queens even though it covers Rikers Island (Bronx) + WHEN fc.unit_short_normalized = 'E263' THEN '4' + -- For all other cases: the ETL spec (Table 25, Note 1) documents the "correct" + -- method as a company-centroid point-in-polygon match against boroughs, but + -- recommends an AtomicPolygon-attribute-match shortcut instead "for performance + -- efficiency" - ESRI-desktop-era guidance that doesn't apply here. That shortcut + -- (picking an arbitrary matching AtomicPolygon - spec's own words: "any of the + -- matching AP's") disagreed with prod for every company whose coverage spans a + -- borough boundary, since prod's arbitrary pick and ours have no reason to agree. + -- Verified this centroid method against all 12 known-mismatched companies plus + -- the 3 special cases above (which it resolves correctly without needing them, + -- kept anyway as an explicit backstop matching the spec's documented exceptions) + -- and the full company roster: exactly those 12 change, zero regressions + -- elsewhere. Same centroid/fallback pattern as the police precinct/sector/ + -- patrol-borough joins - see docs/prod_bugs/002-police-geo-centroid-mismatch.md. + ELSE ( + SELECT b.borocode + FROM {{ ref('stg__borough') }} AS b + WHERE st_within( + CASE + WHEN st_within(st_centroid(fc.geom), fc.geom) THEN st_centroid(fc.geom) + WHEN st_pointonsurface(fc.geom) IS NOT NULL THEN st_pointonsurface(fc.geom) + ELSE st_centroid(fc.geom) + END, + b.geom + ) + ) + END AS borough +FROM fire_companies_normalized AS fc diff --git a/products/cscl/models/product/districts/_districts.yml b/products/cscl/models/product/districts/_districts.yml index 78312557a1..55fe14a98f 100644 --- a/products/cscl/models/product/districts/_districts.yml +++ b/products/cscl/models/product/districts/_districts.yml @@ -5,7 +5,7 @@ version: 2 models: - name: gdb_nyad - description: State assembly districts clipped to the shoreline. + description: State assembly districts built from the AtomicPolygon topology. columns: - name: "'AssemDist'" data_tests: [ not_null ] @@ -14,7 +14,7 @@ models: data_tests: [ not_null ] - name: gdb_nyadwi - description: State assembly districts, water areas included (as stored in CSCL). + description: State assembly districts built from the AtomicPolygon topology, water areas included. columns: - name: "'AssemDist'" data_tests: [ not_null ] @@ -32,7 +32,7 @@ models: data_tests: [ not_null ] - name: gdb_nybb - description: Borough boundaries clipped to the shoreline. + description: Borough boundaries built from the AtomicPolygon topology. columns: - name: "'BoroCode'" data_tests: [ not_null ] @@ -41,7 +41,7 @@ models: data_tests: [ not_null ] - name: gdb_nybbwi - description: Borough boundaries, water areas included. + description: Borough boundaries built from the AtomicPolygon topology, water areas included. columns: - name: "'BoroCode'" data_tests: [ not_null ] @@ -59,7 +59,7 @@ models: data_tests: [ not_null ] - name: gdb_nycb2010 - description: 2010 census blocks clipped to the shoreline. + description: 2010 census blocks built from the AtomicPolygon topology. columns: - name: "'BCTCB2010'" data_tests: [ not_null ] @@ -68,7 +68,7 @@ models: data_tests: [ not_null ] - name: gdb_nycb2010wi - description: 2010 census blocks, water areas included. + description: 2010 census blocks built from the AtomicPolygon topology, water areas included. columns: - name: "'BCTCB2010'" data_tests: [ not_null ] @@ -77,7 +77,7 @@ models: data_tests: [ not_null ] - name: gdb_nycb2020 - description: 2020 census blocks clipped to the shoreline. + description: 2020 census blocks built from the AtomicPolygon topology. columns: - name: "'BCTCB2020'" data_tests: [ not_null ] @@ -86,7 +86,7 @@ models: data_tests: [ not_null ] - name: gdb_nycb2020wi - description: 2020 census blocks, water areas included. + description: 2020 census blocks built from the AtomicPolygon topology, water areas included. columns: - name: "'BCTCB2020'" data_tests: [ not_null ] @@ -95,7 +95,7 @@ models: data_tests: [ not_null ] - name: gdb_nycc - description: City council districts clipped to the shoreline. + description: City council districts built from the AtomicPolygon topology. columns: - name: "'CounDist'" data_tests: [ not_null ] @@ -104,7 +104,7 @@ models: data_tests: [ not_null ] - name: gdb_nyccwi - description: City council districts, water areas included. + description: City council districts built from the AtomicPolygon topology, water areas included. columns: - name: "'CounDist'" data_tests: [ not_null ] @@ -114,7 +114,7 @@ models: - name: gdb_nycd description: >- - Community districts clipped to the shoreline. One of ~43 layers bundled together + Community districts built from the AtomicPolygon topology. One of ~43 layers bundled together in a single citywide district-boundary geodatabase zip file, not its own standalone file. config: @@ -134,7 +134,7 @@ models: data_tests: [ not_null ] - name: gdb_nycdta2020 - description: 2020 community district tabulation areas clipped to the shoreline. + description: 2020 community district tabulation areas built from the AtomicPolygon topology. columns: - name: "'CDTA2020'" data_tests: [ not_null ] @@ -143,7 +143,7 @@ models: data_tests: [ not_null ] - name: gdb_nycdwi - description: Community districts, water areas included. + description: Community districts built from the AtomicPolygon topology, water areas included. columns: - name: "'BoroCD'" data_tests: [ not_null ] @@ -152,7 +152,7 @@ models: data_tests: [ not_null ] - name: gdb_nycg - description: Congressional districts clipped to the shoreline. + description: Congressional districts built from the AtomicPolygon topology. columns: - name: "'CongDist'" data_tests: [ not_null ] @@ -161,7 +161,7 @@ models: data_tests: [ not_null ] - name: gdb_nycgwi - description: Congressional districts, water areas included. + description: Congressional districts built from the AtomicPolygon topology, water areas included. columns: - name: "'CongDist'" data_tests: [ not_null ] @@ -170,7 +170,7 @@ models: data_tests: [ not_null ] - name: gdb_nyct2010 - description: 2010 census tracts clipped to the shoreline. + description: 2010 census tracts built from the AtomicPolygon topology. columns: - name: "'BoroCT2010'" data_tests: [ not_null ] @@ -179,7 +179,7 @@ models: data_tests: [ not_null ] - name: gdb_nyct2010wi - description: 2010 census tracts, water areas included. + description: 2010 census tracts built from the AtomicPolygon topology, water areas included. columns: - name: "'BoroCT2010'" data_tests: [ not_null ] @@ -188,7 +188,7 @@ models: data_tests: [ not_null ] - name: gdb_nyct2020 - description: 2020 census tracts clipped to the shoreline. + description: 2020 census tracts built from the AtomicPolygon topology. columns: - name: "'BoroCT2020'" data_tests: [ not_null ] @@ -197,7 +197,7 @@ models: data_tests: [ not_null ] - name: gdb_nyct2020wi - description: 2020 census tracts, water areas included. + description: 2020 census tracts built from the AtomicPolygon topology, water areas included. columns: - name: "'BoroCT2020'" data_tests: [ not_null ] @@ -206,7 +206,7 @@ models: data_tests: [ not_null ] - name: gdb_nyed - description: Election districts clipped to the shoreline. + description: Election districts built from the AtomicPolygon topology. columns: - name: "'ElectDist'" data_tests: [ not_null ] @@ -215,7 +215,7 @@ models: data_tests: [ not_null ] - name: gdb_nyedwi - description: Election districts, water areas included. + description: Election districts built from the AtomicPolygon topology, water areas included. columns: - name: "'ElectDist'" data_tests: [ not_null ] @@ -224,7 +224,7 @@ models: data_tests: [ not_null ] - name: gdb_nyfb - description: Fire battalions clipped to the shoreline. + description: Fire battalions built from the AtomicPolygon topology. columns: - name: "'FireBN'" data_tests: [ not_null ] @@ -233,7 +233,7 @@ models: data_tests: [ not_null ] - name: gdb_nyfc - description: Fire companies clipped to the shoreline. + description: Fire companies built from the AtomicPolygon topology. columns: - name: "'FireCoNum'" data_tests: [ not_null ] @@ -242,7 +242,7 @@ models: data_tests: [ not_null ] - name: gdb_nyfd - description: Fire divisions clipped to the shoreline. + description: Fire divisions built from the AtomicPolygon topology. columns: - name: "'FireDiv'" data_tests: [ not_null ] @@ -251,7 +251,7 @@ models: data_tests: [ not_null ] - name: gdb_nyha - description: Health areas clipped to the shoreline. + description: Health areas built from the AtomicPolygon topology. columns: - name: "'HealthArea'" data_tests: [ not_null ] @@ -260,7 +260,7 @@ models: data_tests: [ not_null ] - name: gdb_nyhc - description: Health center districts clipped to the shoreline. + description: Health center districts built from the AtomicPolygon topology. columns: - name: "'HCentDist'" data_tests: [ not_null ] @@ -278,7 +278,7 @@ models: data_tests: [ not_null ] - name: gdb_nyhez - description: Hurricane evacuation zones, water areas included. + description: Hurricane evacuation zones built from the AtomicPolygon topology, water areas assigned a zone included. columns: - name: "'HURRICANE_EVACUATION_ZONE'" data_tests: [ not_null ] @@ -287,7 +287,7 @@ models: data_tests: [ not_null ] - name: gdb_nymc - description: Municipal court districts clipped to the shoreline, excluding the '00' placeholder. + description: Municipal court districts built from the AtomicPolygon topology, excluding the '00' placeholder. columns: - name: "'MuniCourt'" data_tests: [ not_null ] @@ -296,7 +296,7 @@ models: data_tests: [ not_null ] - name: gdb_nymcea - description: Minor census economic areas, dissolved from 2000 census tracts, clipped and split to singlepart. + description: Minor census economic areas, built from the AtomicPolygon topology and split to singlepart. columns: - name: "'MCEA'" data_tests: [ not_null ] @@ -305,7 +305,7 @@ models: data_tests: [ not_null ] - name: gdb_nymcwi - description: Municipal court districts, water areas included, excluding the '00' placeholder. + description: Municipal court districts built from the AtomicPolygon topology, water areas included, excluding the '00' placeholder. columns: - name: "'MuniCourt'" data_tests: [ not_null ] @@ -314,7 +314,7 @@ models: data_tests: [ not_null ] - name: gdb_nynta2010 - description: 2010 neighborhood tabulation areas clipped to the shoreline. + description: 2010 neighborhood tabulation areas built from the AtomicPolygon topology. columns: - name: "'NTACode'" data_tests: [ not_null ] @@ -323,7 +323,7 @@ models: data_tests: [ not_null ] - name: gdb_nynta2020 - description: 2020 neighborhood tabulation areas clipped to the shoreline. + description: 2020 neighborhood tabulation areas built from the AtomicPolygon topology. columns: - name: "'NTA2020'" data_tests: [ not_null ] @@ -332,7 +332,7 @@ models: data_tests: [ not_null ] - name: gdb_nypp - description: Police precincts clipped to the shoreline. + description: Police precincts, the NYPD source precinct polygons clipped to land (NYPD lines don't follow AtomicPolygon edges). columns: - name: "'Precinct'" data_tests: [ not_null ] @@ -341,7 +341,7 @@ models: data_tests: [ not_null ] - name: gdb_nypuma2010 - description: 2010 public use microdata areas, dissolved from 2010 census tracts and clipped to the shoreline. + description: 2010 public use microdata areas, built from the AtomicPolygon topology. columns: - name: "'PUMA'" data_tests: [ not_null ] @@ -350,7 +350,7 @@ models: data_tests: [ not_null ] - name: gdb_nypuma2020 - description: 2020 public use microdata areas, dissolved from 2020 census tracts and clipped to the shoreline. + description: 2020 public use microdata areas, built from the AtomicPolygon topology. columns: - name: "'PUMA'" data_tests: [ not_null ] @@ -359,7 +359,7 @@ models: data_tests: [ not_null ] - name: gdb_nysd - description: School districts clipped to the shoreline. District 10 is split into one polygon per borough. + description: School districts built from the AtomicPolygon topology. District 10 is split into one polygon per borough. columns: - name: "'SchoolDist'" data_tests: [ not_null ] @@ -368,7 +368,7 @@ models: data_tests: [ not_null ] - name: gdb_nyss - description: State senate districts clipped to the shoreline. + description: State senate districts built from the AtomicPolygon topology. columns: - name: "'StSenDist'" data_tests: [ not_null ] @@ -377,7 +377,7 @@ models: data_tests: [ not_null ] - name: gdb_nysswi - description: State senate districts, water areas included. + description: State senate districts built from the AtomicPolygon topology, water areas included. columns: - name: "'StSenDist'" data_tests: [ not_null ] diff --git a/products/cscl/models/product/districts/gdb_nyad.sql b/products/cscl/models/product/districts/gdb_nyad.sql index 67369fe58c..963a58e729 100644 --- a/products/cscl/models/product/districts/gdb_nyad.sql +++ b/products/cscl/models/product/districts/gdb_nyad.sql @@ -3,17 +3,19 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} -WITH clipped AS ( +-- State assembly districts. Boundaries come from the AtomicPolygon topology +-- (int__boundary__ad); columns, column order and types match the published layer. + +WITH boundaries AS ( SELECT - d.assemdist::int AS "AssemDist", - {{ clipped_geom('d.geom') }} AS geom - FROM {{ ref('stg__assemblydistrict') }} AS d - {{ clip_to_shoreline('d.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__ad') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + b.entity_id::smallint AS "AssemDist", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b diff --git a/products/cscl/models/product/districts/gdb_nyadwi.sql b/products/cscl/models/product/districts/gdb_nyadwi.sql index 452ad9e9da..7b1525f03e 100644 --- a/products/cscl/models/product/districts/gdb_nyadwi.sql +++ b/products/cscl/models/product/districts/gdb_nyadwi.sql @@ -3,9 +3,20 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} +-- State assembly districts, water included (the *wi variant). Boundaries come from the +-- AtomicPolygon topology (int__boundary__adwi); columns, column order and types match +-- the published layer. + +WITH boundaries AS ( + SELECT + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__adwi') }} +) + SELECT - d.assemdist::int AS "AssemDist", - d.geom, - st_perimeter(d.geom) AS "SHAPE_Length", - st_area(d.geom) AS "SHAPE_Area" -FROM {{ ref('stg__assemblydistrict') }} AS d + b.entity_id::smallint AS "AssemDist", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b diff --git a/products/cscl/models/product/districts/gdb_nybb.sql b/products/cscl/models/product/districts/gdb_nybb.sql index 78bfaba3c6..cf54bcaa4d 100644 --- a/products/cscl/models/product/districts/gdb_nybb.sql +++ b/products/cscl/models/product/districts/gdb_nybb.sql @@ -3,18 +3,21 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} -WITH clipped AS ( +-- Borough boundaries. Boundaries come from the AtomicPolygon topology +-- (int__boundary__bb); columns, column order and types match the published layer. + +WITH boundaries AS ( SELECT - d.borocode::int AS "BoroCode", - d.boroname AS "BoroName", - {{ clipped_geom('d.geom') }} AS geom - FROM {{ ref('stg__borough') }} AS d - {{ clip_to_shoreline('d.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__bb') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + b.entity_id::smallint AS "BoroCode", + boro.boroname::varchar(32) AS "BoroName", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b +LEFT JOIN {{ ref('stg__borough') }} AS boro ON b.entity_id::text = boro.borocode diff --git a/products/cscl/models/product/districts/gdb_nybbwi.sql b/products/cscl/models/product/districts/gdb_nybbwi.sql index 9910228141..b83d5614c0 100644 --- a/products/cscl/models/product/districts/gdb_nybbwi.sql +++ b/products/cscl/models/product/districts/gdb_nybbwi.sql @@ -3,10 +3,22 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} +-- Borough boundaries, water included (the *wi variant). Boundaries come from the +-- AtomicPolygon topology (int__boundary__bbwi); columns, column order and types match +-- the published layer. + +WITH boundaries AS ( + SELECT + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__bbwi') }} +) + SELECT - d.borocode::int AS "BoroCode", - d.boroname AS "BoroName", - d.geom, - st_perimeter(d.geom) AS "SHAPE_Length", - st_area(d.geom) AS "SHAPE_Area" -FROM {{ ref('stg__borough') }} AS d + b.entity_id::smallint AS "BoroCode", + boro.boroname::varchar(32) AS "BoroName", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b +LEFT JOIN {{ ref('stg__borough') }} AS boro ON b.entity_id::text = boro.borocode diff --git a/products/cscl/models/product/districts/gdb_nycb2010.sql b/products/cscl/models/product/districts/gdb_nycb2010.sql index f7bfb55a67..ca0eff578f 100644 --- a/products/cscl/models/product/districts/gdb_nycb2010.sql +++ b/products/cscl/models/product/districts/gdb_nycb2010.sql @@ -3,22 +3,25 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} -WITH clipped AS ( +-- 2010 census blocks (entity_id is BCTCB2010: borocode(1) || tract(6) || block(4)). +-- Boundaries come from the AtomicPolygon topology (int__boundary__cb2010); columns, +-- column order and types match the published layer. + +WITH boundaries AS ( SELECT - d.cb AS "CB2010", - d.borocode AS "BoroCode", - b.boroname AS "BoroName", - d.ct AS "CT2010", - d.bctcb AS "BCTCB2010", - {{ clipped_geom('d.geom') }} AS geom - FROM {{ ref('stg__censusblock2010') }} AS d - INNER JOIN {{ ref('stg__borough') }} AS b ON d.borocode = b.borocode - {{ clip_to_shoreline('d.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__cb2010') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + right(b.entity_id, 4)::varchar(4) AS "CB2010", + left(b.entity_id, 1)::varchar(1) AS "BoroCode", + boro.boroname::varchar(32) AS "BoroName", + substr(b.entity_id, 2, 6)::varchar(6) AS "CT2010", + b.entity_id::varchar(11) AS "BCTCB2010", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b +LEFT JOIN {{ ref('stg__borough') }} AS boro ON left(b.entity_id, 1) = boro.borocode diff --git a/products/cscl/models/product/districts/gdb_nycb2010_by_field.sql b/products/cscl/models/product/districts/gdb_nycb2010_by_field.sql new file mode 100644 index 0000000000..939c3de3df --- /dev/null +++ b/products/cscl/models/product/districts/gdb_nycb2010_by_field.sql @@ -0,0 +1,16 @@ +{{ config(materialized='table') }} + +-- Our own gdb_nycb2010, narrowed/renamed to match qa_int__prod_fgdb_nycb2010's column +-- set exactly (lowercase, same rounding) so generate_diff_summary compares like for +-- like. See that model for why geometry itself is dropped in favor of rounded +-- shape_length/shape_area, and why they're rounded at all. + +SELECT + "BCTCB2010" AS bctcb2010, + "CB2010" AS cb2010, + "BoroCode" AS borocode, + "BoroName" AS boroname, + "CT2010" AS ct2010, + round("SHAPE_Length"::numeric) AS shape_length, + round("SHAPE_Area"::numeric, -1) AS shape_area +FROM {{ ref('gdb_nycb2010') }} diff --git a/products/cscl/models/product/districts/gdb_nycb2010wi.sql b/products/cscl/models/product/districts/gdb_nycb2010wi.sql index f7a15b6415..ddb04d8401 100644 --- a/products/cscl/models/product/districts/gdb_nycb2010wi.sql +++ b/products/cscl/models/product/districts/gdb_nycb2010wi.sql @@ -3,14 +3,25 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} +-- 2010 census blocks (entity_id is BCTCB2010: borocode(1) || tract(6) || block(4)), +-- water included (the *wi variant). Boundaries come from the AtomicPolygon topology +-- (int__boundary__cb2010wi); columns, column order and types match the published layer. + +WITH boundaries AS ( + SELECT + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__cb2010wi') }} +) + SELECT - d.cb AS "CB2010", - d.borocode AS "BoroCode", - b.boroname AS "BoroName", - d.ct AS "CT2010", - d.bctcb AS "BCTCB2010", - d.geom, - st_perimeter(d.geom) AS "SHAPE_Length", - st_area(d.geom) AS "SHAPE_Area" -FROM {{ ref('stg__censusblock2010') }} AS d -INNER JOIN {{ ref('stg__borough') }} AS b ON d.borocode = b.borocode + right(b.entity_id, 4)::varchar(4) AS "CB2010", + left(b.entity_id, 1)::varchar(1) AS "BoroCode", + boro.boroname::varchar(32) AS "BoroName", + substr(b.entity_id, 2, 6)::varchar(6) AS "CT2010", + b.entity_id::varchar(11) AS "BCTCB2010", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b +LEFT JOIN {{ ref('stg__borough') }} AS boro ON left(b.entity_id, 1) = boro.borocode diff --git a/products/cscl/models/product/districts/gdb_nycb2020.sql b/products/cscl/models/product/districts/gdb_nycb2020.sql index 58a3964ac6..fd371fe21c 100644 --- a/products/cscl/models/product/districts/gdb_nycb2020.sql +++ b/products/cscl/models/product/districts/gdb_nycb2020.sql @@ -3,23 +3,26 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} -WITH clipped AS ( +-- 2020 census blocks (entity_id is BCTCB2020: borocode(1) || tract(6) || block(4)). +-- Boundaries come from the AtomicPolygon topology (int__boundary__cb2020); columns, +-- column order and types match the published layer. + +WITH boundaries AS ( SELECT - d.cb AS "CB2020", - d.borocode AS "BoroCode", - b.boroname AS "BoroName", - d.ct AS "CT2020", - d.bctcb AS "BCTCB2020", - '36' || b.fips || d.ct || d.cb AS "GEOID", - {{ clipped_geom('d.geom') }} AS geom - FROM {{ ref('stg__censusblock2020') }} AS d - INNER JOIN {{ ref('stg__borough') }} AS b ON d.borocode = b.borocode - {{ clip_to_shoreline('d.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__cb2020') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + right(b.entity_id, 4)::varchar(4) AS "CB2020", + left(b.entity_id, 1)::varchar(1) AS "BoroCode", + boro.boroname::varchar(32) AS "BoroName", + substr(b.entity_id, 2, 6)::varchar(6) AS "CT2020", + b.entity_id::varchar(11) AS "BCTCB2020", + ('36' || boro.fips || substr(b.entity_id, 2))::varchar(15) AS "GEOID", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b +LEFT JOIN {{ ref('stg__borough') }} AS boro ON left(b.entity_id, 1) = boro.borocode diff --git a/products/cscl/models/product/districts/gdb_nycb2020wi.sql b/products/cscl/models/product/districts/gdb_nycb2020wi.sql index 10b9ae71a3..940c610dc1 100644 --- a/products/cscl/models/product/districts/gdb_nycb2020wi.sql +++ b/products/cscl/models/product/districts/gdb_nycb2020wi.sql @@ -3,15 +3,26 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} +-- 2020 census blocks (entity_id is BCTCB2020: borocode(1) || tract(6) || block(4)), +-- water included (the *wi variant). Boundaries come from the AtomicPolygon topology +-- (int__boundary__cb2020wi); columns, column order and types match the published layer. + +WITH boundaries AS ( + SELECT + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__cb2020wi') }} +) + SELECT - d.cb AS "CB2020", - d.borocode AS "BoroCode", - b.boroname AS "BoroName", - d.ct AS "CT2020", - d.bctcb AS "BCTCB2020", - '36' || b.fips || d.ct || d.cb AS "GEOID", - d.geom, - st_perimeter(d.geom) AS "SHAPE_Length", - st_area(d.geom) AS "SHAPE_Area" -FROM {{ ref('stg__censusblock2020') }} AS d -INNER JOIN {{ ref('stg__borough') }} AS b ON d.borocode = b.borocode + right(b.entity_id, 4)::varchar(4) AS "CB2020", + left(b.entity_id, 1)::varchar(1) AS "BoroCode", + boro.boroname::varchar(32) AS "BoroName", + substr(b.entity_id, 2, 6)::varchar(6) AS "CT2020", + b.entity_id::varchar(11) AS "BCTCB2020", + ('36' || boro.fips || substr(b.entity_id, 2))::varchar(15) AS "GEOID", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b +LEFT JOIN {{ ref('stg__borough') }} AS boro ON left(b.entity_id, 1) = boro.borocode diff --git a/products/cscl/models/product/districts/gdb_nycc.sql b/products/cscl/models/product/districts/gdb_nycc.sql index 54128bddbd..7831e3c269 100644 --- a/products/cscl/models/product/districts/gdb_nycc.sql +++ b/products/cscl/models/product/districts/gdb_nycc.sql @@ -3,17 +3,19 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} -WITH clipped AS ( +-- City council districts. Boundaries come from the AtomicPolygon topology +-- (int__boundary__cc); columns, column order and types match the published layer. + +WITH boundaries AS ( SELECT - d.coundist::int AS "CounDist", - {{ clipped_geom('d.geom') }} AS geom - FROM {{ ref('stg__citycouncildistrict') }} AS d - {{ clip_to_shoreline('d.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__cc') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + b.entity_id::smallint AS "CounDist", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b diff --git a/products/cscl/models/product/districts/gdb_nyccwi.sql b/products/cscl/models/product/districts/gdb_nyccwi.sql index 32e4a98396..ce2338e418 100644 --- a/products/cscl/models/product/districts/gdb_nyccwi.sql +++ b/products/cscl/models/product/districts/gdb_nyccwi.sql @@ -3,9 +3,20 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} +-- City council districts, water included (the *wi variant). Boundaries come from the +-- AtomicPolygon topology (int__boundary__ccwi); columns, column order and types match +-- the published layer. + +WITH boundaries AS ( + SELECT + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__ccwi') }} +) + SELECT - d.coundist::int AS "CounDist", - d.geom, - st_perimeter(d.geom) AS "SHAPE_Length", - st_area(d.geom) AS "SHAPE_Area" -FROM {{ ref('stg__citycouncildistrict') }} AS d + b.entity_id::smallint AS "CounDist", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b diff --git a/products/cscl/models/product/districts/gdb_nycd.sql b/products/cscl/models/product/districts/gdb_nycd.sql index 4096467518..12ffeb95d0 100644 --- a/products/cscl/models/product/districts/gdb_nycd.sql +++ b/products/cscl/models/product/districts/gdb_nycd.sql @@ -3,17 +3,19 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} -WITH clipped AS ( +-- Community districts. Boundaries come from the AtomicPolygon topology +-- (int__boundary__cd); columns, column order and types match the published layer. + +WITH boundaries AS ( SELECT - d.commdist::int AS "BoroCD", - {{ clipped_geom('d.geom') }} AS geom - FROM {{ ref('stg__communitydistrict') }} AS d - {{ clip_to_shoreline('d.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__cd') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + b.entity_id::smallint AS "BoroCD", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b diff --git a/products/cscl/models/product/districts/gdb_nycdta2020.sql b/products/cscl/models/product/districts/gdb_nycdta2020.sql index d56712468f..26806b919b 100644 --- a/products/cscl/models/product/districts/gdb_nycdta2020.sql +++ b/products/cscl/models/product/districts/gdb_nycdta2020.sql @@ -3,57 +3,41 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} --- Borough is assigned by point-on-surface containment per the ETL spec's centroid --- rule; point-on-surface keeps the probe inside concave shapes. --- --- Bounded to its own assigned borough (shrunk 0.001 ft) before shoreline clipping - --- same fix as gdb_nynta2010/2020 and gdb_nypuma2010/2020 (see CSCL-DISTRICTS-03, chat --- log 2026-10-03). SI01 North Shore's raw stg__cdta2020 boundary runs out across the --- Kill Van Kull/Arthur Kill toward NJ - a genuine AtomicPolygon coverage void at the --- jurisdictional line, same mechanism as the original SI12 case, just not yet fixed --- for this layer. --- --- 0.001 ft, not nynta's larger 5 ft: measured on SI01 across a 0.0001-5 ft sweep, the --- fix is already ~99.99% complete at the smallest size tested (SHAPE_Length 153,715.6 --- vs. 153,706.7 ft at 5 ft - 0.006% apart) while losing 1,420x less real land (3.4 vs. --- 4,830.7 sq ft) and zero interior-ring holes at every size tested. Unlike the --- PUMA/census-tract dissolve case, the spurious boundary here sticks straight out into --- open water, nowhere near the true coastline, so there's no real hairline-mismatch --- risk to buy margin against with a bigger shrink - any nonzero shrink severs it --- cleanly. 0.001 ft matches the overlay-precision gridSize used elsewhere in this --- pipeline, not a value independently tuned for this layer. +-- 2020 community district tabulation areas. Boundaries come from the AtomicPolygon +-- topology (int__boundary__cdta2020); columns, column order and types match the +-- published layer. -WITH bounded AS ( - SELECT - b.borocode::int AS "BoroCode", - b.boroname AS "BoroName", - b.fips AS "CountyFIPS", - d.cdta_code AS "CDTA2020", - cdta.cdta_name AS "CDTAName", - cdta.cdta_type AS "CDTAType", - st_intersection(d.geom, st_buffer(b.geom, -0.001)) AS geom - FROM {{ ref('stg__cdta2020') }} AS d - INNER JOIN {{ ref('stg__borough') }} AS b - ON st_contains(b.geom, st_pointonsurface(d.geom)) - LEFT JOIN {{ ref('stg__cdtaequiv2020') }} AS cdta ON d.cdta_code = cdta.cdta_code +-- borough holding most of the district's AP area - the borough it physically sits in +-- (Rikers Island's NTA QN0151 belongs to a Queens CDTA but is in the Bronx), robust to a +-- few APs across a borough line (QN99, Queens parks and cemeteries, has Brooklyn APs) +WITH district_borough AS ( + SELECT DISTINCT ON (m.cdta2020) -- noqa: AM01 + m.cdta2020 AS entity_id, + m.borocode + FROM {{ ref('int__topology__ap_entities') }} AS m + WHERE m.cdta2020 IS NOT NULL + GROUP BY m.cdta2020, m.borocode + ORDER BY m.cdta2020, sum(m.area_sqft) DESC ), -clipped AS ( +boundaries AS ( SELECT - bounded."BoroCode", - bounded."BoroName", - bounded."CountyFIPS", - bounded."CDTA2020", - bounded."CDTAName", - bounded."CDTAType", - {{ clipped_geom('bounded.geom') }} AS geom - FROM bounded - {{ clip_to_shoreline('bounded.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__cdta2020') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + db.borocode::smallint AS "BoroCode", + boro.boroname::varchar(13) AS "BoroName", + boro.fips::varchar(3) AS "CountyFIPS", + b.entity_id::varchar(4) AS "CDTA2020", + cdta.cdta_name::varchar(75) AS "CDTAName", + cdta.cdta_type::varchar(3) AS "CDTAType", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b +LEFT JOIN district_borough AS db ON b.entity_id = db.entity_id +LEFT JOIN {{ ref('stg__borough') }} AS boro ON db.borocode = boro.borocode +LEFT JOIN {{ ref('stg__cdtaequiv2020') }} AS cdta ON b.entity_id = cdta.cdta_code diff --git a/products/cscl/models/product/districts/gdb_nycdwi.sql b/products/cscl/models/product/districts/gdb_nycdwi.sql index 0484e1a376..78b0eb0cda 100644 --- a/products/cscl/models/product/districts/gdb_nycdwi.sql +++ b/products/cscl/models/product/districts/gdb_nycdwi.sql @@ -3,9 +3,20 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} +-- Community districts, water included (the *wi variant). Boundaries come from the +-- AtomicPolygon topology (int__boundary__cdwi); columns, column order and types match +-- the published layer. + +WITH boundaries AS ( + SELECT + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__cdwi') }} +) + SELECT - d.commdist::int AS "BoroCD", - d.geom, - st_perimeter(d.geom) AS "SHAPE_Length", - st_area(d.geom) AS "SHAPE_Area" -FROM {{ ref('stg__communitydistrict') }} AS d + b.entity_id::smallint AS "BoroCD", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b diff --git a/products/cscl/models/product/districts/gdb_nycg.sql b/products/cscl/models/product/districts/gdb_nycg.sql index 538fbc5282..61d6f894ae 100644 --- a/products/cscl/models/product/districts/gdb_nycg.sql +++ b/products/cscl/models/product/districts/gdb_nycg.sql @@ -3,17 +3,19 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} -WITH clipped AS ( +-- Congressional districts. Boundaries come from the AtomicPolygon topology +-- (int__boundary__cg); columns, column order and types match the published layer. + +WITH boundaries AS ( SELECT - d.congdist::int AS "CongDist", - {{ clipped_geom('d.geom') }} AS geom - FROM {{ ref('stg__congressionaldistrict') }} AS d - {{ clip_to_shoreline('d.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__cg') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + b.entity_id::smallint AS "CongDist", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b diff --git a/products/cscl/models/product/districts/gdb_nycgwi.sql b/products/cscl/models/product/districts/gdb_nycgwi.sql index a9c7e2bf1e..118205c1f4 100644 --- a/products/cscl/models/product/districts/gdb_nycgwi.sql +++ b/products/cscl/models/product/districts/gdb_nycgwi.sql @@ -3,9 +3,20 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} +-- Congressional districts, water included (the *wi variant). Boundaries come from the +-- AtomicPolygon topology (int__boundary__cgwi); columns, column order and types match +-- the published layer. + +WITH boundaries AS ( + SELECT + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__cgwi') }} +) + SELECT - d.congdist::int AS "CongDist", - d.geom, - st_perimeter(d.geom) AS "SHAPE_Length", - st_area(d.geom) AS "SHAPE_Area" -FROM {{ ref('stg__congressionaldistrict') }} AS d + b.entity_id::smallint AS "CongDist", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b diff --git a/products/cscl/models/product/districts/gdb_nyct2010.sql b/products/cscl/models/product/districts/gdb_nyct2010.sql index 134e527db1..bc4fe3081b 100644 --- a/products/cscl/models/product/districts/gdb_nyct2010.sql +++ b/products/cscl/models/product/districts/gdb_nyct2010.sql @@ -3,28 +3,31 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} -WITH clipped AS ( +-- 2010 census tracts. Boundaries come from the AtomicPolygon topology +-- (int__boundary__ct2010); columns, column order and types match the published layer. + +WITH boundaries AS ( SELECT - d.ctlabel AS "CTLabel", - d.borocode AS "BoroCode", - b.boroname AS "BoroName", - d.ct AS "CT2010", - d.boroct AS "BoroCT2010", - d.cd_eligibility AS "CDEligibil", - npc.neighborhood_code AS "NTACode", - npc.neighborhood_name AS "NTAName", - d.puma AS "PUMA", - {{ clipped_geom('d.geom') }} AS geom - FROM {{ ref('stg__censustract2010') }} AS d - INNER JOIN {{ ref('stg__borough') }} AS b ON d.borocode = b.borocode - LEFT JOIN {{ ref('stg__neighborhoodpumacodes') }} AS npc - ON d.borocode = npc.borough AND d.ct::int = npc.censustract - {{ clip_to_shoreline('d.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__ct2010') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + d.ctlabel::varchar(7) AS "CTLabel", + d.borocode::varchar(1) AS "BoroCode", + boro.boroname::varchar(32) AS "BoroName", + d.ct::varchar(6) AS "CT2010", + b.entity_id::varchar(7) AS "BoroCT2010", + d.cd_eligibility::varchar(1) AS "CDEligibil", + npc.neighborhood_code::varchar(4) AS "NTACode", + npc.neighborhood_name::varchar(75) AS "NTAName", + d.puma::varchar(4) AS "PUMA", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b +LEFT JOIN {{ ref('stg__censustract2010') }} AS d ON b.entity_id = d.boroct +LEFT JOIN {{ ref('stg__borough') }} AS boro ON d.borocode = boro.borocode +LEFT JOIN {{ ref('stg__neighborhoodpumacodes') }} AS npc + ON d.borocode = npc.borough AND d.ct::int = npc.censustract diff --git a/products/cscl/models/product/districts/gdb_nyct2010wi.sql b/products/cscl/models/product/districts/gdb_nyct2010wi.sql index eaed12aac7..d6c24aacdf 100644 --- a/products/cscl/models/product/districts/gdb_nyct2010wi.sql +++ b/products/cscl/models/product/districts/gdb_nyct2010wi.sql @@ -3,20 +3,32 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} +-- 2010 census tracts, water included (the *wi variant). Boundaries come from the +-- AtomicPolygon topology (int__boundary__ct2010wi); columns, column order and types +-- match the published layer. + +WITH boundaries AS ( + SELECT + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__ct2010wi') }} +) + SELECT - d.ctlabel AS "CTLabel", - d.borocode AS "BoroCode", - b.boroname AS "BoroName", - d.ct AS "CT2010", - d.boroct AS "BoroCT2010", - d.cd_eligibility AS "CDEligibil", - npc.neighborhood_code AS "NTACode", - npc.neighborhood_name AS "NTAName", - d.puma AS "PUMA", - d.geom, - st_perimeter(d.geom) AS "SHAPE_Length", - st_area(d.geom) AS "SHAPE_Area" -FROM {{ ref('stg__censustract2010') }} AS d -INNER JOIN {{ ref('stg__borough') }} AS b ON d.borocode = b.borocode + d.ctlabel::varchar(7) AS "CTLabel", + d.borocode::varchar(1) AS "BoroCode", + boro.boroname::varchar(32) AS "BoroName", + d.ct::varchar(6) AS "CT2010", + b.entity_id::varchar(7) AS "BoroCT2010", + d.cd_eligibility::varchar(1) AS "CDEligibil", + npc.neighborhood_code::varchar(4) AS "NTACode", + npc.neighborhood_name::varchar(75) AS "NTAName", + d.puma::varchar(4) AS "PUMA", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b +LEFT JOIN {{ ref('stg__censustract2010') }} AS d ON b.entity_id = d.boroct +LEFT JOIN {{ ref('stg__borough') }} AS boro ON d.borocode = boro.borocode LEFT JOIN {{ ref('stg__neighborhoodpumacodes') }} AS npc ON d.borocode = npc.borough AND d.ct::int = npc.censustract diff --git a/products/cscl/models/product/districts/gdb_nyct2020.sql b/products/cscl/models/product/districts/gdb_nyct2020.sql index efadeb5d96..f7faf18f3b 100644 --- a/products/cscl/models/product/districts/gdb_nyct2020.sql +++ b/products/cscl/models/product/districts/gdb_nyct2020.sql @@ -3,48 +3,34 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} --- CDTANAME is upper-cased here but mixed-case in nynta2020; that inconsistency is --- present in the published product and is preserved deliberately. +-- 2020 census tracts. Boundaries come from the AtomicPolygon topology +-- (int__boundary__ct2020); columns, column order and types match the published layer. -WITH clipped AS ( +WITH boundaries AS ( SELECT - d.ctlabel AS "CTLabel", - d.borocode AS "BoroCode", - b.boroname AS "BoroName", - d.ct AS "CT2020", - d.boroct AS "BoroCT2020", - d.cd_eligibility AS "CDEligibil", - nta.nta_name AS "NTAName", - d.neighborhood_code AS "NTA2020", - d.cdta_code AS "CDTA2020", - cdta.cdta_name AS "CDTANAME", - '36' || b.fips || d.ct AS "GEOID", - d.puma, - {{ clipped_geom('d.geom') }} AS geom - FROM {{ ref('stg__censustract2020') }} AS d - INNER JOIN {{ ref('stg__borough') }} AS b ON d.borocode = b.borocode - LEFT JOIN {{ ref('stg__ntaequiv2020') }} AS nta ON d.neighborhood_code = nta.nta_code - LEFT JOIN {{ ref('stg__cdtaequiv2020') }} AS cdta ON d.cdta_code = cdta.cdta_code - {{ clip_to_shoreline('d.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__ct2020') }} ) --- columns are listed rather than SELECT * so PUMA lands after the shape fields, --- matching prod's field order SELECT - "CTLabel", - "BoroCode", - "BoroName", - "CT2020", - "BoroCT2020", - "CDEligibil", - "NTAName", - "NTA2020", - "CDTA2020", - "CDTANAME", - "GEOID", - geom, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area", - puma AS "PUMA" -FROM clipped -WHERE NOT st_isempty(geom) + d.ctlabel::varchar(7) AS "CTLabel", + d.borocode::varchar(1) AS "BoroCode", + boro.boroname::varchar(32) AS "BoroName", + d.ct::varchar(6) AS "CT2020", + b.entity_id::varchar(7) AS "BoroCT2020", + d.cd_eligibility::varchar(1) AS "CDEligibil", + nta.nta_name::varchar(75) AS "NTAName", + d.neighborhood_code::varchar(6) AS "NTA2020", + d.cdta_code::varchar(4) AS "CDTA2020", + cdta.cdta_name::varchar(75) AS "CDTANAME", + ('36' || boro.fips || d.ct)::varchar(11) AS "GEOID", + d.puma::varchar(4) AS "PUMA", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b +LEFT JOIN {{ ref('stg__censustract2020') }} AS d ON b.entity_id = d.boroct +LEFT JOIN {{ ref('stg__borough') }} AS boro ON d.borocode = boro.borocode +LEFT JOIN {{ ref('stg__ntaequiv2020') }} AS nta ON d.neighborhood_code = nta.nta_code +LEFT JOIN {{ ref('stg__cdtaequiv2020') }} AS cdta ON d.cdta_code = cdta.cdta_code diff --git a/products/cscl/models/product/districts/gdb_nyct2020wi.sql b/products/cscl/models/product/districts/gdb_nyct2020wi.sql index 489a1be83f..9b2808a6d1 100644 --- a/products/cscl/models/product/districts/gdb_nyct2020wi.sql +++ b/products/cscl/models/product/districts/gdb_nyct2020wi.sql @@ -3,26 +3,35 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} --- CDTANAME is upper-cased here but mixed-case in nynta2020; that inconsistency is --- present in the published product and is preserved deliberately. +-- 2020 census tracts, water included (the *wi variant). Boundaries come from the +-- AtomicPolygon topology (int__boundary__ct2020wi); columns, column order and types +-- match the published layer. + +WITH boundaries AS ( + SELECT + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__ct2020wi') }} +) SELECT - d.ctlabel AS "CTLabel", - d.borocode AS "BoroCode", - b.boroname AS "BoroName", - d.ct AS "CT2020", - d.boroct AS "BoroCT2020", - d.cd_eligibility AS "CDEligibil", - nta.nta_name AS "NTAName", - d.neighborhood_code AS "NTA2020", - d.cdta_code AS "CDTA2020", - cdta.cdta_name AS "CDTANAME", - '36' || b.fips || d.ct AS "GEOID", - d.geom, - st_perimeter(d.geom) AS "SHAPE_Length", - st_area(d.geom) AS "SHAPE_Area", - d.puma AS "PUMA" -FROM {{ ref('stg__censustract2020') }} AS d -INNER JOIN {{ ref('stg__borough') }} AS b ON d.borocode = b.borocode + d.ctlabel::varchar(7) AS "CTLabel", + d.borocode::varchar(1) AS "BoroCode", + boro.boroname::varchar(32) AS "BoroName", + d.ct::varchar(6) AS "CT2020", + b.entity_id::varchar(7) AS "BoroCT2020", + d.cd_eligibility::varchar(1) AS "CDEligibil", + nta.nta_name::varchar(75) AS "NTAName", + d.neighborhood_code::varchar(6) AS "NTA2020", + d.cdta_code::varchar(4) AS "CDTA2020", + cdta.cdta_name::varchar(75) AS "CDTANAME", + ('36' || boro.fips || d.ct)::varchar(11) AS "GEOID", + d.puma::varchar(4) AS "PUMA", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b +LEFT JOIN {{ ref('stg__censustract2020') }} AS d ON b.entity_id = d.boroct +LEFT JOIN {{ ref('stg__borough') }} AS boro ON d.borocode = boro.borocode LEFT JOIN {{ ref('stg__ntaequiv2020') }} AS nta ON d.neighborhood_code = nta.nta_code LEFT JOIN {{ ref('stg__cdtaequiv2020') }} AS cdta ON d.cdta_code = cdta.cdta_code diff --git a/products/cscl/models/product/districts/gdb_nyed.sql b/products/cscl/models/product/districts/gdb_nyed.sql index 8564b029ea..77d0fe9344 100644 --- a/products/cscl/models/product/districts/gdb_nyed.sql +++ b/products/cscl/models/product/districts/gdb_nyed.sql @@ -3,19 +3,20 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} --- ElectDist packs the assembly district into the high digits (AD 23, ED 003 -> 23003). +-- Election districts (entity_id is assembly district * 1000 + ED). Boundaries come from +-- the AtomicPolygon topology (int__boundary__ed); columns, column order and types match +-- the published layer. -WITH clipped AS ( +WITH boundaries AS ( SELECT - (d.assembly_district::int * 1000 + d.electdist::int) AS "ElectDist", - {{ clipped_geom('d.geom') }} AS geom - FROM {{ ref('stg__electiondistrict') }} AS d - {{ clip_to_shoreline('d.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__ed') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + b.entity_id::int AS "ElectDist", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b diff --git a/products/cscl/models/product/districts/gdb_nyedwi.sql b/products/cscl/models/product/districts/gdb_nyedwi.sql index 8ac01b7108..49172ec79b 100644 --- a/products/cscl/models/product/districts/gdb_nyedwi.sql +++ b/products/cscl/models/product/districts/gdb_nyedwi.sql @@ -3,11 +3,20 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} --- ElectDist packs the assembly district into the high digits (AD 23, ED 003 -> 23003). +-- Election districts (entity_id is assembly district * 1000 + ED), water included (the +-- *wi variant). Boundaries come from the AtomicPolygon topology (int__boundary__edwi); +-- columns, column order and types match the published layer. + +WITH boundaries AS ( + SELECT + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__edwi') }} +) SELECT - (d.assembly_district::int * 1000 + d.electdist::int) AS "ElectDist", - d.geom, - st_perimeter(d.geom) AS "SHAPE_Length", - st_area(d.geom) AS "SHAPE_Area" -FROM {{ ref('stg__electiondistrict') }} AS d + b.entity_id::int AS "ElectDist", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b diff --git a/products/cscl/models/product/districts/gdb_nyfb.sql b/products/cscl/models/product/districts/gdb_nyfb.sql index 6357af917b..9361120a1e 100644 --- a/products/cscl/models/product/districts/gdb_nyfb.sql +++ b/products/cscl/models/product/districts/gdb_nyfb.sql @@ -3,17 +3,19 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} -WITH clipped AS ( +-- Fire battalions. Boundaries come from the AtomicPolygon topology (int__boundary__fb); +-- columns, column order and types match the published layer. + +WITH boundaries AS ( SELECT - d.battalion::int AS "FireBN", - {{ clipped_geom('d.geom') }} AS geom - FROM {{ ref('stg__firebattalion') }} AS d - {{ clip_to_shoreline('d.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__fb') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + b.entity_id::smallint AS "FireBN", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b diff --git a/products/cscl/models/product/districts/gdb_nyfc.sql b/products/cscl/models/product/districts/gdb_nyfc.sql index 386ac325f4..96e6c67b2b 100644 --- a/products/cscl/models/product/districts/gdb_nyfc.sql +++ b/products/cscl/models/product/districts/gdb_nyfc.sql @@ -3,22 +3,24 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} --- UNIT_SHORT packs company type and number into one field (e.g. 'E 14'). +-- Fire companies (entity_id is the company's UNIT_SHORT, e.g. 'E 14'). Boundaries come +-- from the AtomicPolygon topology (int__boundary__fc); columns, column order and types +-- match the published layer. -WITH clipped AS ( +WITH boundaries AS ( SELECT - split_part(d.unit_short, ' ', 1) AS "FireCoType", - split_part(d.unit_short, ' ', 2)::int AS "FireCoNum", - d.fire_battalion::int AS "FireBN", - d.fire_division::int AS "FireDiv", - {{ clipped_geom('d.geom') }} AS geom - FROM {{ ref('stg__firecompany') }} AS d - {{ clip_to_shoreline('d.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__fc') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + split_part(b.entity_id, ' ', 1)::varchar(1) AS "FireCoType", + split_part(b.entity_id, ' ', 2)::smallint AS "FireCoNum", + fc.fire_battalion::smallint AS "FireBN", + fc.fire_division::smallint AS "FireDiv", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b +LEFT JOIN {{ ref('thinfire_by_field_unformatted') }} AS fc ON b.entity_id = fc.unit_short diff --git a/products/cscl/models/product/districts/gdb_nyfd.sql b/products/cscl/models/product/districts/gdb_nyfd.sql index 106a35498b..30d891e8f9 100644 --- a/products/cscl/models/product/districts/gdb_nyfd.sql +++ b/products/cscl/models/product/districts/gdb_nyfd.sql @@ -3,17 +3,19 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} -WITH clipped AS ( +-- Fire divisions. Boundaries come from the AtomicPolygon topology (int__boundary__fd); +-- columns, column order and types match the published layer. + +WITH boundaries AS ( SELECT - d.div::int AS "FireDiv", - {{ clipped_geom('d.geom') }} AS geom - FROM {{ ref('stg__firedivision') }} AS d - {{ clip_to_shoreline('d.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__fd') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + b.entity_id::smallint AS "FireDiv", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b diff --git a/products/cscl/models/product/districts/gdb_nyha.sql b/products/cscl/models/product/districts/gdb_nyha.sql index 3cdc54e02d..85f252a595 100644 --- a/products/cscl/models/product/districts/gdb_nyha.sql +++ b/products/cscl/models/product/districts/gdb_nyha.sql @@ -3,20 +3,23 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} -WITH clipped AS ( +-- Health areas (entity_id is borocode(1) || health area(4)). Boundaries come from the +-- AtomicPolygon topology (int__boundary__ha); columns, column order and types match the +-- published layer. + +WITH boundaries AS ( SELECT - b.borocode::int AS "BoroCode", - b.boroname AS "BoroName", - d.healtharea::int AS "HealthArea", - {{ clipped_geom('d.geom') }} AS geom - FROM {{ ref('stg__healtharea') }} AS d - INNER JOIN {{ ref('stg__borough') }} AS b ON d.borough = b.borocode - {{ clip_to_shoreline('d.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__ha') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + left(b.entity_id, 1)::smallint AS "BoroCode", + boro.boroname::varchar(32) AS "BoroName", + right(b.entity_id, 4)::smallint AS "HealthArea", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b +LEFT JOIN {{ ref('stg__borough') }} AS boro ON left(b.entity_id, 1) = boro.borocode diff --git a/products/cscl/models/product/districts/gdb_nyhc.sql b/products/cscl/models/product/districts/gdb_nyhc.sql index a7a41133e4..01c8e8bccb 100644 --- a/products/cscl/models/product/districts/gdb_nyhc.sql +++ b/products/cscl/models/product/districts/gdb_nyhc.sql @@ -3,23 +3,23 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} --- Health center districts are numbered by borough (e.g. 3x = Brooklyn); the source --- carries no borough column, so the leading digit is the join key. +-- Health center districts (the first digit is the borough). Boundaries come from the +-- AtomicPolygon topology (int__boundary__hc); columns, column order and types match the +-- published layer. -WITH clipped AS ( +WITH boundaries AS ( SELECT - b.borocode::int AS "BoroCode", - b.boroname AS "BoroName", - d.hcentdist::int AS "HCentDist", - {{ clipped_geom('d.geom') }} AS geom - FROM {{ ref('stg__healthcenterdistrict') }} AS d - INNER JOIN {{ ref('stg__borough') }} AS b ON left(d.hcentdist, 1) = b.borocode - {{ clip_to_shoreline('d.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__hc') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + left(b.entity_id, 1)::smallint AS "BoroCode", + boro.boroname::varchar(32) AS "BoroName", + b.entity_id::smallint AS "HCentDist", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b +LEFT JOIN {{ ref('stg__borough') }} AS boro ON left(b.entity_id, 1) = boro.borocode diff --git a/products/cscl/models/product/districts/gdb_nyhez.sql b/products/cscl/models/product/districts/gdb_nyhez.sql index 63fff0f892..5e3b6d0a21 100644 --- a/products/cscl/models/product/districts/gdb_nyhez.sql +++ b/products/cscl/models/product/districts/gdb_nyhez.sql @@ -3,10 +3,19 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} +-- Hurricane evacuation zones. Boundaries come from the AtomicPolygon topology +-- (int__boundary__hez); columns, column order and types match the published layer. + +WITH boundaries AS ( + SELECT + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__hez') }} +) + SELECT - d.hurricane_evacuation_zone AS "HURRICANE_EVACUATION_ZONE", - d.geom, - st_perimeter(d.geom) AS "SHAPE_Length", - st_area(d.geom) AS "SHAPE_Area" -FROM {{ ref('stg__hurricaneevacuationzone') }} AS d -WHERE d.hurricane_evacuation_zone IS NOT NULL + b.entity_id::varchar(50) AS "HURRICANE_EVACUATION_ZONE", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b diff --git a/products/cscl/models/product/districts/gdb_nymc.sql b/products/cscl/models/product/districts/gdb_nymc.sql index 38961f6116..eb8f287f29 100644 --- a/products/cscl/models/product/districts/gdb_nymc.sql +++ b/products/cscl/models/product/districts/gdb_nymc.sql @@ -3,24 +3,23 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} --- MUNICOURT '00' is an unassigned placeholder (6 of the 34 source rows) and is --- excluded, which is what takes prod to 28 features. +-- Municipal court districts (entity_id is borocode(1) || court(2); the '00' placeholder +-- never gets an id). Boundaries come from the AtomicPolygon topology +-- (int__boundary__mc); columns, column order and types match the published layer. -WITH clipped AS ( +WITH boundaries AS ( SELECT - b.borocode::int AS "BoroCode", - b.boroname AS "BoroName", - d.municourt AS "MuniCourt", - {{ clipped_geom('d.geom') }} AS geom - FROM {{ ref('stg__municourtdistrict') }} AS d - INNER JOIN {{ ref('stg__borough') }} AS b ON d.borough = b.borocode - {{ clip_to_shoreline('d.geom') }} - WHERE d.municourt != '00' + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__mc') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + left(b.entity_id, 1)::smallint AS "BoroCode", + boro.boroname::varchar(32) AS "BoroName", + substr(b.entity_id, 2)::varchar(2) AS "MuniCourt", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b +LEFT JOIN {{ ref('stg__borough') }} AS boro ON left(b.entity_id, 1) = boro.borocode diff --git a/products/cscl/models/product/districts/gdb_nymcea.sql b/products/cscl/models/product/districts/gdb_nymcea.sql index 39babf4acf..ddf40ed63f 100644 --- a/products/cscl/models/product/districts/gdb_nymcea.sql +++ b/products/cscl/models/product/districts/gdb_nymcea.sql @@ -3,50 +3,32 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} --- Minor Census Economic Areas are not modelled in CSCL; they are dissolved from the --- MCEA attribute on 2000 census tracts. Clipping splits some areas into disjoint --- pieces and prod writes each as its own feature, so the dissolved groups are --- exploded back to singlepart (115 groups -> 122 features). No water-included --- version is required. +-- Minor census economic areas. Boundaries come from the AtomicPolygon topology, grouped +-- into the published rows by int__boundary__mcea_rows; columns, column order, types and +-- rows match the published layer. -WITH dissolved AS ( - SELECT - borocode, - mcea, - -- gridSize=0.001: a plain st_union over many source-tract polygons meeting at - -- exactly-coincident vertices can still produce spurious needle-shaped holes - -- (a GEOS overlay-robustness artifact, not a real gap) - see - -- int__water_mask.sql's comment and chat log 2026-10-03. - st_union(geom, 0.001) AS geom - FROM {{ ref('stg__censustract2000') }} - WHERE mcea IS NOT NULL - GROUP BY borocode, mcea -), - -clipped AS ( - SELECT - d.borocode, - d.mcea, - {{ clipped_geom('d.geom') }} AS geom - FROM dissolved AS d - {{ clip_to_shoreline('d.geom') }} -), - -singlepart AS ( - SELECT - c.borocode AS "BOROCODE", - c.mcea AS "MCEA", - b.boroname AS "BoroName", - st_multi((st_dump(c.geom)).geom) AS geom - FROM clipped AS c - INNER JOIN {{ ref('stg__borough') }} AS b ON c.borocode = b.borocode - WHERE NOT st_isempty(c.geom) +-- borough holding most of the district's AP area - the borough it physically sits in +-- (Rikers Island's NTA QN0151 belongs to a Queens CDTA but is in the Bronx), robust to a +-- few APs across a borough line (QN99, Queens parks and cemeteries, has Brooklyn APs) +WITH district_borough AS ( + SELECT DISTINCT ON (m.mcea) -- noqa: AM01 + m.mcea AS entity_id, + m.borocode + FROM {{ ref('int__topology__ap_entities') }} AS m + WHERE m.mcea IS NOT NULL + GROUP BY m.mcea, m.borocode + ORDER BY m.mcea, sum(m.area_sqft) DESC ) -- prod spells these mixed-case on this layer alone; every other feature class -- uses SHAPE_Length/SHAPE_Area SELECT - *, - st_perimeter(geom) AS "Shape_Length", - st_area(geom) AS "Shape_Area" -FROM singlepart + db.borocode::varchar(1) AS "BOROCODE", + s.entity_id::varchar(20) AS "MCEA", + boro.boroname::varchar(32) AS "BoroName", + s.geom, + st_perimeter(s.geom) AS "Shape_Length", + st_area(s.geom) AS "Shape_Area" +FROM {{ ref('int__boundary__mcea_rows') }} AS s +LEFT JOIN district_borough AS db ON s.entity_id = db.entity_id +LEFT JOIN {{ ref('stg__borough') }} AS boro ON db.borocode = boro.borocode diff --git a/products/cscl/models/product/districts/gdb_nymcwi.sql b/products/cscl/models/product/districts/gdb_nymcwi.sql index 8617137e3e..ccd1de4fb0 100644 --- a/products/cscl/models/product/districts/gdb_nymcwi.sql +++ b/products/cscl/models/product/districts/gdb_nymcwi.sql @@ -3,16 +3,24 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} --- MUNICOURT '00' is an unassigned placeholder (6 of the 34 source rows) and is --- excluded, which is what takes prod to 28 features. +-- Municipal court districts (entity_id is borocode(1) || court(2); the '00' placeholder +-- never gets an id), water included (the *wi variant). Boundaries come from the +-- AtomicPolygon topology (int__boundary__mcwi); columns, column order and types match +-- the published layer. + +WITH boundaries AS ( + SELECT + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__mcwi') }} +) SELECT - b.borocode::int AS "BoroCode", - b.boroname AS "BoroName", - d.municourt AS "MuniCourt", - d.geom, - st_perimeter(d.geom) AS "SHAPE_Length", - st_area(d.geom) AS "SHAPE_Area" -FROM {{ ref('stg__municourtdistrict') }} AS d -INNER JOIN {{ ref('stg__borough') }} AS b ON d.borough = b.borocode -WHERE d.municourt != '00' + left(b.entity_id, 1)::smallint AS "BoroCode", + boro.boroname::varchar(32) AS "BoroName", + substr(b.entity_id, 2)::varchar(2) AS "MuniCourt", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b +LEFT JOIN {{ ref('stg__borough') }} AS boro ON left(b.entity_id, 1) = boro.borocode diff --git a/products/cscl/models/product/districts/gdb_nynta2010.sql b/products/cscl/models/product/districts/gdb_nynta2010.sql index 300e876e87..8aebdc1ee3 100644 --- a/products/cscl/models/product/districts/gdb_nynta2010.sql +++ b/products/cscl/models/product/districts/gdb_nynta2010.sql @@ -3,64 +3,44 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} --- Borough is assigned by point-on-surface containment per the ETL spec's centroid --- rule; point-on-surface keeps the probe inside concave shapes. --- --- d.geom is bounded to its own assigned borough (shrunk 5 ft) before shoreline --- clipping - see CSCL-DISTRICTS-03. Two distinct problems this catches: --- --- 1. stg__neighborhood's raw boundary sometimes runs out to a legal/jurisdictional --- line - e.g. the NY/NJ state line - that has no reason to coincide with any real --- AtomicPolygon edge. Subtracting water then leaves a long, thin, un-clipped --- "spur" reaching into open water, because there's no AtomicPolygon at all out --- there to subtract (a genuine coverage void, not a data gap - confirmed by --- checking whether any AtomicPolygon of any WATER_FLAG covers the spur, not just --- whether water='1' does). Confirmed on SI12: a ~14,000 ft SHAPE_Length gap --- against prod dropped to -9 ft once bounded to Staten Island's own extent. --- 2. It also catches real errors in stg__neighborhood itself: QN99's raw polygon --- extended all the way past Staten Island to Perth Amboy, NJ - a ~2.9B sq ft --- polygon standing in for what should be a ~308M sq ft scatter of Queens parks --- and cemeteries (prod's own QN99 is exactly that scatter). Bounding to Queens' --- extent brings the area back in line with prod to within 0.24%. --- --- The 5 ft shrink keeps the new intersection boundary from itself becoming a fresh --- hairline mismatch against the true coastline in the much more common case where a --- neighborhood's edge already sits right at its own borough's real edge. +-- 2010 neighborhood tabulation areas. Boundaries come from the AtomicPolygon topology +-- (int__boundary__nta2010); columns, column order and types match the published layer. -WITH bounded AS ( - SELECT - b.borocode::int AS "BoroCode", - b.boroname AS "BoroName", - b.fips AS "CountyFIPS", - d.neighborhood_code AS "NTACode", - npc.neighborhood_name AS "NTAName", - st_intersection(d.geom, st_buffer(b.geom, -5)) AS geom - FROM {{ ref('stg__neighborhood') }} AS d - INNER JOIN {{ ref('stg__borough') }} AS b - ON st_contains(b.geom, st_pointonsurface(d.geom)) - LEFT JOIN ( - SELECT DISTINCT - neighborhood_code, - neighborhood_name - FROM {{ ref('stg__neighborhoodpumacodes') }} - ) AS npc ON d.neighborhood_code = npc.neighborhood_code +-- borough holding most of the district's AP area - the borough it physically sits in +-- (Rikers Island's NTA QN0151 belongs to a Queens CDTA but is in the Bronx), robust to a +-- few APs across a borough line (QN99, Queens parks and cemeteries, has Brooklyn APs) +WITH district_borough AS ( + SELECT DISTINCT ON (m.nta2010) -- noqa: AM01 + m.nta2010 AS entity_id, + m.borocode + FROM {{ ref('int__topology__ap_entities') }} AS m + WHERE m.nta2010 IS NOT NULL + GROUP BY m.nta2010, m.borocode + ORDER BY m.nta2010, sum(m.area_sqft) DESC ), -clipped AS ( +boundaries AS ( SELECT - "BoroCode", - "BoroName", - "CountyFIPS", - "NTACode", - "NTAName", - {{ clipped_geom('bounded.geom') }} AS geom - FROM bounded - {{ clip_to_shoreline('bounded.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__nta2010') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + db.borocode::smallint AS "BoroCode", + boro.boroname::varchar(13) AS "BoroName", + boro.fips::varchar(3) AS "CountyFIPS", + b.entity_id::varchar(4) AS "NTACode", + npc.neighborhood_name::varchar(75) AS "NTAName", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b +LEFT JOIN district_borough AS db ON b.entity_id = db.entity_id +LEFT JOIN {{ ref('stg__borough') }} AS boro ON db.borocode = boro.borocode +LEFT JOIN ( + SELECT DISTINCT + neighborhood_code, + neighborhood_name + FROM {{ ref('stg__neighborhoodpumacodes') }} +) AS npc ON b.entity_id = npc.neighborhood_code diff --git a/products/cscl/models/product/districts/gdb_nynta2020.sql b/products/cscl/models/product/districts/gdb_nynta2020.sql index b8f402d3bf..d12e8c2eed 100644 --- a/products/cscl/models/product/districts/gdb_nynta2020.sql +++ b/products/cscl/models/product/districts/gdb_nynta2020.sql @@ -3,57 +3,55 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} --- Borough is assigned by point-on-surface containment per the ETL spec's centroid --- rule; point-on-surface keeps the probe inside concave shapes. --- CDTA is matched the same way. --- --- d.geom is bounded to its own assigned borough (shrunk 5 ft) before shoreline --- clipping - see gdb_nynta2010.sql and CSCL-DISTRICTS-03 for why (a genuine --- AtomicPolygon coverage void past legal/jurisdictional lines like the NY/NJ state --- line, plus at least one real stg__neighborhood/stg__nta2020-family data error --- caught the same way). +-- 2020 neighborhood tabulation areas. Boundaries come from the AtomicPolygon topology +-- (int__boundary__nta2020); columns, column order and types match the published layer. -WITH bounded AS ( +-- borough holding most of the district's AP area - the borough it physically sits in +-- (Rikers Island's NTA QN0151 belongs to a Queens CDTA but is in the Bronx), robust to a +-- few APs across a borough line (QN99, Queens parks and cemeteries, has Brooklyn APs) +WITH district_borough AS ( + SELECT DISTINCT ON (m.nta2020) -- noqa: AM01 + m.nta2020 AS entity_id, + m.borocode + FROM {{ ref('int__topology__ap_entities') }} AS m + WHERE m.nta2020 IS NOT NULL + GROUP BY m.nta2020, m.borocode + ORDER BY m.nta2020, sum(m.area_sqft) DESC +), + +-- every NTA nests in exactly one CDTA (via its 2020 tracts) +nta_cdta AS ( SELECT - b.borocode::int AS "BoroCode", - b.boroname AS "BoroName", - b.fips AS "CountyFIPS", - d.neighborhood_code AS "NTA2020", - nta.nta_name AS "NTAName", - nta.nta_abbrev AS "NTAAbbrev", - nta.nta_type AS "NTAType", - cdta.cdta_code AS "CDTA2020", - cdta_equiv.cdta_name AS "CDTAName", - st_intersection(d.geom, st_buffer(b.geom, -5)) AS geom - FROM {{ ref('stg__nta2020') }} AS d - INNER JOIN {{ ref('stg__borough') }} AS b - ON st_contains(b.geom, st_pointonsurface(d.geom)) - LEFT JOIN {{ ref('stg__ntaequiv2020') }} AS nta ON d.neighborhood_code = nta.nta_code - LEFT JOIN {{ ref('stg__cdta2020') }} AS cdta - ON st_contains(cdta.geom, st_pointonsurface(d.geom)) - LEFT JOIN {{ ref('stg__cdtaequiv2020') }} AS cdta_equiv - ON cdta.cdta_code = cdta_equiv.cdta_code + nta2020 AS entity_id, + min(cdta2020) AS cdta2020 + FROM {{ ref('int__topology__ap_entities') }} + WHERE nta2020 IS NOT NULL + GROUP BY nta2020 ), -clipped AS ( +boundaries AS ( SELECT - "BoroCode", - "BoroName", - "CountyFIPS", - "NTA2020", - "NTAName", - "NTAAbbrev", - "NTAType", - "CDTA2020", - "CDTAName", - {{ clipped_geom('bounded.geom') }} AS geom - FROM bounded - {{ clip_to_shoreline('bounded.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__nta2020') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + db.borocode::smallint AS "BoroCode", + boro.boroname::varchar(13) AS "BoroName", + boro.fips::varchar(3) AS "CountyFIPS", + b.entity_id::varchar(6) AS "NTA2020", + nta.nta_name::varchar(75) AS "NTAName", + nta.nta_abbrev::varchar(10) AS "NTAAbbrev", + nta.nta_type::varchar(1) AS "NTAType", + nc.cdta2020::varchar(4) AS "CDTA2020", + cdta.cdta_name::varchar(75) AS "CDTAName", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b +LEFT JOIN district_borough AS db ON b.entity_id = db.entity_id +LEFT JOIN {{ ref('stg__borough') }} AS boro ON db.borocode = boro.borocode +LEFT JOIN {{ ref('stg__ntaequiv2020') }} AS nta ON b.entity_id = nta.nta_code +LEFT JOIN nta_cdta AS nc ON b.entity_id = nc.entity_id +LEFT JOIN {{ ref('stg__cdtaequiv2020') }} AS cdta ON nc.cdta2020 = cdta.cdta_code diff --git a/products/cscl/models/product/districts/gdb_nypp.sql b/products/cscl/models/product/districts/gdb_nypp.sql index ace0f50a64..a67e1a61b7 100644 --- a/products/cscl/models/product/districts/gdb_nypp.sql +++ b/products/cscl/models/product/districts/gdb_nypp.sql @@ -3,17 +3,13 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} -WITH clipped AS ( - SELECT - d.precinct::int AS "Precinct", - {{ clipped_geom('d.geom') }} AS geom - FROM {{ ref('stg__nypdprecinct') }} AS d - {{ clip_to_shoreline('d.geom') }} -) +-- Police precincts: the NYPD source precinct polygons clipped to land (int__boundary__pp; +-- see that model for why this layer isn't built from the topology). Columns and types +-- match the published layer. SELECT - *, + entity_id AS "Precinct", + geom, st_perimeter(geom) AS "SHAPE_Length", st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) +FROM {{ ref('int__boundary__pp') }} diff --git a/products/cscl/models/product/districts/gdb_nypuma2010.sql b/products/cscl/models/product/districts/gdb_nypuma2010.sql index 01ee105469..8a54db24a7 100644 --- a/products/cscl/models/product/districts/gdb_nypuma2010.sql +++ b/products/cscl/models/product/districts/gdb_nypuma2010.sql @@ -3,32 +3,19 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} --- PUMAs are not modelled in CSCL; they are dissolved from 2010 census tracts. +-- 2010 public use microdata areas. Boundaries come from the AtomicPolygon topology +-- (int__boundary__puma2010); columns, column order and types match the published layer. -WITH dissolved AS ( +WITH boundaries AS ( SELECT - puma, - -- gridSize=0.001: a plain st_union over many source-tract polygons meeting at - -- exactly-coincident vertices can still produce spurious needle-shaped holes - -- (a GEOS overlay-robustness artifact, not a real gap) - see - -- int__water_mask.sql's comment and chat log 2026-10-03. - st_union(geom, 0.001) AS geom - FROM {{ ref('stg__censustract2010') }} - WHERE puma IS NOT NULL - GROUP BY puma -), - -clipped AS ( - SELECT - d.puma AS "PUMA", - {{ clipped_geom('d.geom') }} AS geom - FROM dissolved AS d - {{ clip_to_shoreline('d.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__puma2010') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + b.entity_id::varchar(4) AS "PUMA", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b diff --git a/products/cscl/models/product/districts/gdb_nypuma2020.sql b/products/cscl/models/product/districts/gdb_nypuma2020.sql index fa6a13e2aa..5158f72aec 100644 --- a/products/cscl/models/product/districts/gdb_nypuma2020.sql +++ b/products/cscl/models/product/districts/gdb_nypuma2020.sql @@ -3,33 +3,19 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} --- PUMAs are not modelled in CSCL; they are dissolved from 2020 census tracts. --- Table 39 predates this feature class but prod ships it. +-- 2020 public use microdata areas. Boundaries come from the AtomicPolygon topology +-- (int__boundary__puma2020); columns, column order and types match the published layer. -WITH dissolved AS ( +WITH boundaries AS ( SELECT - puma, - -- gridSize=0.001: a plain st_union over many source-tract polygons meeting at - -- exactly-coincident vertices can still produce spurious needle-shaped holes - -- (a GEOS overlay-robustness artifact, not a real gap) - see - -- int__water_mask.sql's comment and chat log 2026-10-03. - st_union(geom, 0.001) AS geom - FROM {{ ref('stg__censustract2020') }} - WHERE puma IS NOT NULL - GROUP BY puma -), - -clipped AS ( - SELECT - d.puma AS "PUMA", - {{ clipped_geom('d.geom') }} AS geom - FROM dissolved AS d - {{ clip_to_shoreline('d.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__puma2020') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + b.entity_id::varchar(4) AS "PUMA", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b diff --git a/products/cscl/models/product/districts/gdb_nysd.sql b/products/cscl/models/product/districts/gdb_nysd.sql index 4205a1b044..5521cd84d2 100644 --- a/products/cscl/models/product/districts/gdb_nysd.sql +++ b/products/cscl/models/product/districts/gdb_nysd.sql @@ -3,39 +3,20 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} -WITH clipped AS ( - SELECT - d.schooldist::int AS "SchoolDist", - {{ clipped_geom('d.geom') }} AS geom - FROM {{ ref('stg__schooldistrict') }} AS d - {{ clip_to_shoreline('d.geom') }} - WHERE d.schooldist::int != 10 -), +-- School districts; District 10 is published once per borough, every other district whole. Boundaries come from +-- the AtomicPolygon topology (int__boundary__sd_boro); columns, column order and types +-- match the published layer. --- District 10 spans two boroughs and is written out once per borough, both parts --- keeping the same district number (ETL spec ch. 10, table 39 note **). -district_10_split AS ( +WITH boundaries AS ( SELECT - d.schooldist::int AS "SchoolDist", - {{ clipped_geom('st_intersection(d.geom, b.geom)') }} AS geom - FROM {{ ref('stg__schooldistrict') }} AS d - INNER JOIN {{ ref('stg__borough') }} AS b ON st_intersects(d.geom, b.geom) - {{ clip_to_shoreline('st_intersection(d.geom, b.geom)') }} - WHERE d.schooldist::int = 10 + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__sd_boro') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) - -UNION ALL - -SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM district_10_split -WHERE NOT st_isempty(geom) + split_part(b.entity_id, '-', 1)::smallint AS "SchoolDist", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b diff --git a/products/cscl/models/product/districts/gdb_nyss.sql b/products/cscl/models/product/districts/gdb_nyss.sql index 1579d70641..41929501fe 100644 --- a/products/cscl/models/product/districts/gdb_nyss.sql +++ b/products/cscl/models/product/districts/gdb_nyss.sql @@ -3,17 +3,19 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} -WITH clipped AS ( +-- State senate districts. Boundaries come from the AtomicPolygon topology +-- (int__boundary__ss); columns, column order and types match the published layer. + +WITH boundaries AS ( SELECT - d.stsendist::int AS "StSenDist", - {{ clipped_geom('d.geom') }} AS geom - FROM {{ ref('stg__statesenatedistrict') }} AS d - {{ clip_to_shoreline('d.geom') }} + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__ss') }} ) SELECT - *, - st_perimeter(geom) AS "SHAPE_Length", - st_area(geom) AS "SHAPE_Area" -FROM clipped -WHERE NOT st_isempty(geom) + b.entity_id::smallint AS "StSenDist", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b diff --git a/products/cscl/models/product/districts/gdb_nysswi.sql b/products/cscl/models/product/districts/gdb_nysswi.sql index 59c5a0eec4..b4018ba647 100644 --- a/products/cscl/models/product/districts/gdb_nysswi.sql +++ b/products/cscl/models/product/districts/gdb_nysswi.sql @@ -3,9 +3,20 @@ indexes=[{'columns': ['geom'], 'type': 'gist'}] ) }} +-- State senate districts, water included (the *wi variant). Boundaries come from the +-- AtomicPolygon topology (int__boundary__sswi); columns, column order and types match +-- the published layer. + +WITH boundaries AS ( + SELECT + entity_id, + st_multi(geom) AS geom + FROM {{ ref('int__boundary__sswi') }} +) + SELECT - d.stsendist::int AS "StSenDist", - d.geom, - st_perimeter(d.geom) AS "SHAPE_Length", - st_area(d.geom) AS "SHAPE_Area" -FROM {{ ref('stg__statesenatedistrict') }} AS d + b.entity_id::smallint AS "StSenDist", + b.geom, + st_perimeter(b.geom) AS "SHAPE_Length", + st_area(b.geom) AS "SHAPE_Area" +FROM boundaries AS b diff --git a/products/cscl/models/product/thined/thined_by_field_unformatted.sql b/products/cscl/models/product/thined/thined_by_field_unformatted.sql index 8768264cd4..5bd91ce3dd 100644 --- a/products/cscl/models/product/thined/thined_by_field_unformatted.sql +++ b/products/cscl/models/product/thined/thined_by_field_unformatted.sql @@ -1,4 +1,16 @@ -WITH election_districts_with_borough AS ( +-- Borough of each ED (Election District + Assembly District is unique citywide): the +-- borough of its lowest-atomicid AtomicPolygon. Computed in one pass rather than as a +-- per-ED correlated subquery, which scanned every AP once per ED (~10+ min). +WITH ed_borough AS ( + SELECT DISTINCT ON (electdist, assemdist) + electdist, + assemdist, + borocode + FROM {{ ref('stg__atomicpolygons') }} + ORDER BY electdist, assemdist, atomicid +), + +election_districts_with_borough AS ( SELECT ed.globalid, ed.electdist AS election_district, @@ -7,18 +19,10 @@ WITH election_districts_with_borough AS ( ed.state_sen_district, ed.muni_court_district, ed.city_council_district, - -- Derive borough from atomicpolygons by finding any AP with matching ED+AD - -- The combination of Election District + Assembly District is unique citywide - ( - SELECT ap.borocode - FROM {{ ref('stg__atomicpolygons') }} AS ap - WHERE - ap.electdist = ed.electdist - AND ap.assemdist = ed.assembly_district - ORDER BY ap.atomicid - LIMIT 1 - ) AS borough + eb.borocode AS borough FROM {{ ref('stg__electiondistrict') }} AS ed + LEFT JOIN ed_borough AS eb + ON ed.electdist = eb.electdist AND ed.assembly_district = eb.assemdist ) SELECT diff --git a/products/cscl/models/product/thinfire/thinfire_by_field_unformatted.sql b/products/cscl/models/product/thinfire/thinfire_by_field_unformatted.sql index 2162de8881..2e6d8ddfd1 100644 --- a/products/cscl/models/product/thinfire/thinfire_by_field_unformatted.sql +++ b/products/cscl/models/product/thinfire/thinfire_by_field_unformatted.sql @@ -4,7 +4,6 @@ WITH fire_companies_normalized AS ( fc.unit_short, fc.fire_division, fc.fire_battalion, - fc.geom, -- Normalize unit_short by removing spaces REPLACE(fc.unit_short, ' ', '') AS unit_short_normalized, -- Extract fire company type (first character after removing spaces) @@ -21,41 +20,9 @@ fire_companies_with_borough AS ( fc.fire_battalion, fc.fire_company_type, fc.fire_company_number, - -- Handle special cases and determine borough - CASE - -- Special case: E-81 belongs to Bronx even though it covers Marble Hill (Manhattan) - WHEN fc.unit_short_normalized = 'E81' THEN '2' - -- Special case: E-260 belongs to Queens even though it covers Roosevelt Island (Manhattan) - WHEN fc.unit_short_normalized = 'E260' THEN '4' - -- Special case: E-263 belongs to Queens even though it covers Rikers Island (Bronx) - WHEN fc.unit_short_normalized = 'E263' THEN '4' - -- For all other cases: the ETL spec (Table 25, Note 1) documents the "correct" - -- method as a company-centroid point-in-polygon match against boroughs, but - -- recommends an AtomicPolygon-attribute-match shortcut instead "for performance - -- efficiency" - ESRI-desktop-era guidance that doesn't apply here. That shortcut - -- (picking an arbitrary matching AtomicPolygon - spec's own words: "any of the - -- matching AP's") disagreed with prod for every company whose coverage spans a - -- borough boundary, since prod's arbitrary pick and ours have no reason to agree. - -- Verified this centroid method against all 12 known-mismatched companies plus - -- the 3 special cases above (which it resolves correctly without needing them, - -- kept anyway as an explicit backstop matching the spec's documented exceptions) - -- and the full company roster: exactly those 12 change, zero regressions - -- elsewhere. Same centroid/fallback pattern as the police precinct/sector/ - -- patrol-borough joins - see docs/prod_bugs/002-police-geo-centroid-mismatch.md. - ELSE ( - SELECT b.borocode - FROM {{ ref('stg__borough') }} AS b - WHERE ST_WITHIN( - CASE - WHEN ST_WITHIN(ST_CENTROID(fc.geom), fc.geom) THEN ST_CENTROID(fc.geom) - WHEN ST_POINTONSURFACE(fc.geom) IS NOT NULL THEN ST_POINTONSURFACE(fc.geom) - ELSE ST_CENTROID(fc.geom) - END, - b.geom - ) - ) - END AS borough + fcb.borough FROM fire_companies_normalized AS fc + INNER JOIN {{ ref('int__fire_company_borough') }} AS fcb ON fc.globalid = fcb.globalid ) SELECT diff --git a/products/cscl/models/product/thinlion/thinlion_by_field_unformatted.sql b/products/cscl/models/product/thinlion/thinlion_by_field_unformatted.sql index a3b6cf1bfd..3c4f4006a8 100644 --- a/products/cscl/models/product/thinlion/thinlion_by_field_unformatted.sql +++ b/products/cscl/models/product/thinlion/thinlion_by_field_unformatted.sql @@ -27,7 +27,7 @@ WITH atomic_polygons_with_lookups AS ( END AS health_area, ha.health_ct_district AS health_center_district, -- NULL AS police_patrol_borough_command, -- TL12: NYPDPrecinct doesn't have this field - prec.precinct AS police_precinct, + police.police_precinct, ap.water_flag AS water_block_mapping_suppression_flag, CASE WHEN TRIM(ap.fire_company_type) IN ('', 'null', 'NULL', '0') THEN '' @@ -94,19 +94,19 @@ WITH atomic_polygons_with_lookups AS ( ELSE ap.hurricane_evacuation_zone END AS hurricane_evacuation_zone, CASE - WHEN pb.patrol_borough = 'MS' THEN '1' - WHEN pb.patrol_borough = 'MN' THEN '2' - WHEN pb.patrol_borough = 'BX' THEN '3' -- Temporary: fallback for current FGDB until XN/XS polygons added - WHEN pb.patrol_borough = 'XS' THEN '3' - WHEN pb.patrol_borough = 'BS' THEN '4' - WHEN pb.patrol_borough = 'BN' THEN '5' - WHEN pb.patrol_borough = 'QN' THEN '6' - WHEN pb.patrol_borough = 'SI' THEN '7' - WHEN pb.patrol_borough = 'QS' THEN '8' - WHEN pb.patrol_borough = 'XN' THEN '9' + WHEN police.patrol_borough = 'MS' THEN '1' + WHEN police.patrol_borough = 'MN' THEN '2' + WHEN police.patrol_borough = 'BX' THEN '3' -- Temporary: fallback for current FGDB until XN/XS polygons added + WHEN police.patrol_borough = 'XS' THEN '3' + WHEN police.patrol_borough = 'BS' THEN '4' + WHEN police.patrol_borough = 'BN' THEN '5' + WHEN police.patrol_borough = 'QN' THEN '6' + WHEN police.patrol_borough = 'SI' THEN '7' + WHEN police.patrol_borough = 'QS' THEN '8' + WHEN police.patrol_borough = 'XN' THEN '9' END AS police_patrol_borough_command, - pb.patrol_borough, - beat.sector AS police_sector, + police.patrol_borough, + police.police_sector, ct2020.neighborhood_code AS nta2020, ct2020.cdta_code AS cdta, ap.commercial_waste_zone AS cwz, @@ -115,9 +115,9 @@ WITH atomic_polygons_with_lookups AS ( ct2010.globalid AS ct2010_globalid, ct2020.globalid AS ct2020_globalid, ha.globalid AS healtharea_globalid, - prec.globalid AS nypdprecinct_globalid, - pb.globalid AS nypdpatrolborough_globalid, - beat.globalid AS nypdbeat_globalid + police.nypdprecinct_globalid, + police.nypdpatrolborough_globalid, + police.nypdbeat_globalid FROM {{ ref("stg__atomicpolygons") }} AS ap -- Join CensusTract2010 via concatenated key LEFT JOIN {{ ref("stg__censustract2010") }} AS ct2010 @@ -128,35 +128,11 @@ WITH atomic_polygons_with_lookups AS ( -- Join HealthArea via health_area from CensusTract2010 LEFT JOIN {{ ref("stg__healtharea") }} AS ha ON ct2010.health_area = ha.healtharea AND ct2010.borocode = ha.borough - -- Spatial joins using point-in-polygon with C# centroid fallback logic - -- First try centroid, if outside polygon use ST_PointOnSurface, else fallback to centroid - LEFT JOIN {{ ref("stg__nypdprecinct") }} AS prec - ON ST_WITHIN( - CASE - WHEN ST_WITHIN(ST_CENTROID(ap.geom), ap.geom) THEN ST_CENTROID(ap.geom) - WHEN ST_POINTONSURFACE(ap.geom) IS NOT NULL THEN ST_POINTONSURFACE(ap.geom) - ELSE ST_CENTROID(ap.geom) - END, - prec.geom - ) - LEFT JOIN {{ ref("stg__nypdpatrolborough") }} AS pb - ON ST_WITHIN( - CASE - WHEN ST_WITHIN(ST_CENTROID(ap.geom), ap.geom) THEN ST_CENTROID(ap.geom) - WHEN ST_POINTONSURFACE(ap.geom) IS NOT NULL THEN ST_POINTONSURFACE(ap.geom) - ELSE ST_CENTROID(ap.geom) - END, - pb.geom - ) - LEFT JOIN {{ ref("stg__nypdbeat") }} AS beat - ON ST_WITHIN( - CASE - WHEN ST_WITHIN(ST_CENTROID(ap.geom), ap.geom) THEN ST_CENTROID(ap.geom) - WHEN ST_POINTONSURFACE(ap.geom) IS NOT NULL THEN ST_POINTONSURFACE(ap.geom) - ELSE ST_CENTROID(ap.geom) - END, - beat.geom - ) + -- Police geography (precinct, patrol borough, sector) is computed once per AP in + -- int__topology__ap_police: the polygon at the AP's centroid, or majority of area + -- where the centroid falls outside the AP - the rule that reproduces prod. + LEFT JOIN {{ ref("int__topology__ap_police") }} AS police + ON ap.atomicid = police.atomicid ) SELECT * FROM atomic_polygons_with_lookups diff --git a/products/cscl/models/qa_int/qa_int__prod_fgdb_nycb2010.sql b/products/cscl/models/qa_int/qa_int__prod_fgdb_nycb2010.sql new file mode 100644 index 0000000000..34d55ceaf8 --- /dev/null +++ b/products/cscl/models/qa_int/qa_int__prod_fgdb_nycb2010.sql @@ -0,0 +1,45 @@ +{{ + config( + materialized='table', + tags=['qa', 'qa_prod'], + indexes=[ + {'columns': ['bctcb2010']} + ] + ) +}} + +-- Production nycb2010 (2010 Census Blocks clipped to shoreline), narrowed to the +-- columns worth comparing - keyed on BCTCB2010 (lion_outputs.csv's declared key for +-- this layer). objectid (an ESRI load-time sequence, not stable source data) and the +-- geometry itself are dropped: raw geometry essentially never matches byte-for-byte +-- between our build and prod (different overlay engines, see CSCL-DISTRICTS-03), so a +-- direct geometry diff would just report ~100% of rows "modified" with no signal in +-- it. shape_length/shape_area stand in for geometry instead, same as +-- poc_validation/compare_gdb.py's own approach for file-based comparison. +-- +-- shape_length is rounded to the nearest whole ft, shape_area to the nearest 10 sq ft, +-- before comparing - GDAL recomputes both from scratch on every read, so small +-- disagreement is expected read-to-read noise, not a real difference (see +-- dcpy.geospatial.gdb.compare.DEFAULT_GEOMETRY_DERIVED_FLOAT_COLUMNS, which uses a +-- relative 5e-4 tolerance for the same reason). These are fixed, not relative, so +-- confirmed empirically rather than derived: on a random sample, every nycb2010 +-- shape_area "diff" at nearest-whole-sqft rounding was exactly +-1 sqft (a rounding- +-- boundary coincidence, areas here run tens of thousands to low hundreds of +-- thousands sqft) - nearest-10 clears that noise. Not a general-purpose relative +-- tolerance; a layer with much larger or smaller areas may need a different value. + +{% set prod_relation = adapter.get_relation( + database = "db-cscl", + schema = "production_outputs", + identifier = "fgdb_nycb2010" +) -%} + +SELECT + bctcb2010, + cb2010, + borocode, + boroname, + ct2010, + round(shape_length::numeric) AS shape_length, + round(shape_area::numeric, -1) AS shape_area +FROM {{ prod_relation }} diff --git a/products/cscl/models/staging/stg__assemblydistrict.sql b/products/cscl/models/staging/stg__assemblydistrict.sql deleted file mode 100644 index 86c59e9f8f..0000000000 --- a/products/cscl/models/staging/stg__assemblydistrict.sql +++ /dev/null @@ -1,13 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - -SELECT - globalid, - assemdist, - st_makevalid(linearize(geom)) AS geom, - geom AS raw_geom -FROM {{ source("recipe_sources", "dcp_cscl_assemblydistrict") }} diff --git a/products/cscl/models/staging/stg__cdta2020.sql b/products/cscl/models/staging/stg__cdta2020.sql deleted file mode 100644 index 9c4d13189e..0000000000 --- a/products/cscl/models/staging/stg__cdta2020.sql +++ /dev/null @@ -1,14 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - -SELECT - globalid, - cdta_code, - - st_makevalid(linearize(geom)) AS geom, - geom AS raw_geom -FROM {{ source("recipe_sources", "dcp_cscl_cdta2020") }} diff --git a/products/cscl/models/staging/stg__censusblock2010.sql b/products/cscl/models/staging/stg__censusblock2010.sql deleted file mode 100644 index ededfd6c5b..0000000000 --- a/products/cscl/models/staging/stg__censusblock2010.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - -SELECT - globalid, - borocode, - ct, - cb, - cb_suffix, - bctcb, - st_makevalid(linearize(geom)) AS geom, - geom AS raw_geom -FROM {{ source("recipe_sources", "dcp_cscl_censusblock2010") }} diff --git a/products/cscl/models/staging/stg__censusblock2020.sql b/products/cscl/models/staging/stg__censusblock2020.sql deleted file mode 100644 index 1074f761a3..0000000000 --- a/products/cscl/models/staging/stg__censusblock2020.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - -SELECT - globalid, - borocode, - ct, - cb, - cb_suffix, - bctcb, - st_makevalid(linearize(geom)) AS geom, - geom AS raw_geom -FROM {{ source("recipe_sources", "dcp_cscl_censusblock2020") }} diff --git a/products/cscl/models/staging/stg__citycouncildistrict.sql b/products/cscl/models/staging/stg__citycouncildistrict.sql deleted file mode 100644 index 55b938c70f..0000000000 --- a/products/cscl/models/staging/stg__citycouncildistrict.sql +++ /dev/null @@ -1,13 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - -SELECT - globalid, - coundist, - st_makevalid(linearize(geom)) AS geom, - geom AS raw_geom -FROM {{ source("recipe_sources", "dcp_cscl_citycouncildistrict") }} diff --git a/products/cscl/models/staging/stg__communitydistrict.sql b/products/cscl/models/staging/stg__communitydistrict.sql deleted file mode 100644 index 63cbaad706..0000000000 --- a/products/cscl/models/staging/stg__communitydistrict.sql +++ /dev/null @@ -1,13 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - -SELECT - globalid, - commdist, - st_makevalid(linearize(geom)) AS geom, - geom AS raw_geom -FROM {{ source("recipe_sources", "dcp_cscl_communitydistrict") }} diff --git a/products/cscl/models/staging/stg__congressionaldistrict.sql b/products/cscl/models/staging/stg__congressionaldistrict.sql deleted file mode 100644 index 6203bfb70e..0000000000 --- a/products/cscl/models/staging/stg__congressionaldistrict.sql +++ /dev/null @@ -1,13 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - -SELECT - globalid, - congdist, - st_makevalid(linearize(geom)) AS geom, - geom AS raw_geom -FROM {{ source("recipe_sources", "dcp_cscl_congressionaldistrict") }} diff --git a/products/cscl/models/staging/stg__firebattalion.sql b/products/cscl/models/staging/stg__firebattalion.sql deleted file mode 100644 index 68ebb4e871..0000000000 --- a/products/cscl/models/staging/stg__firebattalion.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - -SELECT - globalid, - battalion, - division, - boroughcommand, - st_makevalid(linearize(geom)) AS geom, - geom AS raw_geom -FROM {{ source("recipe_sources", "dcp_cscl_firebattalion") }} diff --git a/products/cscl/models/staging/stg__firedivision.sql b/products/cscl/models/staging/stg__firedivision.sql deleted file mode 100644 index bc2ded1663..0000000000 --- a/products/cscl/models/staging/stg__firedivision.sql +++ /dev/null @@ -1,14 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - -SELECT - globalid, - div, - boroughcommand, - st_makevalid(linearize(geom)) AS geom, - geom AS raw_geom -FROM {{ source("recipe_sources", "dcp_cscl_firedivision") }} diff --git a/products/cscl/models/staging/stg__healthcenterdistrict.sql b/products/cscl/models/staging/stg__healthcenterdistrict.sql deleted file mode 100644 index fbb014323c..0000000000 --- a/products/cscl/models/staging/stg__healthcenterdistrict.sql +++ /dev/null @@ -1,13 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - -SELECT - globalid, - hcentdist, - st_makevalid(linearize(geom)) AS geom, - geom AS raw_geom -FROM {{ source("recipe_sources", "dcp_cscl_healthcenterdistrict") }} diff --git a/products/cscl/models/staging/stg__hurricaneevacuationzone.sql b/products/cscl/models/staging/stg__hurricaneevacuationzone.sql deleted file mode 100644 index 457fc025c6..0000000000 --- a/products/cscl/models/staging/stg__hurricaneevacuationzone.sql +++ /dev/null @@ -1,13 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - -SELECT - globalid, - hurricane_evacuation_zone, - st_makevalid(linearize(geom)) AS geom, - geom AS raw_geom -FROM {{ source("recipe_sources", "dcp_cscl_hurricaneevacuationzone") }} diff --git a/products/cscl/models/staging/stg__municourtdistrict.sql b/products/cscl/models/staging/stg__municourtdistrict.sql deleted file mode 100644 index 9efc53a4a3..0000000000 --- a/products/cscl/models/staging/stg__municourtdistrict.sql +++ /dev/null @@ -1,14 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - -SELECT - globalid, - municourt, - borough, - st_makevalid(linearize(geom)) AS geom, - geom AS raw_geom -FROM {{ source("recipe_sources", "dcp_cscl_municourtdistrict") }} diff --git a/products/cscl/models/staging/stg__neighborhood.sql b/products/cscl/models/staging/stg__neighborhood.sql deleted file mode 100644 index f573ef5a8b..0000000000 --- a/products/cscl/models/staging/stg__neighborhood.sql +++ /dev/null @@ -1,13 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - -SELECT - globalid, - neighborhood_code, - st_makevalid(linearize(geom)) AS geom, - geom AS raw_geom -FROM {{ source("recipe_sources", "dcp_cscl_neighborhood") }} diff --git a/products/cscl/models/staging/stg__nta2020.sql b/products/cscl/models/staging/stg__nta2020.sql deleted file mode 100644 index 8e81183e24..0000000000 --- a/products/cscl/models/staging/stg__nta2020.sql +++ /dev/null @@ -1,14 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - -SELECT - globalid, - neighborhood_code, - - st_makevalid(linearize(geom)) AS geom, - geom AS raw_geom -FROM {{ source("recipe_sources", "dcp_cscl_nta2020") }} diff --git a/products/cscl/models/staging/stg__schooldistrict.sql b/products/cscl/models/staging/stg__schooldistrict.sql deleted file mode 100644 index 6216f6ff22..0000000000 --- a/products/cscl/models/staging/stg__schooldistrict.sql +++ /dev/null @@ -1,13 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - -SELECT - globalid, - schooldist, - st_makevalid(linearize(geom)) AS geom, - geom AS raw_geom -FROM {{ source("recipe_sources", "dcp_cscl_schooldistrict") }} diff --git a/products/cscl/models/staging/stg__statesenatedistrict.sql b/products/cscl/models/staging/stg__statesenatedistrict.sql deleted file mode 100644 index 0165b36f75..0000000000 --- a/products/cscl/models/staging/stg__statesenatedistrict.sql +++ /dev/null @@ -1,13 +0,0 @@ -{{ config( - materialized = 'table', - indexes=[ - {'columns': ['geom'], 'type': 'gist'}, - ] -) }} - -SELECT - globalid, - stsendist, - st_makevalid(linearize(geom)) AS geom, - geom AS raw_geom -FROM {{ source("recipe_sources", "dcp_cscl_statesenatedistrict") }} diff --git a/products/cscl/poc_validation/compare_gdb.py b/products/cscl/poc_validation/compare_gdb.py index a8e77c900b..bb07f32985 100644 --- a/products/cscl/poc_validation/compare_gdb.py +++ b/products/cscl/poc_validation/compare_gdb.py @@ -36,8 +36,22 @@ dcpy.geospatial.gdb (compare.py / report.py) - this script only supplies CSCL's own inputs (declared keys, known/expected diffs) and CLI/S3 wiring. -Report-only: writes a per-column CSV per gdb to output/validation_output/_comparison.csv -and logs a report. It never fails the build on a data mismatch. +Report-only: writes two CSVs per gdb (per-column summary and per-row flagged-feature +identifiers) to output/validation_output/, and logs a report. It never fails the +build on a data mismatch. + +output/validation_output/_comparison.csv - one row per (layer, column): null +rates, nunique, and the layer-level row counts/note. Says THAT a layer has N rows +modified/dev-only/prod-only. + +output/validation_output/_flagged_rows.csv - one row per flagged FEATURE (not +column): layer, status (only_in_dev/only_in_prod/modified), key_columns, key. Says +WHICH rows those were, so a flagged layer can be chased down by key directly (e.g. +SELECT ... WHERE "BoroCD" = '...') instead of re-deriving the affected rows by hand +(chat log 2026-10-06). Only written when at least one layer in this gdb actually has +a declared/guessed key - a keyless layer's diffs only ever show up as the aggregate +counts in the per-column CSV. Can be large for a layer with many real diffs - that's +the point, it's meant to enumerate exactly what's flagged, not summarize it. """ import csv @@ -258,6 +272,21 @@ def _compare_layers( writer.writerows(rows) logger.info(f"Per-column CSV written to {out_csv}") + # Per-column CSV above says THAT a layer has N rows modified/dev-only/ + # prod-only; this says WHICH ones, so a flagged layer can be chased down + # directly instead of re-deriving the affected keys by hand (chat log + # 2026-10-05/06). One row per flagged feature, not per column - can be + # large for a layer with many real diffs (that's the point: it's meant to + # enumerate exactly what's flagged, not summarize it). + flagged_rows = report.flagged_rows() + if flagged_rows: + out_flagged_csv = OUTPUT_DIR / f"{report_name}_flagged_rows.csv" + with out_flagged_csv.open("w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=list(flagged_rows[0].keys())) + writer.writeheader() + writer.writerows(flagged_rows) + logger.info(f"{len(flagged_rows):,} flagged rows written to {out_flagged_csv}") + @app.command() def run( diff --git a/products/cscl/poc_validation/prod_data_loader.py b/products/cscl/poc_validation/prod_data_loader.py index 5367b8af88..6472e6f873 100644 --- a/products/cscl/poc_validation/prod_data_loader.py +++ b/products/cscl/poc_validation/prod_data_loader.py @@ -263,6 +263,45 @@ def create_citywide_table(file: str): ) +def _pg_connection_string() -> str: + """ogr2ogr's `PG:...` connection string for CLIENT, shared by every fgdb loader.""" + engine_url = CLIENT.engine.url + host = engine_url.host + port = engine_url.port or 5432 + db_name = engine_url.database + user = engine_url.username + password = engine_url.password + return f"PG:host={host} port={port} dbname={db_name} user={user} password={password} active_schema={CLIENT.schema}" + + +def _ogr2ogr_load_layer(gdb_path: Path, layer: str, table_name: str) -> None: + """Load one FGDB layer into CLIENT's schema as table_name, replacing it if present. + + -nlt GEOMETRY handles layers of mixed/unknown geometry type; PG_USE_COPY=NO is + required for ogr2ogr to run its usual geometry conversions (COPY mode skips them). + """ + subprocess.run( + [ + "ogr2ogr", + "-f", + "PostgreSQL", + _pg_connection_string(), + str(gdb_path), + layer, + "-nln", + table_name, + "-nlt", + "GEOMETRY", + "-overwrite", + "-progress", + "--config", + "PG_USE_COPY", + "NO", + ], + check=True, + ) + + def load_production_lion_fgdb_layers(version: str): """ Downloads LION FGDB from DCP website and loads all layers into production_outputs schema. @@ -296,51 +335,86 @@ def load_production_lion_fgdb_layers(version: str): f"Expected layers {expected_layers} but found {layers}" ) - # Get connection info from CLIENT - # Build the PG connection string for ogr2ogr - engine_url = CLIENT.engine.url - host = engine_url.host - port = engine_url.port or 5432 - db_name = engine_url.database - user = engine_url.username - password = engine_url.password - - pg_connection = f"PG:host={host} port={port} dbname={db_name} user={user} password={password} active_schema={CLIENT.schema}" - # Load each layer with fgdb_ prefix for layer in layers: table_name = f"fgdb_{layer.lower()}" print(f"Loading {layer} -> {CLIENT.schema}.{table_name}") - - # Use ogr2ogr to load the layer - # -overwrite: replace table if it exists - # -nln: set the new layer name (table name) - # -nlt GEOMETRY: Use generic geometry type to handle all geometry types - # --config PG_USE_COPY NO: Disable COPY mode to allow geometry conversions - subprocess.run( - [ - "ogr2ogr", - "-f", - "PostgreSQL", - pg_connection, - str(gdb_path), - layer, - "-nln", - table_name, - "-nlt", - "GEOMETRY", - "-overwrite", - "-progress", - "--config", - "PG_USE_COPY", - "NO", - ], - check=True, - ) + _ogr2ogr_load_layer(gdb_path, layer, table_name) print(f"Successfully loaded {len(layers)} layers from LION {version}") +def _district_gdb_filename(version: str) -> str: + """Prod's district gdb filename, e.g. "v26C_Districts.gdb.zip" for version "26c". + + Capitalization is inconsistent with everywhere else this script deals with + `version`: the S3 folder segment stays lowercase (cscl_etl/26c/...) but the file + itself only uppercases the trailing release letter - confirmed against the real + object at cscl_etl/26c/v26C_Districts.gdb.zip, not a documented convention. + """ + return f"v{version[:-1]}{version[-1].upper()}_Districts.gdb.zip" + + +def _frozen_district_layers() -> set[str]: + """District gdb layers flagged `status: blocked` in seeds/lion_outputs.csv. + + These haven't been genuinely rebuilt in prod since 2009-2015, only re-exported + (docs/prod_bugs/013-gdb-creadate-staleness-fossils.md) - still under separate + investigation with another team as of this writing. Skipped entirely rather than + loaded-and-ignored: comparing our output against a FileGDB object prod hasn't + touched in over a decade wouldn't tell us anything about our own pipeline. + """ + with Path("seeds/lion_outputs.csv").open(newline="") as f: + return { + row["filename"] + for row in csv.DictReader(f) + if row["output_group"] == "district_gdb" and row["status"] == "blocked" + } + + +def load_production_district_gdb(version: str, force: bool = False): + """ + Downloads the district boundary FGDB from edm-private and loads every non-frozen + layer into production_outputs, same fgdb_ prefix convention as + load_production_lion_fgdb_layers - so gdb_nynta2010 compares against + production_outputs.fgdb_nynta2010, and so on. + + Frozen layers (see _frozen_district_layers) are skipped, not loaded. + """ + filename = _district_gdb_filename(version) + frozen = _frozen_district_layers() + + with tempfile.TemporaryDirectory() as tmpdir: + gdb_path = Path(tmpdir) / filename + print(f"Downloading {filename} ({version}) from edm-private...") + s3.download_file("edm-private", f"cscl_etl/{version}/{filename}", gdb_path) + + layers = fgdb.get_layers(gdb_path) + to_load = sorted(layer for layer in layers if layer not in frozen) + skipped = sorted(layer for layer in layers if layer in frozen) + print( + f"Found {len(layers)} layers; skipping {len(skipped)} frozen " + f"(docs/prod_bugs/013): {skipped}" + ) + + loaded = 0 + for layer in to_load: + table_name = f"fgdb_{layer.lower()}" + if not force and already_loaded(table_name, version): + print(f"{table_name} already holds {version}, skipping") + continue + + print(f"Loading {layer} -> {CLIENT.schema}.{table_name}") + _ogr2ogr_load_layer(gdb_path, layer, table_name) + record_load(table_name, version) + loaded += 1 + + print( + f"Successfully loaded {loaded} layers from district gdb {version} " + f"({len(to_load) - loaded} already up to date, {len(skipped)} frozen skipped)" + ) + + app = typer.Typer() @@ -565,5 +639,29 @@ def _load_prod_ldf( record_load(table_name, version) +@app.command("load_prod_district_gdb") +def _load_prod_district_gdb( + version: str | None = typer.Option(version, "--version", "-v"), + force: bool = typer.Option( + False, + "--force", + "-F", + help="Reload every non-frozen layer even if this version is already logged as loaded", + ), +): + """ + Load this release's production district gdb into production_outputs, for + in-database comparison against our own gdb_* district models. + + Skips layers frozen in prod (status=blocked in seeds/lion_outputs.csv, see + docs/prod_bugs/013-gdb-creadate-staleness-fossils.md) - still under separate + investigation with another team as of this writing. + """ + if not version: + raise Exception("Specify version with '-v'") + + load_production_district_gdb(version, force) + + if __name__ == "__main__": app() diff --git a/products/cscl/seeds/lion_outputs.csv b/products/cscl/seeds/lion_outputs.csv index fef1ce8b77..a7778ad893 100644 --- a/products/cscl/seeds/lion_outputs.csv +++ b/products/cscl/seeds/lion_outputs.csv @@ -55,46 +55,46 @@ lion_gdb,gdb,LION gdb,layer,node,gdb_node,true,true,NODEID,,done,Matches prod ex lion_gdb,gdb,LION gdb,layer,lion,gdb_lion,true,true,SegmentID|Join_ID,,open,ArcCenterX/ArcCenterY/Radius all implemented (CSCL-LION-07) - layer has no key that achieves precise row-level diffing (SAF-replicant multiplicity); Generic-segment house numbers not consistently zeroed in prod (CSCL-LION-15),Switched key_columns from LBoro|FaceCode|SeqNum - see CSCL-LION-10/CSCL-LION-07/CSCL-LION-15 updates. House-number check: qa__diffs_lion_house_numbers lion_gdb,gdb,LION gdb,layer,node_stname,gdb_node_stname,true,true,NODEID|STNAME,,done,Matches prod (38 residual rows out of 245K), lion_gdb,gdb,LION gdb,layer,altnames,gdb_altnames,true,true,PDir|PType|SName|SType|SDir|Street|Join_ID,,blocked,Prod altnames output is roughly a decade stale,See docs/prod_bugs/011-altnames-saf-replicant-join-ids.md -district_gdb,gdb,District gdb,layer,nyad,gdb_nyad,true,true,AssemDist,Assembly Districts clipped to shoreline,blocked,Layer frozen in prod since 2012,Layer frozen since 2012 - see docs/prod_bugs/013-gdb-creadate-staleness-fossils.md -district_gdb,gdb,District gdb,layer,nyadwi,gdb_nyadwi,true,true,AssemDist,"Assembly Districts, unclipped",done,"Matches prod exactly (unclipped, unaffected by CSCL-DISTRICTS-03)", -district_gdb,gdb,District gdb,layer,nyap,gdb_nyap,true,true,ATOMICID,Atomic Polygons,blocked,Layer frozen in prod since 2015,Layer frozen since 2015 - see docs/prod_bugs/013-gdb-creadate-staleness-fossils.md -district_gdb,gdb,District gdb,layer,nybb,gdb_nybb,true,true,BoroCode,Borough Boundaries clipped to shoreline,blocked,Layer frozen in prod since 2012,Layer frozen since 2012 - see docs/prod_bugs/013-gdb-creadate-staleness-fossils.md -district_gdb,gdb,District gdb,layer,nybbwi,gdb_nybbwi,true,true,BoroCode,"Borough Boundaries, unclipped",done,"Matches prod exactly (unclipped, unaffected by CSCL-DISTRICTS-03)", -district_gdb,gdb,District gdb,layer,nybid,gdb_nybid,true,true,BIDID,Business Improvement Districts,blocked,Layer frozen in prod since 2009,Layer frozen since 2009 - see docs/prod_bugs/013-gdb-creadate-staleness-fossils.md -district_gdb,gdb,District gdb,layer,nycb2010,gdb_nycb2010,true,true,BCTCB2010,2010 Census Blocks clipped to shoreline,open,"Shoreline-clip geometry noise (CSCL-DISTRICTS-03), not yet borough-bound", -district_gdb,gdb,District gdb,layer,nycb2010wi,gdb_nycb2010wi,true,true,BCTCB2010,"2010 Census Blocks, unclipped",done,"Matches prod exactly (unclipped, unaffected by CSCL-DISTRICTS-03)", -district_gdb,gdb,District gdb,layer,nycb2020,gdb_nycb2020,true,true,BCTCB2020,2020 Census Blocks clipped to shoreline,open,"Shoreline-clip geometry noise (CSCL-DISTRICTS-03), not yet borough-bound", -district_gdb,gdb,District gdb,layer,nycb2020wi,gdb_nycb2020wi,true,true,BCTCB2020,"2020 Census Blocks, unclipped",done,"Matches prod exactly (unclipped, unaffected by CSCL-DISTRICTS-03)", -district_gdb,gdb,District gdb,layer,nycc,gdb_nycc,true,true,CounDist,City Council Districts,open,"Shoreline-clip SHAPE_Length noise, worst on Queens coastline","See CSCL-DISTRICTS-03 in data_issues.md. Audited 2026-09-21: not a borough-bound (mechanism 2) issue like nynta - CD8's real cross-borough span isn't the cause either; residual is unclosed coastline seams on Queens' most complex shoreline (CD31/19/32)." -district_gdb,gdb,District gdb,layer,nyccwi,gdb_nyccwi,true,true,CounDist,"City Council Districts, unclipped",done,"Matches prod exactly (unclipped, unaffected by CSCL-DISTRICTS-03)", -district_gdb,gdb,District gdb,layer,nycd,gdb_nycd,true,true,BoroCD,Community Districts clipped to shoreline,open,"Shoreline-clip geometry noise (CSCL-DISTRICTS-03), not yet borough-bound", -district_gdb,gdb,District gdb,layer,nycdta2020,gdb_nycdta2020,true,true,CDTA2020,Community District Tabulation Areas 2020,open,"Shoreline-clip geometry noise (CSCL-DISTRICTS-03), not yet borough-bound", -district_gdb,gdb,District gdb,layer,nycdwi,gdb_nycdwi,true,true,BoroCD,"Community Districts, unclipped",blocked,Layer frozen in prod since 2009,Layer frozen since 2009 - see docs/prod_bugs/013-gdb-creadate-staleness-fossils.md -district_gdb,gdb,District gdb,layer,nycg,gdb_nycg,true,true,CongDist,Congressional Districts,open,"Shoreline-clip geometry noise (CSCL-DISTRICTS-03), not yet borough-bound", -district_gdb,gdb,District gdb,layer,nycgwi,gdb_nycgwi,true,true,CongDist,"Congressional Districts, unclipped",done,"Matches prod exactly (unclipped, unaffected by CSCL-DISTRICTS-03)", -district_gdb,gdb,District gdb,layer,nyct2010,gdb_nyct2010,true,true,BoroCT2010,2010 Census Tracts clipped to shoreline,open,"Shoreline-clip geometry noise (CSCL-DISTRICTS-03), not yet borough-bound", -district_gdb,gdb,District gdb,layer,nyct2010wi,gdb_nyct2010wi,true,true,BoroCT2010,"2010 Census Tracts, unclipped",Needs Investigation,,"Unclipped layer still shows 4 row-count mismatches (2 dev-only, 2 prod-only) - doesn't fit the shoreline-clip explanation since there's no clipping here." -district_gdb,gdb,District gdb,layer,nyct2020,gdb_nyct2020,true,true,BoroCT2020,2020 Census Tracts clipped to shoreline,open,Column order differs from prod; also has shoreline-clip geometry noise,"compare_gdb.py flags column ORDER differs from prod (separate from the row-level diff) in addition to the usual shoreline-clip geometry noise (CSCL-DISTRICTS-03) - the ordering issue looks like an easy, independent fix worth doing first." -district_gdb,gdb,District gdb,layer,nyct2020wi,gdb_nyct2020wi,true,true,BoroCT2020,"2020 Census Tracts, unclipped",done,"Matches prod exactly (unclipped, unaffected by CSCL-DISTRICTS-03)", -district_gdb,gdb,District gdb,layer,nyed,gdb_nyed,true,true,ElectDist,Election Districts,open,"Shoreline-clip geometry noise (CSCL-DISTRICTS-03), not yet borough-bound", -district_gdb,gdb,District gdb,layer,nyedwi,gdb_nyedwi,true,true,ElectDist,"Election Districts, unclipped",done,"Matches prod exactly (unclipped, unaffected by CSCL-DISTRICTS-03)", -district_gdb,gdb,District gdb,layer,nyfb,gdb_nyfb,true,true,FireBN,Fire Battalions,open,"Shoreline-clip geometry noise (CSCL-DISTRICTS-03), not yet borough-bound", -district_gdb,gdb,District gdb,layer,nyfc,gdb_nyfc,true,true,FireCoType|FireCoNum,Fire Companies,open,"Shoreline-clip geometry noise (CSCL-DISTRICTS-03), not yet borough-bound", -district_gdb,gdb,District gdb,layer,nyfd,gdb_nyfd,true,true,FireDiv,Fire Divisions,open,"Shoreline-clip SHAPE_Length noise; FireDiv 13 also has an above-noise area delta","See CSCL-DISTRICTS-03 in data_issues.md. Audited 2026-09-21: same Queens-coastline seam-gap pattern as nycc/nypuma; FireDiv 13's -0.118% area delta (vs <0.005% on other divisions) may be a distinct, second mechanism - not yet isolated." -district_gdb,gdb,District gdb,layer,nyha,gdb_nyha,true,true,BoroCode|HealthArea,Health Areas clipped to shoreline,open,"Shoreline-clip geometry noise (CSCL-DISTRICTS-03), not yet borough-bound", -district_gdb,gdb,District gdb,layer,nyhc,gdb_nyhc,true,true,HCentDist,Health Center Districts clipped to shoreline,open,"Shoreline-clip geometry noise (CSCL-DISTRICTS-03), not yet borough-bound", -district_gdb,gdb,District gdb,layer,nyhd,gdb_nyhd,true,true,NAME,Historic Districts,blocked,Layer frozen in prod since 2009,Layer frozen since 2009 - see docs/prod_bugs/013-gdb-creadate-staleness-fossils.md -district_gdb,gdb,District gdb,layer,nyhez,gdb_nyhez,true,true,HURRICANE_EVACUATION_ZONE,Hurricane Evacuation Zones,done,Matches prod exactly, -district_gdb,gdb,District gdb,layer,nymc,gdb_nymc,true,true,BoroCode|MuniCourt,Municipal Court Districts,open,"Shoreline-clip geometry noise (CSCL-DISTRICTS-03), not yet borough-bound", -district_gdb,gdb,District gdb,layer,nymcea,gdb_nymcea,true,true,BOROCODE|MCEA,"Minor Census Economic Areas, dissolved from 2000 Census Tracts by MCEA attribute, clipped to shoreline",blocked,Layer frozen in prod since 2009,Layer frozen since 2009 - see docs/prod_bugs/013-gdb-creadate-staleness-fossils.md -district_gdb,gdb,District gdb,layer,nymcwi,gdb_nymcwi,true,true,BoroCode|MuniCourt,"Municipal Court Districts, unclipped",done,"Matches prod exactly (unclipped, unaffected by CSCL-DISTRICTS-03)", -district_gdb,gdb,District gdb,layer,nynta2010,gdb_nynta2010,true,true,NTACode,Neighborhood Tabulation Areas 2010,in progress,Resolving water-boundary clipping issues,See CSCL-DISTRICTS-03 in data_issues.md -district_gdb,gdb,District gdb,layer,nynta2020,gdb_nynta2020,true,true,NTA2020,Neighborhood Tabulation Areas 2020,in progress,Resolving water-boundary clipping issues,See CSCL-DISTRICTS-03 in data_issues.md -district_gdb,gdb,District gdb,layer,nypp,gdb_nypp,true,true,Precinct,Police Precincts,open,"Shoreline-clip geometry noise (CSCL-DISTRICTS-03), not yet borough-bound", -district_gdb,gdb,District gdb,layer,nypuma2010,gdb_nypuma2010,true,true,PUMA,Public Use Microdata Areas (PUMA) 2010,open,"Shoreline-clip SHAPE_Length noise, worst on Rockaway/Jamaica Bay (PUMA 4105)","See CSCL-DISTRICTS-03 in data_issues.md. Audited 2026-09-21: every PUMA row shows some inflation; not a borough-bound issue. Also found an unrelated BoroCode mislabeling in source census tracts feeding PUMA 4101's dissolve (doesn't appear to explain the gap)." -district_gdb,gdb,District gdb,layer,nypuma2020,gdb_nypuma2020,true,true,PUMA,Public Use Microdata Areas (PUMA) 2020,open,"Shoreline-clip SHAPE_Length noise, worst on Rockaway/Jamaica Bay (PUMA 4105)","See CSCL-DISTRICTS-03 in data_issues.md. Audited 2026-09-21, same finding as nypuma2010 (worst on PUMA 4105/Rockaway) - see that row's notes for detail." -district_gdb,gdb,District gdb,layer,nysd,gdb_nysd,true,true,SchoolDist,School Districts,done,Matches prod exactly, -district_gdb,gdb,District gdb,layer,nyss,gdb_nyss,true,true,StSenDist,State Senatorial Districts,open,"Shoreline-clip geometry noise (CSCL-DISTRICTS-03), not yet borough-bound", -district_gdb,gdb,District gdb,layer,nysswi,gdb_nysswi,true,true,StSenDist,"State Senatorial Districts, unclipped",done,"Matches prod exactly (unclipped, unaffected by CSCL-DISTRICTS-03)", -district_gdb,gdb,District gdb,layer,nyura,gdb_nyura,true,true,URAID,Urban Renewal Areas,blocked,Layer frozen in prod since 2009 (nybid's stale batch),Layer frozen since 2009 (same batch as nybid - see KNOWN_STRUCTURAL_DIFFS in compare_gdb.py) - see docs/prod_bugs/013-gdb-creadate-staleness-fossils.md -district_gdb,gdb,District gdb,layer,nyzip,gdb_nyzip,true,true,ZIP_CODE,Zip Code Areas,blocked,Layer frozen in prod since 2009,Layer frozen since 2009 - see docs/prod_bugs/013-gdb-creadate-staleness-fossils.md +district_gdb,gdb,District gdb,layer,nyad,gdb_nyad,true,true,AssemDist,Assembly Districts built from the AtomicPolygon topology,done,Matches prod exactly,"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved." +district_gdb,gdb,District gdb,layer,nyadwi,gdb_nyadwi,true,true,AssemDist,"Assembly Districts, water included, built from the AtomicPolygon topology",done,Matches prod exactly,"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Water APs carrying the district's id are part of it." +district_gdb,gdb,District gdb,layer,nyap,gdb_nyap,true,true,ATOMICID,Atomic Polygons,done,Matches prod exactly,"The AtomicPolygon layer itself, published from the CSCL source." +district_gdb,gdb,District gdb,layer,nybb,gdb_nybb,true,true,BoroCode,Borough Boundaries built from the AtomicPolygon topology,done,Matches prod exactly,"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved." +district_gdb,gdb,District gdb,layer,nybbwi,gdb_nybbwi,true,true,BoroCode,"Borough Boundaries, water included, built from the AtomicPolygon topology",done,Matches prod exactly,"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Water APs carrying the district's id are part of it." +district_gdb,gdb,District gdb,layer,nybid,gdb_nybid,true,true,BIDID,Business Improvement Districts,done,Matches prod exactly,"Published from the CSCL source polygons as-is: its boundaries don't follow AtomicPolygon edges, so it isn't built from the topology." +district_gdb,gdb,District gdb,layer,nycb2010,gdb_nycb2010,true,true,BCTCB2010,2010 Census Blocks built from the AtomicPolygon topology,done,"12 rows differ, all explained: 10 rows are 5 block pairs where prod's block layer disagrees with the AtomicPolygons; 2 are a key label (our 10223013005 = prod 10223003005)","Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Prod's census block layer draws the line between a few block pairs somewhere other than the AtomicPolygon edges (an AP's area lands in the neighbouring block); we follow the APs. In both pairs that cross a tract line, prod's own tract layer agrees with the APs, contradicting prod's block layer. Pairs: 10212003000/10212004002 (Mn), 40383021003/40383021016 (Qn), 30393001000/30393001001 (Bk), 30294001008/30304001008 (Bk), 20235022000/20381002000 (Bx). Pending confirmation with GR. Label: the APs put block 10223003005's land in tract 022301, so we publish 10223013005 (identical geometry); prod's block layer says tract 022300." +district_gdb,gdb,District gdb,layer,nycb2010wi,gdb_nycb2010wi,true,true,BCTCB2010,"2010 Census Blocks, water included, built from the AtomicPolygon topology",done,"15 rows differ, all prod-side: the nycb2010 block pairs and label, block 39901000007/8 (prod's block layer contradicts prod's AP layer), block 10001001000 (Ellis Island patch outside the APs), and hole/ring representation","Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Water APs carrying the block are part of it. Prod's census block layer draws the line between a few block pairs somewhere other than the AtomicPolygon edges (an AP's area lands in the neighbouring block); we follow the APs. In both pairs that cross a tract line, prod's own tract layer agrees with the APs, contradicting prod's block layer. Pairs: 10212003000/10212004002 (Mn), 40383021003/40383021016 (Qn), 30393001000/30393001001 (Bk), 30294001008/30304001008 (Bk), 20235022000/20381002000 (Bx). Pending confirmation with GR. Block 39901000007/8: water AP 3990100031 is block 0007 in prod's own nyap, but prod's block layer draws its 34,508 sq ft into 39901000008; we follow the AP. Tract 1000100 / block 10001001000: prod's census layers include a 1,417 sq ft patch at Ellis Island that no AP covers and prod's own nybbwi and nyap don't either (prod's census lines run past the AP / borough edge there); we follow the APs. Blocks where we publish a hole and prod a self-touching ring (equal area), or two parts prod joins with a zero-area bridge, are representation only (see GR question inv-3vi.9)." +district_gdb,gdb,District gdb,layer,nycb2020,gdb_nycb2020,true,true,BCTCB2020,2020 Census Blocks built from the AtomicPolygon topology,done,"8 rows differ, all explained: 3 block pairs where prod's block layer disagrees with the AtomicPolygons, plus 2 blocks split by a source AP overlap/pinch point (area identical)","Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Prod's census block layer draws the line between a few block pairs somewhere other than the AtomicPolygon edges (an AP's area lands in the neighbouring block); we follow the APs. In both pairs that cross a tract line, prod's own tract layer agrees with the APs, contradicting prod's block layer. Pairs: 10212003000/10212004002 (Mn), 40383021003/40383021016 (Qn), 30393001000/30393001001 (Bk), 30294001008/30304001008 (Bk), 20235022000/20381002000 (Bx). Pending confirmation with GR. Blocks 40214003005 and 20395004012: identical area; ours has 2 parts where prod has 1 - in 40214003005 a 45 sq ft sliver AP (4021400227) that overlaps its neighbour in the source data becomes its own part (adds 386 ft of length); in 20395004012 the block's two halves meet at a single point." +district_gdb,gdb,District gdb,layer,nycb2020wi,gdb_nycb2020wi,true,true,BCTCB2020,"2020 Census Blocks, water included, built from the AtomicPolygon topology",done,"10 rows differ, all prod-side: the nycb2020 block pairs, block 10001001000 (Ellis Island patch outside the APs), and hole/part representation","Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Water APs carrying the block are part of it. Prod's census block layer draws the line between a few block pairs somewhere other than the AtomicPolygon edges (an AP's area lands in the neighbouring block); we follow the APs. In both pairs that cross a tract line, prod's own tract layer agrees with the APs, contradicting prod's block layer. Pairs: 10212003000/10212004002 (Mn), 40383021003/40383021016 (Qn), 30393001000/30393001001 (Bk), 30294001008/30304001008 (Bk), 20235022000/20381002000 (Bx). Pending confirmation with GR. Tract 1000100 / block 10001001000: prod's census layers include a 1,417 sq ft patch at Ellis Island that no AP covers and prod's own nybbwi and nyap don't either (prod's census lines run past the AP / borough edge there); we follow the APs. Blocks where we publish a hole and prod a self-touching ring (equal area), or two parts prod joins with a zero-area bridge, are representation only (see GR question inv-3vi.9)." +district_gdb,gdb,District gdb,layer,nycc,gdb_nycc,true,true,CounDist,City Council Districts built from the AtomicPolygon topology,done,"4 rows differ, all explained: council 38/47 (an ED the roster assigns to 47 that prod puts in 38), council 24 (the source AP overlap shared with ED 24035), council 32 (prod slivers, SHAPE_Length only)","Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Membership via the THINED ED roster. ED 61/109 (a water_flag 3 AP, 35,172 sq ft) is council 47 in the roster but in 38 in prod. Pending confirmation with GR." +district_gdb,gdb,District gdb,layer,nyccwi,gdb_nyccwi,true,true,CounDist,"City Council Districts, water included, built from the AtomicPolygon topology",done,"3 rows differ, all explained: council 38/47 (ED 61/112, a 27.7M sq ft water-only ED the roster assigns to 47, prod to 38) and council 24 (source AP overlap)","Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Water APs carrying the district are part of it. Pending confirmation with GR." +district_gdb,gdb,District gdb,layer,nycd,gdb_nycd,true,true,BoroCD,Community Districts built from the AtomicPolygon topology,done,Matches prod exactly,"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved." +district_gdb,gdb,District gdb,layer,nycdta2020,gdb_nycdta2020,true,true,CDTA2020,Community District Tabulation Areas 2020 built from the AtomicPolygon topology,done,"17 rows differ, all prod-side: needle sliver parts (366 parts < 1 sq ft), and spikes inherited from the NTAs","Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Prod's overlay-built layer carries hairline sliver parts (< 1 sq ft each) that inflate SHAPE_Length and part counts; ours has none. Spikes carry over from the NTAs (see nynta2020): MN03, MN01 and BX09 have spikes over 5 ft. Pending confirmation with GR that dropping them is fine." +district_gdb,gdb,District gdb,layer,nycdwi,gdb_nycdwi,true,true,BoroCD,"Community Districts, water included, built from the AtomicPolygon topology",done,Matches prod exactly,"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Water APs carrying the district's id are part of it." +district_gdb,gdb,District gdb,layer,nycg,gdb_nycg,true,true,CongDist,Congressional Districts built from the AtomicPolygon topology,done,Matches prod exactly,"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved." +district_gdb,gdb,District gdb,layer,nycgwi,gdb_nycgwi,true,true,CongDist,"Congressional Districts, water included, built from the AtomicPolygon topology",done,Matches prod exactly,"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Water APs carrying the district's id are part of it." +district_gdb,gdb,District gdb,layer,nyct2010,gdb_nyct2010,true,true,BoroCT2010,2010 Census Tracts built from the AtomicPolygon topology,done,"2 rows differ, both a key label: the Lower Bay islands off Staten Island are tract 5990100 in the APs, labeled 4990100 (a Queens tract) in prod","Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. The two islands (presumably Hoffman and Swinburne) are borough 5 in the AP attributes; prod's tract layer labels them 4990100 while prod's block layer labels the same land 59901000003 (SI). Geometry identical. Pending confirmation with GR." +district_gdb,gdb,District gdb,layer,nyct2010wi,gdb_nyct2010wi,true,true,BoroCT2010,"2010 Census Tracts, water included, built from the AtomicPolygon topology",done,"5 rows differ, all prod-side: 4 are tract-990100 labels (prod labels the Brooklyn, Queens and SI water tracts all 4990100; areas match); tract 1000100 has a 1,417 sq ft Ellis Island patch outside the APs","Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Water APs carrying the tract are part of it. Tract 1000100 / block 10001001000: prod's census layers include a 1,417 sq ft patch at Ellis Island that no AP covers and prod's own nybbwi and nyap don't either (prod's census lines run past the AP / borough edge there); we follow the APs." +district_gdb,gdb,District gdb,layer,nyct2020,gdb_nyct2020,true,true,BoroCT2020,2020 Census Tracts built from the AtomicPolygon topology,done,Matches prod exactly,"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved." +district_gdb,gdb,District gdb,layer,nyct2020wi,gdb_nyct2020wi,true,true,BoroCT2020,"2020 Census Tracts, water included, built from the AtomicPolygon topology",done,"1 row differs, prod-side: tract 1000100 has a 1,417 sq ft Ellis Island patch outside the APs","Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Water APs carrying the tract are part of it. Tract 1000100 / block 10001001000: prod's census layers include a 1,417 sq ft patch at Ellis Island that no AP covers and prod's own nybbwi and nyap don't either (prod's census lines run past the AP / borough edge there); we follow the APs." +district_gdb,gdb,District gdb,layer,nyed,gdb_nyed,true,true,ElectDist,Election Districts built from the AtomicPolygon topology,done,"1 row differs (ED 24035): area identical, +386 ft SHAPE_Length from a sliver AP that overlaps its neighbour in the source data","Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. AP 4021400227 (a 45 sq ft triangle) overlaps AP 4021400220 by 0.012 sq ft in the source; their edges cross, and after noding the triangle is its own part, adding its 386 ft perimeter. Membership via the THINED ED roster (AD + ED)." +district_gdb,gdb,District gdb,layer,nyedwi,gdb_nyedwi,true,true,ElectDist,"Election Districts, water included, built from the AtomicPolygon topology",done,1 row differs (ED 24035): the same source AP overlap as nyed (area identical),"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Water APs carrying the ED are part of it." +district_gdb,gdb,District gdb,layer,nyfb,gdb_nyfb,true,true,FireBN,Fire Battalions built from the AtomicPolygon topology,done,Matches prod exactly,"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved." +district_gdb,gdb,District gdb,layer,nyfc,gdb_nyfc,true,true,FireCoType|FireCoNum,Fire Companies built from the AtomicPolygon topology,done,Matches prod exactly,"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved." +district_gdb,gdb,District gdb,layer,nyfd,gdb_nyfd,true,true,FireDiv,Fire Divisions built from the AtomicPolygon topology,done,Matches prod exactly,"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved." +district_gdb,gdb,District gdb,layer,nyha,gdb_nyha,true,true,BoroCode|HealthArea,Health Areas built from the AtomicPolygon topology,done,Matches prod exactly,"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved." +district_gdb,gdb,District gdb,layer,nyhc,gdb_nyhc,true,true,HCentDist,Health Center Districts built from the AtomicPolygon topology,done,Matches prod exactly,"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved." +district_gdb,gdb,District gdb,layer,nyhd,gdb_nyhd,true,true,NAME,Historic Districts,done,Matches prod exactly,"Published from the CSCL source polygons as-is: its boundaries don't follow AtomicPolygon edges, so it isn't built from the topology." +district_gdb,gdb,District gdb,layer,nyhez,gdb_nyhez,true,true,HURRICANE_EVACUATION_ZONE,Hurricane Evacuation Zones built from the AtomicPolygon topology,done,Matches prod exactly,"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Water APs assigned a zone are part of it. Zone 5 wraps a zone-6 block near Clason Point (Bronx) that touches zone 5's outline only at corner points; districts are assembled by face membership rather than ring parity so that block stays zone 6, as in every source (AP attribute, CSCL HEZ polygons, prod). Whether that block really is zone 6 is a question for the data owner, pending." +district_gdb,gdb,District gdb,layer,nymc,gdb_nymc,true,true,BoroCode|MuniCourt,Municipal Court Districts built from the AtomicPolygon topology,done,"3 rows differ, all from the ED roster: court 404 includes ED 33/011 (36 Queens APs the roster labels court 04, absent from prod); courts 305/301 include water_flag 3 APs whose source court polygon is the '00' placeholder","Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Membership via the THINED ED roster; id is borough || court; the '00' placeholder is excluded. Pending confirmation with GR." +district_gdb,gdb,District gdb,layer,nymcea,gdb_nymcea,true,true,BOROCODE|MCEA,Minor Census Economic Areas built from the AtomicPolygon topology,done,"Matches prod: 122 rows, each matching a prod row on MCEA, area and part count (fixed after the 2026-10-09 build, which published one row per part)","Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Rows: one per connected piece of the MCEA's dissolved 2000 census tracts, as prod groups them - tract polygons include their water, so land parts across a waterway share a row (MCEA 0801: three rows holding 1, 1 and 16 parts)." +district_gdb,gdb,District gdb,layer,nymcwi,gdb_nymcwi,true,true,BoroCode|MuniCourt,"Municipal Court Districts, water included, built from the AtomicPolygon topology",done,"3 rows differ, the same ED-roster assignments as nymc","Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Water APs carrying the district are part of it. Pending confirmation with GR." +district_gdb,gdb,District gdb,layer,nynta2010,gdb_nynta2010,true,true,NTACode,Neighborhood Tabulation Areas 2010 built from the AtomicPolygon topology,done,"Matches prod exactly (the 1 row in the 2026-10-09 build, QN99's borough, was fixed after that build)","Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Borough is the borough holding most of the NTA's AP area: QN99 (Queens parks and cemeteries) has a few Brooklyn APs." +district_gdb,gdb,District gdb,layer,nynta2020,gdb_nynta2020,true,true,NTA2020,Neighborhood Tabulation Areas 2020 built from the AtomicPolygon topology,done,"32 rows differ, all prod-side: needle sliver parts (472 parts < 1 sq ft), and needle spikes on 9 NTAs; real part count matches exactly","Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Prod has 472 parts under 1 sq ft (8 sq ft in total, 77,120 ft of perimeter) across 32 NTAs; excluding them prod has exactly our 399 parts. Prod also has zero-area spikes (up to 240 ft) attached to 9 NTAs (MN1102, BX1001, MN1292, MN0101, MN0303, BK1803, MN0302, BX0903, MN0191); every vertex more than 5 ft from the other boundary is on prod's side. Compared at the same linearization (linearize()), no other difference is over 31 sq ft. Borough is the borough holding most of the NTA's AP area (Rikers Island's QN0151 belongs to a Queens CDTA but is in the Bronx)." +district_gdb,gdb,District gdb,layer,nypp,gdb_nypp,true,true,Precinct,Police Precincts: NYPD source precinct polygons clipped to land,done,"Area within 0.01% of prod for 77 of 78 precincts; precinct 113 (0.02%) is prod-side: prod clipped it to a different Jamaica Bay / JFK shoreline than prod's own nybb","Not built from the AtomicPolygon topology: NYPD's precinct and beat lines don't follow AP edges (they cut through hundreds of APs), so no AP-level assignment reproduces them. Land is the union of the topology-built borough boundaries (int__boundary__bb), so the shoreline matches every other layer; parts and holes under 100 sq ft are dropped (hairline slivers where a precinct line runs almost along the shoreline). Prod's own layer carries many more small parts (462 vs our 182), so part counts and SHAPE_Length may still differ. Precinct 113: all 43,419 sq ft only we have lies inside prod's own nybb (Queens), and only 7 of the 31,844 sq ft only prod has does; the pieces follow AP water_flag 2 (ours) vs 1 (prod) along the shore, so prod's precinct clip used an older or different shoreline than its borough layer." +district_gdb,gdb,District gdb,layer,nypuma2010,gdb_nypuma2010,true,true,PUMA,Public Use Microdata Areas (PUMA) 2010 built from the AtomicPolygon topology,done,Matches prod exactly,"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved." +district_gdb,gdb,District gdb,layer,nypuma2020,gdb_nypuma2020,true,true,PUMA,Public Use Microdata Areas (PUMA) 2020 built from the AtomicPolygon topology,done,Matches prod; the one flagged row (PUMA 4414) differs only by 6 needle-sliver parts < 1 sq ft in prod,"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. PUMA 4414: 20 parts ours vs 26 prod, the extra 6 all < 1 sq ft (100 ft of perimeter); area identical." +district_gdb,gdb,District gdb,layer,nysd,gdb_nysd,true,true,SchoolDist,School Districts built from the AtomicPolygon topology,done,Matches prod exactly,"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. District 10 is published once per borough (Bronx and Manhattan), every other district whole, as in prod." +district_gdb,gdb,District gdb,layer,nyss,gdb_nyss,true,true,StSenDist,State Senatorial Districts built from the AtomicPolygon topology,done,Matches prod exactly,"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved." +district_gdb,gdb,District gdb,layer,nysswi,gdb_nysswi,true,true,StSenDist,"State Senatorial Districts, water included, built from the AtomicPolygon topology",done,Matches prod exactly,"Built from the AtomicPolygon topology (int__boundary__*): districts are assembled from AP edges classified by membership, so boundaries follow AP edges exactly and no shoreline clipping or polygon overlay is involved. Water APs carrying the district's id are part of it." +district_gdb,gdb,District gdb,layer,nyura,gdb_nyura,true,true,URAID,Urban Renewal Areas,done,Matches prod (both empty),The source urban renewal area layer has no rows; prod has none either. +district_gdb,gdb,District gdb,layer,nyzip,gdb_nyzip,true,true,ZIP_CODE,Zip Code Areas,done,Matches prod exactly,"Published from the CSCL source polygons as-is: its boundaries don't follow AtomicPolygon edges, so it isn't built from the topology." diff --git a/python/geospatial/dcpy/geospatial/gdb/compare.py b/python/geospatial/dcpy/geospatial/gdb/compare.py index d47ad9108b..8c613b7d78 100644 --- a/python/geospatial/dcpy/geospatial/gdb/compare.py +++ b/python/geospatial/dcpy/geospatial/gdb/compare.py @@ -26,7 +26,7 @@ import uuid from collections import Counter from collections.abc import Collection -from dataclasses import dataclass +from dataclasses import dataclass, field import numpy as np import pandas as pd @@ -306,6 +306,18 @@ class RowLevelDiff: # precision. duplicate_key_dev_rows: int = 0 duplicate_key_prod_rows: int = 0 + # The actual row identities behind the counts above - each a composite key + # tuple (see composite_key), in no particular order. len(only_in_dev_keys) + # == only_in_dev always holds (same for prod/modified) - for a duplicated + # key with an excess on one side, that key appears once per excess copy, + # since individual same-keyed rows can't be told apart (see row_level_diff's + # docstring), only the key and how many more of it exist on one side. + # A caller that only wants counts (the original, pre-2026-10-06 behavior) + # can ignore these; a caller that wants to know WHICH rows triggered a + # diff (e.g. writing flagged identifiers to a file) now can. + only_in_dev_keys: list[tuple] = field(default_factory=list) + only_in_prod_keys: list[tuple] = field(default_factory=list) + modified_keys: list[tuple] = field(default_factory=list) def row_level_diff( @@ -360,20 +372,25 @@ def row_level_diff( dup_only_in_dev = 0 dup_only_in_prod = 0 + only_in_dev_keys: list[tuple] = [] + only_in_prod_keys: list[tuple] = [] if duplicated_keys: dup_dev_counts = Counter(dev_keys[is_dup_dev]) dup_prod_counts = Counter(prod_keys[is_dup_prod]) - dup_only_in_dev = sum( - max(c - dup_prod_counts.get(k, 0), 0) for k, c in dup_dev_counts.items() - ) - dup_only_in_prod = sum( - max(c - dup_dev_counts.get(k, 0), 0) for k, c in dup_prod_counts.items() - ) + for k, c in dup_dev_counts.items(): + excess = max(c - dup_prod_counts.get(k, 0), 0) + dup_only_in_dev += excess + only_in_dev_keys.extend([k] * excess) + for k, c in dup_prod_counts.items(): + excess = max(c - dup_dev_counts.get(k, 0), 0) + dup_only_in_prod += excess + only_in_prod_keys.extend([k] * excess) dev_precise_df = dev_df[~is_dup_dev] prod_precise_df = prod_df[~is_dup_prod] modified: int | None = None + modified_keys: list[tuple] = [] only_in_dev = dup_only_in_dev only_in_prod = dup_only_in_prod if len(dev_precise_df) > 0 or len(prod_precise_df) > 0: @@ -385,20 +402,22 @@ def row_level_diff( modified = 0 if len(common) > 0 and compare_cols: - modified = int( - columns_differ( - dev_indexed.loc[common], - prod_indexed.loc[common], - compare_cols, - blank_as_null, - tolerant_float_cols, - rtol, - atol, - ).sum() + diff_mask = columns_differ( + dev_indexed.loc[common], + prod_indexed.loc[common], + compare_cols, + blank_as_null, + tolerant_float_cols, + rtol, + atol, ) + modified = int(diff_mask.sum()) + modified_keys = list(common[diff_mask.to_numpy()]) only_in_dev += len(dev_only_index) only_in_prod += len(prod_only_index) + only_in_dev_keys.extend(dev_only_index) + only_in_prod_keys.extend(prod_only_index) return RowLevelDiff( only_in_dev=only_in_dev, @@ -407,6 +426,9 @@ def row_level_diff( precise=precise, duplicate_key_dev_rows=int(is_dup_dev.sum()), duplicate_key_prod_rows=int(is_dup_prod.sum()), + only_in_dev_keys=only_in_dev_keys, + only_in_prod_keys=only_in_prod_keys, + modified_keys=modified_keys, ) diff --git a/python/geospatial/dcpy/geospatial/gdb/fgdb.py b/python/geospatial/dcpy/geospatial/gdb/fgdb.py index 8dd5234c87..ee65fa8df4 100644 --- a/python/geospatial/dcpy/geospatial/gdb/fgdb.py +++ b/python/geospatial/dcpy/geospatial/gdb/fgdb.py @@ -78,19 +78,16 @@ def zip_gdb(gdb_dir: Path, zip_path: Path) -> None: # possible TODO: replace this fn with an Info() object, from which methods like .get_layers() can be called def get_layers(gdb: Path) -> list[str]: - # Imported lazily: eagerly importing osgeo.gdal at module load time (this - # module is imported as part of the dcpy CLI bootstrap) triggers GDAL's - # driver registration, which collides with pyarrow's Arrow filesystem - # registration ("Attempted to register factory for scheme 'file' but that - # scheme is already registered") if pyarrow is used anywhere else in the - # same process. - from osgeo import gdal - - with gdal.ExceptionMgr(): - info = gdal.alg.vector.info( - input=gdb, - ).Output() - return info["rootGroup"]["layerNames"] + # Delegates to layer_geometry_types (pyogrio-based, already proven against + # both classic and modern FileGDB formats via compare_gdb.py) rather than + # GDAL's vector.info() algorithm directly - that API's Output()["rootGroup"] + # only exists for geodatabases with a hierarchical group structure (modern/ + # 10.x+ FileGDBs). A "classic" pre-10.x FileGDB (e.g. CSCL's district gdb - + # see docs/prod_bugs/013-gdb-creadate-staleness-fossils.md) has no rootGroup + # at all; vector.info() returns a flat "layers" list instead, and the old + # info["rootGroup"]["layerNames"] lookup raised KeyError('rootGroup') for + # every such file (confirmed on v26C_Districts.gdb.zip, chat log 2026-10-05). + return list(layer_geometry_types(gdb).keys()) def resolve_layer(gdb: Path, layer: str | None = None) -> str: diff --git a/python/geospatial/dcpy/geospatial/gdb/report.py b/python/geospatial/dcpy/geospatial/gdb/report.py index faa7e24a16..aeac0979a2 100644 --- a/python/geospatial/dcpy/geospatial/gdb/report.py +++ b/python/geospatial/dcpy/geospatial/gdb/report.py @@ -329,6 +329,47 @@ def rows(self) -> list[dict]: ) return out + def flagged_rows(self) -> list[dict]: + """One dict per flagged ROW (not column) across every layer added so + far - the actual identities behind rows_only_in_dev/rows_only_in_prod/ + rows_modified in rows()'s per-layer counts, for a caller that needs to + know WHICH rows triggered a diff, not just how many (e.g. writing a + per-row CSV so someone can go look at the specific features flagged by + a build, rather than re-deriving them by hand - see + RowLevelDiff.only_in_dev_keys/only_in_prod_keys/modified_keys). + + Empty for a layer with no declared/guessed key (key_cols == []) or one + added via add_missing_layer (no per-row diff is possible against a + side that doesn't exist at all) - those only ever show up in rows()'s + layer-level counts. + + key is each flagged row's composite key, formatted as its key_cols' + values joined by " | " in key_cols order - readable on its own, and + paired with the key_columns field here so a multi-column key's parts + aren't ambiguous. + """ + out = [] + for entry in self.layers: + key_cols = entry.comparison.key_cols + if not key_cols: + continue + row_level = entry.comparison.row_level + for status, keys in ( + ("only_in_dev", row_level.only_in_dev_keys), + ("only_in_prod", row_level.only_in_prod_keys), + ("modified", row_level.modified_keys), + ): + for k in keys: + out.append( + { + "layer": entry.layer, + "status": status, + "key_columns": ", ".join(key_cols), + "key": " | ".join(str(part) for part in k), + } + ) + return out + @property def clean_layer_count(self) -> int: return sum(1 for entry in self.layers if entry.is_clean) diff --git a/python/geospatial/test/gdb/test_compare.py b/python/geospatial/test/gdb/test_compare.py index a49ebdae97..100e4274ba 100644 --- a/python/geospatial/test/gdb/test_compare.py +++ b/python/geospatial/test/gdb/test_compare.py @@ -41,7 +41,11 @@ def test_row_added_in_dev_counts_as_dev_only(nta_gdf): prod = nta_gdf[nta_gdf.NTACode != NTA_INSANE_KEY].reset_index(drop=True) result = compare.row_level_diff(dev, prod, NTA_KEY, _compare_cols(dev, NTA_KEY)) assert result == compare.RowLevelDiff( - only_in_dev=1, only_in_prod=0, modified=0, precise=True + only_in_dev=1, + only_in_prod=0, + modified=0, + precise=True, + only_in_dev_keys=[(NTA_INSANE_KEY,)], ) @@ -50,7 +54,11 @@ def test_row_removed_from_dev_counts_as_prod_only(nta_gdf): prod = nta_gdf result = compare.row_level_diff(dev, prod, NTA_KEY, _compare_cols(dev, NTA_KEY)) assert result == compare.RowLevelDiff( - only_in_dev=0, only_in_prod=1, modified=0, precise=True + only_in_dev=0, + only_in_prod=1, + modified=0, + precise=True, + only_in_prod_keys=[(NTA_INSANE_KEY,)], ) @@ -74,7 +82,11 @@ def test_modified_attribute_on_insane_geometry_row_counts_as_modified(nta_gdf): prod.loc[prod.NTACode == NTA_INSANE_KEY, "NTAName"] = "changed name" result = compare.row_level_diff(dev, prod, NTA_KEY, _compare_cols(dev, NTA_KEY)) assert result == compare.RowLevelDiff( - only_in_dev=0, only_in_prod=0, modified=1, precise=True + only_in_dev=0, + only_in_prod=0, + modified=1, + precise=True, + modified_keys=[(NTA_INSANE_KEY,)], ) @@ -84,7 +96,11 @@ def test_modified_attribute_on_simple_geometry_row_counts_as_modified(nta_gdf): prod.loc[prod.NTACode == NTA_SIMPLE_KEY, "BoroCode"] = 99 result = compare.row_level_diff(dev, prod, NTA_KEY, _compare_cols(dev, NTA_KEY)) assert result == compare.RowLevelDiff( - only_in_dev=0, only_in_prod=0, modified=1, precise=True + only_in_dev=0, + only_in_prod=0, + modified=1, + precise=True, + modified_keys=[(NTA_SIMPLE_KEY,)], ) @@ -97,7 +113,12 @@ def test_add_remove_and_modify_combine_correctly(nta_gdf): prod.loc[prod.NTACode == NTA_SIMPLE_KEY, "NTAName"] = "changed name" result = compare.row_level_diff(dev, prod, NTA_KEY, _compare_cols(dev, NTA_KEY)) assert result == compare.RowLevelDiff( - only_in_dev=1, only_in_prod=0, modified=1, precise=True + only_in_dev=1, + only_in_prod=0, + modified=1, + precise=True, + only_in_dev_keys=[(NTA_INSANE_KEY,)], + modified_keys=[(NTA_SIMPLE_KEY,)], ) @@ -315,7 +336,11 @@ def test_composite_key_does_not_collide_when_a_part_contains_the_join_separator( assert keys.is_unique result = compare.row_level_diff(dev, prod, ["k1", "k2"], ["val"]) assert result == compare.RowLevelDiff( - only_in_dev=0, only_in_prod=0, modified=1, precise=True + only_in_dev=0, + only_in_prod=0, + modified=1, + precise=True, + modified_keys=[("X|Y", "Z")], ) @@ -326,9 +351,15 @@ def test_null_keyed_rows_are_never_paired_with_each_other(): dev = pd.DataFrame({"id": ["A", None], "val": ["a-value", "dev-unrelated"]}) prod = pd.DataFrame({"id": ["A", None], "val": ["a-value", "prod-unrelated"]}) result = compare.row_level_diff(dev, prod, ["id"], ["val"]) - assert result == compare.RowLevelDiff( - only_in_dev=1, only_in_prod=1, modified=0, precise=True - ) + assert result.only_in_dev == 1 + assert result.only_in_prod == 1 + assert result.modified == 0 + assert result.precise is True + # Each NULL-keyed row gets a one-off salted identity (see composite_key), + # so the exact key value is non-deterministic - just check one of each + # category showed up, not what it says. + assert len(result.only_in_dev_keys) == 1 + assert len(result.only_in_prod_keys) == 1 def test_key_identity_ignores_blank_as_null(): @@ -340,7 +371,12 @@ def test_key_identity_ignores_blank_as_null(): prod = pd.DataFrame({"code": ["A", " "], "val": ["a-value", "prod-record"]}) result = compare.row_level_diff(dev, prod, ["code"], ["val"], blank_as_null=True) assert result == compare.RowLevelDiff( - only_in_dev=1, only_in_prod=1, modified=0, precise=True + only_in_dev=1, + only_in_prod=1, + modified=0, + precise=True, + only_in_dev_keys=[("",)], + only_in_prod_keys=[(" ",)], ) @@ -394,7 +430,12 @@ def test_row_level_diff_on_disjoint_frames_reports_full_replacement(ad_gdf): prod = ad_gdf[ad_gdf.AssemDist == 48].reset_index(drop=True) result = compare.row_level_diff(dev, prod, AD_KEY, _compare_cols(dev, AD_KEY)) assert result == compare.RowLevelDiff( - only_in_dev=1, only_in_prod=1, modified=0, precise=True + only_in_dev=1, + only_in_prod=1, + modified=0, + precise=True, + only_in_dev_keys=[("43",)], + only_in_prod_keys=[("48",)], ) diff --git a/python/geospatial/test/gdb/test_report.py b/python/geospatial/test/gdb/test_report.py index 687477c2de..37f4af0e5d 100644 --- a/python/geospatial/test/gdb/test_report.py +++ b/python/geospatial/test/gdb/test_report.py @@ -220,6 +220,54 @@ def test_rows_produces_one_dict_per_layer_column(nta_gdf, ad_gdf): assert ad_row["area_pct_diff"] == "" +def test_flagged_rows_lists_the_actual_dev_only_prod_only_and_modified_keys(): + """flagged_rows() is the per-ROW counterpart to rows()'s per-column counts - + a caller that sees "1 dev-only, 1 modified" in a layer's note should be + able to find out WHICH rows those were from here, not just how many.""" + dev = pd.DataFrame({"key": ["A", "B", "C"], "val": ["a", "b", "c"]}) + prod = pd.DataFrame({"key": ["A", "B", "D"], "val": ["a", "b-changed", "d"]}) + + r = report.GdbComparisonReport() + result = compare.compare_layer(dev, prod, declared_key=["key"]) + r.add_layer("some_layer", result, len(dev), len(prod)) + + flagged = r.flagged_rows() + by_status = {(row["status"], row["key"]) for row in flagged} + assert by_status == { + ("only_in_dev", "C"), + ("only_in_prod", "D"), + ("modified", "B"), + } + assert all(row["layer"] == "some_layer" for row in flagged) + assert all(row["key_columns"] == "key" for row in flagged) + + +def test_flagged_rows_formats_a_composite_key_with_its_column_names(): + dev = pd.DataFrame({"k1": ["X"], "k2": ["Y"], "val": ["old"]}) + prod = pd.DataFrame({"k1": ["X"], "k2": ["Y"], "val": ["new"]}) + + r = report.GdbComparisonReport() + result = compare.compare_layer(dev, prod, declared_key=["k1", "k2"]) + r.add_layer("some_layer", result, len(dev), len(prod)) + + flagged = r.flagged_rows() + assert len(flagged) == 1 + assert flagged[0]["key_columns"] == "k1, k2" + assert flagged[0]["key"] == "X | Y" + + +def test_flagged_rows_is_empty_for_a_clean_layer_and_for_a_missing_one(): + r = report.GdbComparisonReport() + r.log_layer_structure({"nyzip": "MultiPolygon"}, {}) + r.add_missing_layer("nyzip", dev_row_count=263, prod_row_count=0) + + clean = pd.DataFrame({"key": ["A"], "val": ["a"]}) + result = compare.compare_layer(clean, clean.copy(), declared_key=["key"]) + r.add_layer("some_layer", result, len(clean), len(clean)) + + assert r.flagged_rows() == [] + + def test_log_summary_and_clean_layer_count_track_multiple_layers(nta_gdf, ad_gdf): r = report.GdbComparisonReport() clean = compare.compare_layer(ad_gdf, ad_gdf.copy(), declared_key=AD_KEY)