Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 70 additions & 1 deletion packages/core/src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
* mappit-core load <path> --export output.json
* mappit-core load <path> --export output.kml
* mappit-core load <path> --stats
* mappit-core load <path> --quality # validation + quality metrics
* mappit-core load <path> --sanitize --export clean.json # drop invalid items
*/

import * as fs from 'node:fs';
Expand All @@ -20,7 +22,7 @@
import ora from 'ora';

import { VERSION } from './index';
import { detectFormat, parseAuto } from './loaders/auto-detect';

Check warning on line 25 in packages/core/src/cli.ts

View workflow job for this annotation

GitHub Actions / Lint

'detectFormat' is defined but never used
import { parseRecords } from './loaders/records';
import { parseTimelineStandard } from './loaders/timeline-standard';
import { parseTimelineSemantic } from './loaders/timeline-semantic';
Expand All @@ -29,6 +31,7 @@
import { filterByDateRange, filterByArea, filterByActivityType } from './filters';
import type { BoundingBox } from './filters';
import { computeSummary, computeYearlySummary } from './stats';
import { validateDataset, sanitizeDataset, computeQualityMetrics } from './quality';
import { exportToJson } from './exporters/json-exporter';
import { exportToKml } from './exporters/kml-exporter';
import type { MappitDataset, DataSource } from './types';
Expand Down Expand Up @@ -188,6 +191,48 @@
console.log();
}

/**
* Pretty-print validation report and quality metrics to stdout.
*/
function printQuality(ds: MappitDataset): void {
const v = validateDataset(ds);
const q = computeQualityMetrics(ds);

console.log('\n Data Quality');
console.log('─'.repeat(45));
console.log(`Validation: ${v.valid ? 'ok' : `${v.issueCount.toLocaleString()} issues`}`);
for (const [reason, count] of Object.entries(v.byReason)) {
if (count > 0) console.log(` ${reason.padEnd(20)} ${count.toLocaleString().padStart(10)}`);
}

const cov = q.activityPathCoverage;
if (cov.ratio !== null) {
console.log(
`Activities w/ path: ${cov.withPath.toLocaleString()} / ${cov.activities.toLocaleString()} (${(cov.ratio * 100).toFixed(1)}%)`,
);
}

if (q.accuracy) {
const a = q.accuracy;
console.log(`Accuracy (m): median ${a.median}, p90 ${a.p90}, max ${a.max}`);
for (const b of a.buckets) {
console.log(` ${b.label.padEnd(20)} ${b.count.toLocaleString().padStart(10)}`);
}
}

const g = q.gaps;
if (g.basis !== 'none') {
console.log(
`Gaps > ${g.thresholdMinutes} min: ${g.count.toLocaleString()} (${g.totalHours.toFixed(1)} h, on ${g.basis})`,
);
for (const gap of g.largest) {
console.log(` ${gap.start} → ${gap.end} ${gap.hours.toFixed(1)} h`);
}
}

console.log();
}

/**
* Apply all filters specified via CLI flags.
*/
Expand Down Expand Up @@ -289,6 +334,17 @@
describe: 'Print summary statistics',
type: 'boolean',
default: false,
})
.option('quality', {
alias: 'q',
describe: 'Print validation report and data quality metrics',
type: 'boolean',
default: false,
})
.option('sanitize', {
describe: 'Drop invalid points and timeline entries before filtering',
type: 'boolean',
default: false,
}),
(cmdArgv) => {
const spinner = ora({ stream: process.stdout });
Expand All @@ -302,6 +358,16 @@
`Loaded ${ds.source} — ${ds.points.length} points, ${ds.timeline.length} timeline entries`,
);

// 1b — Sanitize
if (cmdArgv.sanitize) {
const before = ds.points.length + ds.timeline.length;
ds = sanitizeDataset(ds);
const removed = before - ds.points.length - ds.timeline.length;
spinner.succeed(
`Sanitized — ${removed} invalid ${removed === 1 ? 'item' : 'items'} removed`,
);
}

// 2 — Filter
const hasFilters =
cmdArgv.filterDate || cmdArgv.filterArea || cmdArgv.filterActivity;
Expand All @@ -319,9 +385,12 @@
}

// 3 — Stats
if (cmdArgv.stats || (!cmdArgv.export && !hasFilters)) {
if (cmdArgv.stats || (!cmdArgv.export && !hasFilters && !cmdArgv.quality)) {
printSummary(ds);
}
if (cmdArgv.quality) {
printQuality(ds);
}

// 4 — Export
if (cmdArgv.export) {
Expand Down
3 changes: 2 additions & 1 deletion packages/core/src/filters.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
MappitDataset,
LocationPoint,
TimelineEntry,
PlaceVisit,

Check warning on line 11 in packages/core/src/filters.ts

View workflow job for this annotation

GitHub Actions / Lint

'PlaceVisit' is defined but never used
ActivitySegment,
} from './types';

Expand Down Expand Up @@ -37,7 +37,8 @@
);
}

function recomputeDateRange(
/** @internal Recompute `dateRange` from the remaining points and entries. */
export function recomputeDateRange(
points: LocationPoint[],
timeline: TimelineEntry[],
): { min: string; max: string } {
Expand Down
14 changes: 14 additions & 0 deletions packages/core/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,20 @@ export {
type PeriodSummary,
} from './stats';

// Data validation & quality metrics
export {
validateDataset,
sanitizeDataset,
computeQualityMetrics,
type ValidationReason,
type ValidationIssue,
type ValidationReport,
type ValidateOptions,
type TimelineGap,
type QualityMetrics,
type QualityOptions,
} from './quality';

// Phase 2.4–2.5 — Exporters
export {
exportToJson,
Expand Down
Loading
Loading