fix: store the final Drain3 template, not the first line a cluster saw

README.md

@@ -394,7 +394,7 @@ Individually: `uv run ruff format`, `uv run ruff check --fix`,
 | Assume a reused `LogReducer` starts clean | Nothing -- `_reset_components()` rebuilds them per run | The dedup seen-set, both Drain3 miners and the fuzzy LSH accumulate per line. Carried into the next run they drop every line as already-seen and return empty (`core.py`) |
 | Share one deduplicator across hybrid's two passes | Rebuild it before the anomaly pass | The pattern pass consumes the seen-set, so the anomaly pass sees everything as seen and no anomalies survive. Hybrid silently degrades to pattern-only (`core.py`) |
 | Assume an unknown config keyword is ignored | Read the `ValueError` and fix the name | A silently dropped override means a user's tuning quietly does nothing (`core.py`) |
-| Read `LogPattern.template` as the final Drain3 template | Treat it as a first-seen snapshot until #5 lands | `extract_patterns` captures `cluster.get_template()` when a cluster FIRST appears, before later lines widen slots to `<*>`, so a varying cluster stores its raw first line. The `<*>`-count priority boost then runs on that stale snapshot (#5, open) |
+| Assume every template is the final Drain3 one | Allow for a cluster the `max_clusters` cap evicted | `extract_patterns` re-reads each template from the live cluster once the feed ends, so a varying cluster carries its `<*>` slots and the priority boost counts them. A cluster LRU-evicted mid-feed is gone by then and keeps the template it had when first seen (`patterns.py`) |
 | Hand-edit the version | Land a conventional commit and let semantic-release do it | The old regime wrote `VERSION` by hand and took five commits to get tagging working. CONTRIBUTING makes it a standing rule |
 
 ### Where this sits

SECURITY.md

@@ -5,9 +5,10 @@
 If you discover a security vulnerability in this project, please report it
 responsibly. We take security seriously and will respond promptly.
 
-**Email**: [email protected]
+**Email**: <[email protected]>
 
 Please include:
+
 - A description of the vulnerability
 - The affected component or version
 - Steps to reproduce the issue
@@ -55,6 +56,6 @@ The following are generally out of scope:
 
 ## Contact
 
-**Security reports**: [email protected]
+**Security reports**: <[email protected]>
 
 For non-security issues, please use the project's issue tracker.

data/samples/README.md

@@ -11,7 +11,7 @@ demonstration and testing. If you use them, cite LogHub:
 
 > Jieming Zhu, Shilin He, Pinjia He, Jinyang Liu, Michael R. Lyu. "Loghub: A
 > Large Collection of System Log Datasets for AI-driven Log Analytics." ISSRE
-> 2023. https://github.com/logpai/loghub
+> 2023. <https://github.com/logpai/loghub>
 
 Some datasets originate from public sources - the BGL and Thunderbird traces
 come from the USENIX CFDR computer failure data repository (Sandia National

pyproject.toml

@@ -24,7 +24,7 @@
     "Intended Audience :: Developers",
     "Intended Audience :: System Administrators",
     "Topic :: System :: Logging",
     "Topic :: System :: Systems Administration", 
     "Topic :: Software Development :: Libraries :: Python Modules",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
@@ -37,13 +37,13 @@
     "loguru>=0.7.3",
     "numpy>=2.0.0",
     "scikit-learn>=1.5.0",  # imports as sklearn; >=1.5 for numpy 2 support
-    "typer>=0.25.0",        # CLI
+    "typer>=0.27.3",        # CLI
 ]
 
 [project.optional-dependencies]
 enhanced = [
-    "xxhash>=3.0.0",      # Fast xxh64 dedup hashing (memory.BoundedDeduplicator)
-    "datasketch>=1.5.0",  # MinHash/LSH similarity in fuzzy dedup
+    "xxhash>=4.0.1",      # Fast xxh64 dedup hashing (memory.BoundedDeduplicator)
+    "datasketch>=2.0.0",  # MinHash/LSH similarity in fuzzy dedup
 ]
 
 # IO adapters - thin OPTIONAL conveniences over the Source/Sink abstraction.
@@ -110,12 +110,26 @@
 fixable = ["ALL"]
 unfixable = []
 dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+# vulture's own codes, which it reads from `# noqa`.
+external = ["V"]
 
 [tool.ruff.lint.per-file-ignores]
 "tests/**" = [
     "S101", "S105", "S106", "S108", "F541", "RUF013",
     "PT017", "PT011", "PT012", "RUF043",
 ]
+# The quality gate's S and D passes select their rules on the command line,
+# which drops the ignore list above, so their exemptions live here.
+# The class docstring documents construction and the dunder protocol methods.
+"src/**" = ["D105", "D107"]
+# The samplers wrap SQL the caller wrote, by design: identifiers are quoted and
+# every interpolated number goes through int() or float().
+"src/logreducer/sampling.py" = ["S608"]
+"src/logreducer/clickhouse.py" = ["S608"]
+# Sampling and jitter, never anything secret.
+"src/logreducer/core.py" = ["S311"]
+"src/logreducer/memory.py" = ["S311"]
+"src/logreducer/target.py" = ["S311"]
 
 [tool.ruff.lint.pydocstyle]
 convention = "google"
@@ -201,6 +215,10 @@
 # when transitive deps pin older lower bounds; revisit each /deps pass.
 [tool.uv]
 index-strategy = "unsafe-best-match"
+# Supply-chain cooldown (standards/universal/ci.md): never resolve a package
+# version published in the last 7 days. A CVE fix bypasses it through a
+# temporary per-package exclude-newer-package entry, not by lowering this.
+exclude-newer = "7 days"
 override-dependencies = []
 
 [[tool.uv.index]]
@@ -215,6 +233,11 @@
 [tool.ty.src]
 exclude = ["examples/", "docs/", "scripts/"]
 
+# Below 80 vulture reports the library's public API and attributes set on
+# third-party config objects, which buries a real leftover.
+[tool.vulture]
+min_confidence = 80
+
 # vermin: verify no syntax/features newer than the minimum Python.
 [tool.vermin]
 targets = ["3.12"]
@@ -233,7 +256,7 @@
 dev = [
     "pytest>=9.0.0",         # Test framework (9.0 changed warning/path semantics)
     "pytest-cov>=7.0.0",     # Coverage reporting
-    "ruff>=0.15.0",          # Lint + format (replaces black/flake8/isort)
+    "ruff>=0.16.10",         # Lint + format (replaces black/flake8/isort)
     "ty>=0.0.34",            # Astral type checker (runs alongside mypy)
     "mypy>=2.0",             # Static type checking (strict; 2.0 flipped defaults)
     "bandit[toml]>=1.7.0",   # Security linting
@@ -245,7 +268,7 @@
     "sqlalchemy>=2.0.30",         # SQLSource against SQLite (real, no server)
     "clickhouse-connect>=1.0",    # ClickHouseSource (local cluster or docker)
     "confluent-kafka>=2.14.0",    # KafkaSource/KafkaSink
-    "testcontainers[clickhouse,kafka,postgres,mysql]>=4.14.0",  # docker fallback (Redpanda module ships in base)
+    "testcontainers[clickhouse,kafka,postgres,mysql]>=4.15.0",  # docker fallback; modules under testcontainers.community
     "psycopg[binary]>=3.2",       # PostgreSQL driver (real TABLESAMPLE/setseed sampling tests)
     "pymysql>=1.1.0",             # MySQL driver (real RAND(seed) sampling tests)
     "python-dotenv>=1.0.0",       # load the local .env (CLICKHOUSE_*) in tests

renovate.json

@@ -5,12 +5,14 @@
     {
       "description": "pytest and its plugins version together to avoid plugin/core API mismatches.",
       "matchPackageNames": ["pytest", "/^pytest-/"],
-      "groupName": "pytest"
+      "groupName": "pytest",
+      "minimumReleaseAge": "7 days"
     },
     {
       "description": "Scientific stack (numpy, scipy, scikit-learn) tends to move together; group to catch ABI mismatches in one PR.",
       "matchPackageNames": ["numpy", "scipy", "scikit-learn"],
-      "groupName": "scientific"
+      "groupName": "scientific",
+      "minimumReleaseAge": "7 days"
     }
   ]
 }

src/logreducer/anomaly.py

@@ -1,6 +1,4 @@
-"""
-Anomaly detection utilities
-"""
+"""Anomaly detection utilities."""
 
 from loguru import logger
 
@@ -15,7 +13,7 @@
 
 
 class AnomalyDetector:
-    """Detect anomalous log lines"""
+    """Detect anomalous log lines."""
 
     def __init__(self, contamination: float = 0.1):
         self.contamination = contamination
@@ -25,8 +23,7 @@ def __init__(self, contamination: float = 0.1):
             logger.warning("scikit-learn not available, anomaly detection disabled")
 
     def detect_anomalies(self, lines: list[str]) -> tuple[list[str], list[str]]:
-        """
-        Separate anomalous and normal lines
+        """Separate anomalous and normal lines.
 
         Returns:
             (anomalous_lines, normal_lines)

src/logreducer/core.py

@@ -1,6 +1,4 @@
-"""
-Core LogReducer implementation
-"""
+"""Core LogReducer implementation."""
 
 import json
 import os
@@ -10,7 +8,7 @@
 from datetime import datetime
 from enum import Enum
 from pathlib import Path
-from typing import Any
+from typing import Any, Literal, overload
 
 from .anomaly import AnomalyDetector
 from .config import BigDialConfig, OutputFormat, ProcessingLevel, ProcessingMode, get_preset_config
@@ -24,8 +22,7 @@
 
 
 class LogReducer:
-    """
-    Main log reduction class
+    """Main log reduction class.
 
     Example:
         reducer = LogReducer(level="enhanced")
@@ -41,8 +38,7 @@ def __init__(
         config: BigDialConfig | None = None,
         **kwargs: Any,
     ) -> None:
-        """
-        Initialize LogReducer
+        """Initialize LogReducer.
 
         Args:
             level: Processing level (standard/enhanced/maximum)
@@ -134,13 +130,41 @@ def _reset_components(self) -> None:
         else:
             self.temporal_processor = None
 
+    @overload
+    def reduce(
+        self,
+        source: Source,
+        output_file: str | None = None,
+        return_metadata: Literal[False] = False,
+        sink: Sink | None = None,
+    ) -> list[str]: ...
+
+    @overload
+    def reduce(
+        self,
+        source: Source,
+        output_file: str | None = None,
+        *,
+        return_metadata: Literal[True],
+        sink: Sink | None = None,
+    ) -> dict[str, Any]: ...
+
+    @overload
     def reduce(
         self,
         source: Source,
         output_file: str | None = None,
         return_metadata: bool = False,
         sink: Sink | None = None,
-    ) -> list[str] | dict:
+    ) -> list[str] | dict[str, Any]: ...
+
+    def reduce(
+        self,
+        source: Source,
+        output_file: str | None = None,
+        return_metadata: bool = False,
+        sink: Sink | None = None,
+    ) -> list[str] | dict[str, Any]:
         """Reduce a source of log lines to a representative sample.
 
         This is the core entry point: it operates on an abstraction, not on IO.
@@ -241,12 +265,37 @@ def reduce(
             }
         return result_lines
 
+    @overload
+    def process_file(
+        self,
+        input_file: str,
+        output_file: str | None = None,
+        return_metadata: Literal[False] = False,
+    ) -> list[str]: ...
+
+    @overload
+    def process_file(
+        self,
+        input_file: str,
+        output_file: str | None = None,
+        *,
+        return_metadata: Literal[True],
+    ) -> dict[str, Any]: ...
+
+    @overload
+    def process_file(
+        self,
+        input_file: str,
+        output_file: str | None = None,
+        return_metadata: bool = False,
+    ) -> list[str] | dict[str, Any]: ...
+
     def process_file(
         self,
         input_file: str,
         output_file: str | None = None,
         return_metadata: bool = False,
-    ) -> list[str] | dict:
+    ) -> list[str] | dict[str, Any]:
         """Reduce a log file - convenience wrapper over reduce() + FileSource.
 
         Args:
@@ -302,7 +351,7 @@ def _cap_for_anomaly(self, unique_lines: list[str]) -> list[str]:
         return unique_lines
 
     def _process_anomaly_mode(self, source: Source) -> list[str]:
-        """Process using anomaly detection"""
+        """Process using anomaly detection."""
         if not self.anomaly_detector:
             if self.config.enable_logging:
                 self.logger.warning("Anomaly detector not available, falling back to pattern mode")
@@ -329,7 +378,7 @@ def _process_anomaly_mode(self, source: Source) -> list[str]:
         return result
 
     def _process_temporal_mode(self, source: Source) -> list[str]:
-        """Process using temporal analysis"""
+        """Process using temporal analysis."""
         if not self.temporal_processor:
             if self.config.enable_logging:
                 self.logger.warning("Temporal processor not available, falling back to pattern mode")
@@ -356,7 +405,7 @@ def _process_temporal_mode(self, source: Source) -> list[str]:
         return result
 
     def _process_hybrid_mode(self, source: Source) -> list[str]:
-        """Process using combined approach"""
+        """Process using combined approach."""
         if self.config.enable_logging:
             self.logger.info("Hybrid mode: combining pattern and anomaly detection")
 
@@ -400,7 +449,7 @@ def _config_as_dict(self) -> dict[str, Any]:
         }
 
     def _save_output(self, lines: list[str], output_file: str) -> None:
-        """Save output to file in specified format"""
+        """Save output to file in specified format."""
         output_path = Path(output_file)
         output_path.parent.mkdir(parents=True, exist_ok=True)
 
@@ -453,7 +502,7 @@ def _save_output(self, lines: list[str], output_file: str) -> None:
                 self.logger.info(f"Metadata saved to {meta_file}")
 
     def _print_summary(self) -> None:
-        """Log processing summary"""
+        """Log processing summary."""
         self.logger.info("\n" + "=" * 60)
         self.logger.info("LOG REDUCTION SUMMARY")
         self.logger.info("=" * 60)
@@ -474,7 +523,7 @@ def _print_summary(self) -> None:
         self.logger.info("=" * 60)
 
     def estimate_processing(self, file_path: str) -> dict:
-        """Estimate processing requirements before running"""
+        """Estimate processing requirements before running."""
         file_size = os.path.getsize(file_path)
         file_size_gb = file_size / (1024**3)
 

src/logreducer/logging_config.py

@@ -26,7 +26,8 @@
 from loguru import logger
 
 if TYPE_CHECKING:
-    from loguru import Logger
+    # Read by get_logger's quoted return annotation, which vulture does not parse.
+    from loguru import Logger  # noqa: V104
 
 # Solarized palette (https://ethanschoonover.com/solarized/) for the console
 # sink - readable on both light and dark terminal backgrounds.