lint-diff.py

#!/usr/bin/env python3
"""Filter warnings output, to only show output for changed lines."""

# Filter warnings output, to only show output for changed lines.
#
# This is useful when you want to enforce some style guideline, but
# making the changes globally in your project would be too burdensome.
# You can make the requirement only for new and changed lines, so your
# codebase will conform to the new standard gradually, as you edit it.
#
# Note: If the warnings output indicates a catastrophic failure of the
# program that produces warnings, then this program will issue no
# warnings because there will be no warnings on specific lines.
# Example:
#   Exception in thread "main" java.lang.IllegalAccessError

# Usage:  lint-diff.py [options] diff.txt [warnings.txt]
#         If warnings.txt is omitted, use standard input.
# Output: all lines in warnings.txt that are on a changed line.
#         Output status is 1 if it produced any output, 0 if not, 2 if error.
# Options: --guess-strip means guess values for --strip-diff and --strip-warnings.
#          --strip-diff=N means to ignore N leading "/" in diff.txt.
#          --strip-warnings=N means to ignore N leading "/" in warnings.txt.
#              Affects matching, but not output, of lines.
#          --context=N is how many lines adjacent to the changed ones
#              are also considered changed; the default is 2.
#          --debug means to print diagnostic output.

# Here is how you could use this in continuous integration (Azure Pipelines,
# CircleCI, GitHub Actions, and Travis CI are currently supported) to require
# that pull requests satisfy the command `command-that-issues-warnings`:
#
#  PLUME_SCRIPTS="/tmp/${USER:-$(id -un)}/plume-scripts"
#  if [ -d "$PLUME_SCRIPTS" ] ; then
#   git -C "$PLUME_SCRIPTS" pull -q > /dev/null 2>&1
#  else
#   git clone --depth=1 -q https://github.com/plume-lib/plume-scripts.git "$PLUME_SCRIPTS"
#  fi
#  (command-that-issues-warnings > /tmp/warnings.txt 2>&1) || true
#  "$PLUME_SCRIPTS"/ci-lint-diff /tmp/warnings.txt

# Implementation notes:
# 1. It may be possible to achieve a similar result using diff (but not `git diff`):
# https://unix.stackexchange.com/questions/34874/diff-output-line-numbers .
# 2. The documentation for diffFilter (https://github.com/exussum12/coverageChecker)
# suggests it has this same functionality, but my tests indicate it does not.

import argparse
import collections.abc
import os
import re
import sys
from pathlib import Path
from typing import Any

PROGRAM = Path(__file__).name

DEBUG = False

PLUSPLUSPLUS_RE = re.compile(r"\+\+\+ (\S*).*")

# This cannot be multiline because files are read one line at a time.
# The leading "[ \t]*" is necessary because after Gradle outputs all warnings,
# it prints "> Compilation failed; see the compiler output below." and then
# prints one warning, indented by two spaces.
FILENAME_LINENO_RE = re.compile(r"[ \t]*([^:]*):([0-9]+):.*")

# An indented line that is itself a warning rather than the continuation of
# one.  Unlike `FILENAME_LINENO_RE`, the file name may not contain a quotation,
# grouping, or punctuation character, so that a continuation line that quotes
# source code containing "name:digits:" -- say,
#   log.error("Config.java:42: bad");
# -- is still treated as a continuation line, rather than as a warning about a
# file named `log.error("Config.java`.  Every line that this matches is also
# matched by `FILENAME_LINENO_RE`, which parses it.
INDENTED_WARNING_RE = re.compile(r"[ \t]*[^:\s\"'`()\[\]{},;=]*:[0-9]+:")

INITIAL_WHITESPACE_RE = re.compile(r"[ \t]")


def main() -> None:
    """Filter warnings output, to only show output for changed lines."""
    args = parse_args()

    if args.warning_filename is not None:
        warning_path = Path(args.warning_filename)
        if warning_path.is_file() and warning_path.stat().st_size == 0:
            sys.exit(0)

    # A dictionary from file names to a set of ints (line numbers for changed lines).
    changed = changed_lines(args)

    if DEBUG:
        for filename in sorted(changed):
            print(filename, sorted(changed[filename]))

    # True if a warning has been issued about relative directories.
    relative_diff_warned = warn_relative_diff(args)

    if args.warning_filename is None:
        args.warning_filename = "stdin"
        warnings = sys.stdin
    else:
        # pylint: disable=consider-using-with
        warnings = warning_path.open(encoding=encoding(warning_path))

    # 1 if this produced any output, 0 if not.
    status = 0
    # true if we just printed a warning and are looking for continuation lines to print.
    print_multiline_warning = False

    for warning_line in warnings:
        # A line that itself looks like a warning is not a continuation line,
        # even if it is indented (as Gradle indents the last warning it prints).
        if (
            print_multiline_warning
            and INITIAL_WHITESPACE_RE.match(warning_line)
            and not INDENTED_WARNING_RE.match(warning_line)
        ):
            print(warning_line, end="")
            continue
        print_multiline_warning = False

        should_output = False

        # Special case for Java exception in the output.
        if warning_line.startswith("Exception in thread"):
            should_output = True

        match = FILENAME_LINENO_RE.match(warning_line)
        if match:
            try:
                filename = strip_dirs(match.group(1), args.strip_warnings)
            except TypeError:
                filename = "warnings filename above common directory"
                ## It's not an error; it just means this file doesn't appear in warnings output.
                # eprint('Bad --strip-warnings={0} ; line has fewer "/": {1}'.format(
                #   strip_warnings, match.group(1)))
                # sys.exit(2)
            if (
                filename.startswith("/")
                and args.relative_diff is not None
                and args.strip_warnings == 0
            ):
                if not relative_diff_warned:
                    eprint(
                        # No spaces around string literals because this is `eprint`.
                        "warning:",
                        args.diff_filename,
                        "uses relative paths but",
                        args.warning_filename,
                        "uses absolute paths",
                    )
                    relative_diff_warned = True
            lineno = int(match.group(2))
            if filename in changed and lineno in changed[filename]:
                should_output = True

        if should_output:
            print(warning_line, end="")
            status = 1
            print_multiline_warning = True

    if warnings is not sys.stdin:
        warnings.close()

    sys.exit(status)


def eprint(*args: object, **kwargs: Any) -> None:
    """Print to stderr."""
    print(*args, file=sys.stderr, **kwargs)


def strip_dirs(filename: str, num_dirs: int) -> str:
    """Strip off `num_dirs` leading "/" characters.

    Returns:
        A subdirectory, with `num_dirs` top-level enclosing directories removed.
    """
    if num_dirs == 0:
        return filename
    return os.path.join(*(filename.split(os.path.sep)[num_dirs:]))  # ruff:ignore[os-path-join]


## Tests:
"""
import os
assert strip_dirs("/a/b/c/d", 0) == '/a/b/c/d'
assert strip_dirs("/a/b/c/d", 1) == 'a/b/c/d'
assert strip_dirs("/a/b/c/d", 2) == 'b/c/d'
assert strip_dirs("/a/b/c/d", 3) == 'c/d'
assert strip_dirs("/a/b/c/d", 4) == 'd'
assert strip_dirs("/a/b/c/", 0) == '/a/b/c/'
assert strip_dirs("/a/b/c/", 1) == 'a/b/c/'
assert strip_dirs("/a/b/c/", 2) == 'b/c/'
assert strip_dirs("/a/b/c/", 3) == 'c/'
assert strip_dirs("/a/b/c/", 4) == ''
"""


def min_strips(filename1: str, filename2: str) -> tuple[int, int, str, str]:
    """Find matching suffixes of the given filenames.

    Returns a 4-tuple of 2 integers and 2 strings.  The integers
    indicate the smallest strip values that make the two filenames equal,
    or a maximal pair if the files have different basenames.  The last two
    elements of the tuple are the argument strings.

    Returns:
        A 4-tuple of 2 integers and 2 strings.
    """
    components1 = filename1.split(os.path.sep)
    components2 = filename2.split(os.path.sep)
    if components1[-1] != components2[-1]:
        ## TODO: is this special case necessary?
        return (1000, 1000, filename1, filename2)
    while components1 and components2 and components1[-1] == components2[-1]:
        del components1[-1]
        del components2[-1]
    return (len(components1), len(components2), filename1, filename2)


## Tests:
"""
import os
assert min_strips("/a/b/c/d", "/a/b/c/d") == (0, 0, "/a/b/c/d", "/a/b/c/d")
assert min_strips("e1/e2/a/b/c/d", "/a/b/c/d") == (2, 1, "e1/e2/a/b/c/d", "/a/b/c/d")
assert min_strips("/e1/e2/a/b/c/d", "/a/b/c/d") == (3, 1, "/e1/e2/a/b/c/d", "/a/b/c/d")
assert min_strips("e1/e2/a/b/c/d", "a/b/c/d") == (2, 0, "e1/e2/a/b/c/d", "a/b/c/d")
assert min_strips("/e1/e2/a/b/c/d", "a/b/c/d") == (3, 0, "/e1/e2/a/b/c/d", "a/b/c/d")
assert min_strips("/a/b/c/d", "/e/f/g/h") == (1000, 1000, "/a/b/c/d", "/e/f/g/h")
"""


def pair_key(pair: tuple[int, int, str, str]) -> tuple[int, int, str, str]:
    """Given a tuple as produced by `min_strips`, returns a sort key for it.

    Strip-pairs are not always pointwise comparable: the diff and the warnings
    may disagree about the relative depth of two different files, as when a
    lint tool reports some paths relative to a module and others relative to
    the repository.  This key totally orders them, preferring the pair that
    strips the fewest directories overall.

    Returns:
        a sort key that totally orders the tuples produced by `min_strips`.
    """
    # The last two elements make the ordering deterministic when two pairs
    # strip the same number of directories.  They are not otherwise meaningful.
    return (pair[0] + pair[1], pair[0], pair[2], pair[3])


## Tests:
"""
import os
assert min([(3,4,"a","b"), (5,6,"c","d")], key=pair_key) == (3,4,"a","b")
assert min([(4,3,"a","b"), (6,5,"c","d")], key=pair_key) == (4,3,"a","b")
assert min([(30,40,"a","b"), (5,6,"c","d")], key=pair_key) == (5,6,"c","d")
assert min([(40,30,"a","b"), (6,5,"c","d")], key=pair_key) == (6,5,"c","d")
# Incomparable pairs: neither is pointwise lesser than the other.
assert min([(3,1,"a","b"), (2,2,"c","d")], key=pair_key) == (2,2,"c","d")
assert min([(2,2,"c","d"), (3,1,"a","b")], key=pair_key) == (2,2,"c","d")
"""


def diff_filenames(diff_filename: str) -> set[str]:
    """All the filenames in the given diff file.

    Returns:
        All the filenames in the given diff file.
    """
    result = set()
    with Path(diff_filename).open(encoding=encoding(diff_filename)) as diff:
        for diff_line in diff:
            match = PLUSPLUSPLUS_RE.match(diff_line)
            if match:
                filename: str = match.group(1)
                if filename != "/dev/null":
                    result.add(filename)
    return result


def warning_filenames(warning_filename: str) -> set[str]:
    """All the filenames in the given warning file.

    Returns:
        All the filenames in the given warning file.
    """
    result = set()
    with Path(warning_filename).open(encoding=encoding(warning_filename)) as warnings:
        for warning_line in warnings:
            match = FILENAME_LINENO_RE.match(warning_line)
            if match:
                result.add(match.group(1))
    return result


def guess_strip_filenames(
    diff_filenames: set[str], warning_filenames: set[str]
) -> tuple[int, int, str, str]:
    """Match subdirectory structure.

    Arguments are two lists of file names.

    Returns:
        A 4-tuple of 2 integers and 2 strings, as for `min_strips`.
    """
    candidates = [
        min_strips(diff_filename, warning_filename)
        for diff_filename in diff_filenames
        for warning_filename in warning_filenames
    ]
    if not candidates:
        return (1000, 1000, "no files seen yet", "no files seen yet")
    return min(candidates, key=pair_key)


def guess_strip_files(diff_file: str, warning_file: str) -> tuple[int, int, str, str]:
    """Match subdirectory structure.

    Arguments are files produced by diff and a lint tool, respectively.

    Returns:
        A 4-tuple of 2 integers and 2 strings, as for `min_strips`.
    """
    diff_files = diff_filenames(diff_file)
    warning_files = warning_filenames(warning_file)
    result = guess_strip_filenames(diff_files, warning_files)
    diff_prefix = commonpath(diff_files)
    try:
        warnings_prefix = commonpath(warning_files)
    except ValueError:
        with Path.open(Path(warning_file)) as file:
            for line in file:
                # rstrip is necessary because eprint adds a newline.
                eprint(line.rstrip())
        raise
    if result[0] > diff_prefix.count("/") or result[1] > warnings_prefix.count("/"):
        # This is not necessarily a problem.  It is possible that all the
        # diffs, or all the lint output, happens to be in one subdirectory.
        if DEBUG:
            eprint(
                "lint-diff.py: guess_strip_files all in one subdirectory: "
                f"result={result} diff_prefix={diff_prefix} warnings_prefix={warnings_prefix}"
            )
            eprint(f"diff_files={diff_files}")
            eprint(f"warning_files={warning_files}")
    return result


def commonpath(files: collections.abc.Iterable[str]) -> str:
    """Return the common prefix of the given paths.

    Argument is a set or list of paths.  Absolute and relative paths have no
    common prefix, so if the argument mixes them, the result is "".

    Returns:
         the common prefix of the given paths, or "" if they have none.
    """
    # Remove leading null characters.
    files_list = [file.lstrip("\0") for file in files]
    if not files_list:
        return ""
    # `os.path.commonpath` throws `ValueError` on a mix of absolute and
    # relative paths, which a lint tool may produce for different files.
    if len({Path(file).is_absolute() for file in files_list}) != 1:
        return ""
    try:
        return os.path.commonpath(files_list)
    except ValueError as err:
        raise ValueError(str(files)) from err


## Tests:
"""
assert commonpath([]) == ""
assert commonpath(["/a/b/c", "/a/b/d"]) == "/a/b"
assert commonpath(["a/b/c", "a/b/d"]) == "a/b"
# Absolute and relative paths have no common prefix.
assert commonpath(["/a/b/c", "a/b/d"]) == ""
"""


### Main routine


def parse_args() -> argparse.Namespace:
    """Parse and return the command-line arguments.

    Returns:
        The parsed command-line arguments.
    """
    global DEBUG

    parser = argparse.ArgumentParser(
        description="Filter warnings output, to only show output for changed lines"
    )
    parser.add_argument(
        "--guess-strip",
        dest="guess_strip",
        action="store_true",
        default=False,
        help="guess values for --strip-diff and --strip-warnings",
    )
    parser.add_argument(
        "--strip-diff",
        metavar="NUM_SLASHES",
        dest="strip_diff",
        action="store",
        type=int,
        default=0,
        help='ignore N leading "/" in filenames in diff.txt',
    )
    parser.add_argument(
        "--strip-warnings",
        metavar="NUM_SLASHES",
        dest="strip_warnings",
        action="store",
        type=int,
        default=0,
        help='ignore N leading "/" in filenames in warnings.txt',
    )
    # Deprecated; exists for backwards compatibility
    parser.add_argument(
        "--strip-lint",
        metavar="NUM_SLASHES",
        dest="strip_warnings",
        action="store",
        type=int,
        default=0,
        help='(DEPRECATED) ignore N leading "/" in filenames in warnings.txt',
    )
    parser.add_argument(
        "--context",
        metavar="NUM_LINES",
        dest="context_lines",
        action="store",
        type=int,
        default=2,
        help="how many lines around each changed one are also considered changed",
    )
    parser.add_argument(
        "--debug", dest="DEBUG", action="store_true", help="print diagnostic output"
    )
    parser.add_argument("diff_filename", metavar="diff.txt", default=Path.cwd())
    parser.add_argument("warning_filename", metavar="warnings.txt", nargs="?", default=None)

    args = parser.parse_args()
    DEBUG = args.DEBUG

    if args.guess_strip and args.strip_diff != 0:
        eprint(PROGRAM, ": don't supply both --guess-strip and --strip-diff")
        sys.exit(2)

    if args.guess_strip and args.strip_warnings != 0:
        eprint(PROGRAM, ": don't supply both --guess-strip and --strip-warnings")
        sys.exit(2)

    if args.guess_strip and args.warning_filename is None:
        eprint(PROGRAM, 'needs "warnings.txt" file argument when --guess-strip is provided')
        sys.exit(2)

    if args.guess_strip:
        guessed_strip = guess_strip_files(args.diff_filename, args.warning_filename)
        if guessed_strip[0] == 1000:
            if DEBUG:
                eprint(
                    "lint-diff.py: --guess-strip failed to guess values (maybe no files in common?)"
                )
        else:
            args.strip_diff = guessed_strip[0]
            args.strip_warnings = guessed_strip[1]
            if DEBUG:
                eprint(
                    "lint-diff.py inferred "
                    f"--strip-diff={args.strip_diff} --strip-warnings={args.strip_warnings}"
                )

    # A filename if the diff filenames start with "a/" and "b/", otherwise None.
    # Is set by changed_lines().
    args.relative_diff = None

    return args


def changed_lines(args: argparse.Namespace) -> dict[str, set[int]]:
    """Return a dictionary from file names to a set of ints (line numbers for changed lines).

    Returns:
        a dictionary from file names to a set of ints (line numbers for changed lines).
    """
    changed: dict[str, set[int]] = {}

    with Path(args.diff_filename).open(encoding=encoding(args.diff_filename)) as diff:
        atat_re = re.compile(r"@@ -([0-9]+)(,[0-9]+)? \+([0-9]+)(,[0-9]+)? @@.*")
        # content_re = re.compile("[ +-].*")

        filename = ""
        lineno = -1000000
        # Number of old-side and new-side lines remaining in the current hunk.
        # While either is positive, the current line is a hunk body line (which
        # starts with " ", "+", or "-") and must not be mistaken for a header
        # such as "+++"/"---" -- an added line whose content starts with "++ "
        # would otherwise look like a "+++ " file header.
        remaining_old = 0
        remaining_new = 0
        for diff_line in diff:
            if remaining_old <= 0 and remaining_new <= 0:
                # Not inside a hunk body: look for headers.
                if diff_line.startswith("---"):
                    continue
                match = PLUSPLUSPLUS_RE.match(diff_line)
                if match:
                    if match.group(1).startswith("b/"):  # heuristic
                        args.relative_diff = diff_line
                    try:
                        filename = strip_dirs(match.group(1), args.strip_diff)
                    except TypeError:
                        filename = "diff filename above common directory"
                        ## Not an error; it just means this file doesn't appear in warnings output.
                        # eprint('Bad --strip-diff={0} ; line has fewer "/": {1}'.format(
                        #   strip_diff, match.group(1)))
                        # sys.exit(2)
                    if filename not in changed:
                        changed[filename] = set()
                    continue
                match = atat_re.match(diff_line)
                if match:
                    lineno = int(match.group(3))
                    remaining_old = int(match.group(2)[1:]) if match.group(2) else 1
                    remaining_new = int(match.group(4)[1:]) if match.group(4) else 1
                    continue
                continue
            # Inside a hunk body.
            if diff_line.startswith("+"):
                # Not just the changed line: changed[filename].add(lineno)
                for changed_lineno in range(
                    lineno - args.context_lines, lineno + args.context_lines + 1
                ):
                    changed[filename].add(changed_lineno)
                lineno += 1
                remaining_new -= 1
            elif diff_line.startswith("-"):
                for changed_lineno in range(
                    lineno - args.context_lines, lineno + args.context_lines
                ):
                    changed[filename].add(changed_lineno)
                remaining_old -= 1
            elif diff_line.startswith(" "):
                lineno += 1
                remaining_old -= 1
                remaining_new -= 1
            # Any other line (e.g., "\ No newline at end of file") is left as-is
            # and does not consume a hunk line.

    return changed


def warn_relative_diff(args: argparse.Namespace) -> bool:
    """Possibly warn about relative directories.

    Returns:
        a boolean.
    """
    result = False
    if args.relative_diff is not None and args.strip_diff == 0:
        # This is usually not an error, so don't warn.
        if DEBUG:
            eprint(
                "warning:",
                args.diff_filename,
                "may use relative paths (e.g.,",
                args.relative_diff.strip(),
                ") but --strip-diff=0",
                ("(guessed)" if args.guess_strip else ""),
            )
            eprint("warning: (Maybe there were no files in common.)")
        result = True
        if DEBUG:
            eprint(f"lint-diff.py: diff file {args.diff_filename}:")
            eprint(Path(args.diff_filename).read_text(encoding=encoding(args.diff_filename)))
            eprint(f"lint-diff.py: lint file {args.warning_filename}:")
            eprint(
                Path(args.warning_filename).read_text(encoding=encoding(args.warning_filename)),
            )
            eprint("lint-diff.py: end of input files.")

    return result


# As an alternative, could use the `chardet` package, but I don't want external dependencies.
def encoding(file_path):
    """Detect the encoding of a file.

    Returns:
        "utf-8" if the file is encoded using UTF-8, "iso-8859-1" otherwise.
    """
    try:
        Path(file_path).read_bytes().decode("utf-8")
        return "utf-8"
    except UnicodeDecodeError:
        return "iso-8859-1"


if __name__ == "__main__":
    main()