diff --git a/.github/scripts/check-whitespace.py b/.github/scripts/check-whitespace.py new file mode 100644 index 0000000000..7eabd4307c --- /dev/null +++ b/.github/scripts/check-whitespace.py @@ -0,0 +1,377 @@ +# /// script +# dependencies = [ +# "pytest", +# ] +# /// + +"""Check (and by default fix) whitespace issues in tracked source files: +- stray CR, e.g. 'hello\\rworld' -> 'helloworld' +- line ending mismatch, e.g. 'hello\\r\\n' -> 'hello\\n' (or vice versa) +- missing newline at end of file +- extra newline(s) at end of file +- trailing whitespace at end of line +""" + +import argparse +import io +import os +import re +import subprocess +import sys +from collections.abc import Callable +from pathlib import Path +from typing import BinaryIO, Literal, cast + +import pytest + + +class C: + RED = "\033[31m" + GREEN = "\033[32m" + YELLOW = "\033[33m" + RESET = "\033[0m" + + +CR = b"\r" +CRLF = b"\r\n" +LF = b"\n" + +LineSeparator = Literal[b"\r\n", b"\n"] +SYSTEM_LINE_SEPARATOR = cast(LineSeparator, os.linesep.encode()) + + +class Checker: + def __init__(self, newline: LineSeparator = SYSTEM_LINE_SEPARATOR) -> None: + self.newline = newline + self.issues = 0 + + def report(self, label: str, issue: str) -> None: + self.issues += 1 + print(f"{label}: {C.RED}{issue}{C.RESET}") + + def check_stray_cr(self, filepath: Path, check: bool) -> None: + with filepath.open("r+b") as f: + self._check_stray_cr(f, str(filepath), check) + + def _check_stray_cr(self, f: BinaryIO, label: str, check: bool) -> None: + # a CR is "stray" if it isn't immediately followed by a LF, i.e. not part of a CRLF pair + # CRLF/CR mismatch will be reported separately. + stray_cr = re.compile(rb"\r(?!\n)") + + content = f.read() + matches = list(stray_cr.finditer(content)) + if not matches: + return + + line_numbers = dict.fromkeys(content.count(b"\n", 0, m.start()) + 1 for m in matches) + for line_number in line_numbers: + self.report(f"{label}:{line_number}", "stray carriage return") + if check: + return + + f.seek(0) + f.write(stray_cr.sub(b"", content)) + f.truncate() + + def check_line_endings_mismatch(self, filepath: Path, check: bool) -> None: + with filepath.open("r+b") as f: + self._check_line_endings_mismatch(f, str(filepath), check) + + def _check_line_endings_mismatch(self, f: BinaryIO, label: str, check: bool) -> None: + NEWLINE = self.newline + + def get_line_ending(line: bytes) -> LineSeparator | None: + if line.endswith(CRLF): + return CRLF + if line.endswith(LF): + return LF + # last line with no trailing newline at all; check_eof_newline handles that + return None + + changed = False + fixed_lines = [] + for line_number, line in enumerate(f, start=1): + found = get_line_ending(line) + if found in (NEWLINE, None): + fixed_lines.append(line) + continue + + self.report(f"{label}:{line_number}", f"line ending mismatch (expected {NEWLINE!r}, found {found!r})") + changed = True + content = line[: -len(found)] + fixed_lines.append(content + NEWLINE) + + if changed and not check: + f.seek(0) + f.write(b"".join(fixed_lines)) + f.truncate() + + def check_eof_newline(self, filepath: Path, check: bool) -> None: + with filepath.open("r+b") as f: + self._check_eof_newline(f, str(filepath), check) + + def _check_eof_newline(self, f: BinaryIO, label: str, check: bool) -> None: + NEWLINE = self.newline + NEWLINE_SIZE = len(NEWLINE) + + size = f.seek(0, os.SEEK_END) + if size == 0: + return + + trailing_newlines = 0 + while True: + pos = f.seek((-trailing_newlines - 1) * NEWLINE_SIZE, os.SEEK_END) + if f.read(NEWLINE_SIZE) != NEWLINE: + break + trailing_newlines += 1 + if pos == 0: + break + + if trailing_newlines == 0: + self.report(label, "missing newline at end of file") + if check: + return + f.seek(0, os.SEEK_END) + f.write(NEWLINE) + elif trailing_newlines > 1: + self.report(label, f"{trailing_newlines} trailing newlines at end of file") + if check: + return + f.truncate(size - (trailing_newlines - 1) * NEWLINE_SIZE) + + def check_trailing_whitespaces(self, filepath: Path, check: bool) -> None: + with filepath.open("r+b") as f: + self._check_trailing_whitespaces(f, str(filepath), check) + + def _check_trailing_whitespaces(self, f: BinaryIO, label: str, check: bool) -> None: + NEWLINE = self.newline + NEWLINE_SIZE = len(NEWLINE) + + changed = False + fixed_lines = [] + for line_number, line in enumerate(f, start=1): + has_newline = line.endswith(NEWLINE) + content = line[:-NEWLINE_SIZE] if has_newline else line + stripped = content.rstrip() + if stripped != content: + self.report(f"{label}:{line_number}", "trailing whitespace") + changed = True + fixed_lines.append(stripped + (NEWLINE if has_newline else b"")) + + if changed and not check: + f.seek(0) + f.write(b"".join(fixed_lines)) + f.truncate() + + +CheckMethod = Callable[[Checker, BinaryIO, str, bool], None] + + +class TestChecker: + def _assert_check( + self, + method: CheckMethod, + content: bytes, + expected_issues: int, + fixed: bytes, + check: bool, + line_ending: LineSeparator, + *, + transform: bool = True, + ) -> None: + checker = Checker(line_ending) + if line_ending == CRLF and transform: + content = content.replace(LF, CRLF) + fixed = fixed.replace(LF, CRLF) + buffer = io.BytesIO(content) + method(checker, buffer, "test", check) + assert buffer.getvalue() == (content if check else fixed) + assert checker.issues == expected_issues + + @pytest.mark.parametrize( + ("content", "expected_issues", "fixed"), + ( + # OK + (b"", 0, b""), + (b"hello\n", 0, b"hello\n"), + (b"line1\r\nline2\n", 0, b"line1\r\nline2\n"), + # ERR + (b"hello\rworld\n", 1, b"helloworld\n"), + (b"a\rb\rc\n", 1, b"abc\n"), + (b"hello\r", 1, b"hello"), + ), + ) + @pytest.mark.parametrize("check", [False, True]) + def test_check_stray_cr(self, content: bytes, expected_issues: int, fixed: bytes, check: bool) -> None: + # Don't parametrize by line endings, since in this case it doesn't matter. + self._assert_check(Checker._check_stray_cr, content, expected_issues, fixed, check, LF) + + @pytest.mark.parametrize( + ("content", "expected_issues", "fixed", "line_ending"), + ( + # OK + (b"", 0, b"", LF), + (b"hello\n", 0, b"hello\n", LF), + (b"hello\r\n", 0, b"hello\r\n", CRLF), + # ERR + (b"hello\r\n", 1, b"hello\n", LF), + (b"a\nb\r\nc\n", 1, b"a\nb\nc\n", LF), + (b"a\r\nb\r\n", 2, b"a\nb\n", LF), + (b"hello\n", 1, b"hello\r\n", CRLF), + (b"a\r\nb\nc\r\n", 1, b"a\r\nb\r\nc\r\n", CRLF), + ), + ) + @pytest.mark.parametrize("check", [False, True]) + def test_check_line_endings_mismatch( + self, content: bytes, expected_issues: int, fixed: bytes, line_ending: LineSeparator, check: bool + ) -> None: + self._assert_check( + Checker._check_line_endings_mismatch, content, expected_issues, fixed, check, line_ending, transform=False + ) + + @pytest.mark.parametrize( + ("content", "expected_issues", "fixed"), + ( + # OK + (b"", 0, b""), + (b"hello\n", 0, b"hello\n"), + # ERR + (b"hello", 1, b"hello\n"), + (b"hello\n\n\n", 1, b"hello\n"), + (b"\n\n\n", 1, b"\n"), + ), + ) + @pytest.mark.parametrize("check", [False, True]) + @pytest.mark.parametrize("line_ending", [LF, CRLF]) + def test_check_eof_newline( + self, content: bytes, expected_issues: int, fixed: bytes, check: bool, line_ending: LineSeparator + ) -> None: + self._assert_check(Checker._check_eof_newline, content, expected_issues, fixed, check, line_ending) + + @pytest.mark.parametrize( + ("content", "expected_issues", "fixed"), + ( + # OK + (b"", 0, b""), + (b"hello\n", 0, b"hello\n"), + (b"hello", 0, b"hello"), + # ERR + (b" ", 1, b""), + (b"hello ", 1, b"hello"), + ), + ) + @pytest.mark.parametrize("check", [False, True]) + @pytest.mark.parametrize("line_ending", [LF, CRLF]) + def test_check_trailing_whitespaces( + self, content: bytes, expected_issues: int, fixed: bytes, check: bool, line_ending: LineSeparator + ) -> None: + self._assert_check(Checker._check_trailing_whitespaces, content, expected_issues, fixed, check, line_ending) + + @staticmethod + def run_tests(extra_args: list[str] | None = None) -> None: + pytest.main([__file__, *(extra_args or [])]) + + +def existing_path(value: str) -> Path: + path = Path(value) + if not path.exists(): + raise argparse.ArgumentTypeError(f"path not found: {value}") + return path + + +# Python files are covered by `black`. +PATTERNS = ( + "*.cpp", + "*.h", + "*.i", +) + +REPO_ROOT = Path(subprocess.check_output(["git", "rev-parse", "--show-toplevel"], text=True).strip()) + +# Generated files; formatted by the express codegen, not by this script. +IGNORED_DIRS = (REPO_ROOT / "src/ifcparse/schemas",) + + +def get_tracked_files(root: Path | None = None) -> list[Path]: + output = subprocess.check_output( + ["git", "ls-files", "--others", "--cached", "--exclude-standard", *PATTERNS], + cwd=root, + text=True, + ) + base = root if root is not None else Path() + filepaths = [] + for line in output.splitlines(): + filepath = base / line + if not any(filepath.resolve().is_relative_to(d) for d in IGNORED_DIRS): + filepaths.append(filepath) + return filepaths + + +def main() -> int: + # anything after "--" is forwarded to pytest, e.g. `--test -- --capture=no` + argv = sys.argv[1:] + if "--" in argv: + split = argv.index("--") + argv, extra_args = argv[:split], argv[split + 1 :] + else: + extra_args = [] + + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=__doc__, + ) + parser.add_argument("paths", type=existing_path, nargs="*", help="files or directories to check") + parser.add_argument( + "--check", + action="store_true", + help="only check for whitespace issues without applying fixes", + ) + parser.add_argument( + "--test", + action="store_true", + help="run self-tests", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="print each checked path", + ) + args = parser.parse_args(argv) + + if args.test: + TestChecker.run_tests(extra_args) + return 0 + + if args.paths: + filepaths: list[Path] = [] + for path in args.paths: + filepaths.extend(get_tracked_files(path) if path.is_dir() else [path]) + else: + filepaths = get_tracked_files() + + # dict.fromkeys() dedupes while preserving order, unlike set(). + filepaths = list(dict.fromkeys(filepaths)) + + checker = Checker() + for filepath in filepaths: + if args.verbose: + print(f"checking {filepath}") + checker.check_stray_cr(filepath, args.check) + checker.check_line_endings_mismatch(filepath, args.check) + checker.check_eof_newline(filepath, args.check) + checker.check_trailing_whitespaces(filepath, args.check) + print(f"{len(filepaths)} file(s) checked.") + if not checker.issues: + color = C.GREEN + elif args.check: + color = C.RED + else: + color = C.YELLOW + outcome = "found" if args.check else "found and fixed" + print(f"{color}{checker.issues} issue(s) {outcome}.{C.RESET}") + + return 1 if args.check and checker.issues else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pyproject.toml b/pyproject.toml index 572eeca522..6bf5967c8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ include = ''' src/.*.pyi?$ |nix/.*.pyi?$ |pyodide/.*.pyi?$ + |.github/.*.pyi?$ ''' extend-exclude = ''' src/ifcopenshell-python/ifcopenshell/express/rules/* @@ -181,6 +182,8 @@ dev-setup.help = "Install repo packages in editable mode" ruff = "ruff check" +check-whitespace = "uv run .github/scripts/check-whitespace.py" + black = "black ." ty.sequence = ["ty-bonsai", "ty-ios"]