diff --git a/scripts/find_localised_strings.py b/scripts/find_localised_strings.py index 74cfe1b4..b9d8a8b5 100644 --- a/scripts/find_localised_strings.py +++ b/scripts/find_localised_strings.py @@ -1,4 +1,6 @@ """Search all given paths recursively for localised string calls.""" +from __future__ import annotations + import argparse import ast import dataclasses @@ -6,9 +8,6 @@ import json import pathlib import re import sys -from typing import Optional - -# spell-checker: words dedupe deduping deduped def get_func_name(thing: ast.AST) -> str: @@ -16,11 +15,9 @@ def get_func_name(thing: ast.AST) -> str: if isinstance(thing, ast.Name): return thing.id - elif isinstance(thing, ast.Attribute): + if isinstance(thing, ast.Attribute): return get_func_name(thing.value) - - else: - return '' + return '' def get_arg(call: ast.Call) -> str: @@ -31,10 +28,9 @@ def get_arg(call: ast.Call) -> str: arg = call.args[0] if isinstance(arg, ast.Constant): return arg.value - elif isinstance(arg, ast.Name): + if isinstance(arg, ast.Name): return f'VARIABLE! CHECK CODE! {arg.id}' - else: - return f'Unknown! {type(arg)=} {ast.dump(arg)} ||| {ast.unparse(arg)}' + return f'Unknown! {type(arg)=} {ast.dump(arg)} ||| {ast.unparse(arg)}' # type: ignore def find_calls_in_stmt(statement: ast.AST) -> list[ast.Call]: @@ -62,7 +58,7 @@ COMMENT_SAME_LINE_RE = re.compile(r'^.*?(#.*)$') COMMENT_OWN_LINE_RE = re.compile(r'^\s*?(#.*)$') -def extract_comments(call: ast.Call, lines: list[str], file: pathlib.Path) -> Optional[str]: # noqa: CCR001 +def extract_comments(call: ast.Call, lines: list[str], file: pathlib.Path) -> str | None: """ Extract comments from source code based on the given call. @@ -74,51 +70,32 @@ def extract_comments(call: ast.Call, lines: list[str], file: pathlib.Path) -> Op :param file: The path to the file this call node came from :return: The first comment that matches the rules, or None """ - out: Optional[str] = None - above = call.lineno - 2 - current = call.lineno - 1 + above_line_number = call.lineno - 2 + current_line_number = call.lineno - 1 - above_line = lines[above].strip() if len(lines) >= above else None - above_comment: Optional[str] = None - current_line = lines[current].strip() - current_comment: Optional[str] = None + def extract_lang_comment(line: str) -> str | None: + """ + Extract a language comment from a given line. - bad_comment: Optional[str] = None - if above_line is not None: - match = COMMENT_OWN_LINE_RE.match(above_line) - if match: - above_comment = match.group(1).strip() - if not above_comment.startswith('# LANG:'): - bad_comment = f'Unknown comment for {file}:{call.lineno} {above_line}' - above_comment = None + :param line: The line to extract the language comment from. + :return: The extracted language comment, or None if no valid comment is found. + """ + match = COMMENT_OWN_LINE_RE.match(line) + if match and match.group(1).startswith('# LANG:'): + return match.group(1).replace('# LANG:', '').strip() + return None - else: - above_comment = above_comment.replace('# LANG:', '').strip() + above_comment = extract_lang_comment(lines[above_line_number]) if len(lines) >= above_line_number else None + current_comment = extract_lang_comment(lines[current_line_number]) - if current_line is not None: - match = COMMENT_SAME_LINE_RE.match(current_line) - if match: - current_comment = match.group(1).strip() - if not current_comment.startswith('# LANG:'): - bad_comment = f'Unknown comment for {file}:{call.lineno} {current_line}' - current_comment = None + if current_comment is None: + current_comment = above_comment - else: - current_comment = current_comment.replace('# LANG:', '').strip() + if current_comment is None: + print(f'No comment for {file}:{call.lineno} {lines[current_line_number]}', file=sys.stderr) + return None - if current_comment is not None: - out = current_comment - - elif above_comment is not None: - out = above_comment - - elif bad_comment is not None: - print(bad_comment, file=sys.stderr) - - if out is None: - print(f'No comment for {file}:{call.lineno} {current_line}', file=sys.stderr) - - return out + return current_comment def scan_file(path: pathlib.Path) -> list[ast.Call]: @@ -126,17 +103,19 @@ def scan_file(path: pathlib.Path) -> list[ast.Call]: data = path.read_text(encoding='utf-8') lines = data.splitlines() parsed = ast.parse(data) - out: list[ast.Call] = [] + calls: list[ast.Call] = [] for statement in parsed.body: - out.extend(find_calls_in_stmt(statement)) + calls.extend(find_calls_in_stmt(statement)) - # see if we can extract any comments - for call in out: - setattr(call, 'comment', extract_comments(call, lines, path)) + # Extract and assign comments to each call + for call in calls: + call.comment = extract_comments(call, lines, path) # type: ignore - out.sort(key=lambda c: c.lineno) - return out + # Sort the calls by line number + calls.sort(key=lambda c: c.lineno) + + return calls def scan_directory(path: pathlib.Path, skip: list[pathlib.Path] | None = None) -> dict[pathlib.Path, list[ast.Call]]: @@ -150,40 +129,34 @@ def scan_directory(path: pathlib.Path, skip: list[pathlib.Path] | None = None) - for thing in path.iterdir(): if skip is not None and any(s.name == thing.name for s in skip): continue - if thing.is_file(): if not thing.name.endswith('.py'): continue - out[thing] = scan_file(thing) - elif thing.is_dir(): - out |= scan_directory(thing) - + out |= scan_directory(thing) # type: ignore else: raise ValueError(type(thing), thing) - return out -def parse_template(path) -> set[str]: +def parse_template(path: pathlib.Path) -> set[str]: """ Parse a lang.template file. - The regexp this uses was extracted from l10n.py. + The regular expression used here was extracted from l10n.py. :param path: The path to the lang file """ - lang_re = re.compile(r'\s*"((?:[^"]|(?:\"))+)"\s*=\s*"((?:[^"]|(?:\"))+)"\s*;\s*$') - out = set() + lang_re = re.compile(r'\s*"([^"]+)"\s*=\s*"([^"]+)"\s*;\s*$') + result = set() + for line in pathlib.Path(path).read_text(encoding='utf-8').splitlines(): match = lang_re.match(line) - if not match: - continue - if match.group(1) != '!Language': - out.add(match.group(1)) + if match and match.group(1) != '!Language': + result.add(match.group(1)) - return out + return result @dataclasses.dataclass @@ -193,8 +166,8 @@ class FileLocation: path: pathlib.Path line_start: int line_start_col: int - line_end: Optional[int] - line_end_col: Optional[int] + line_end: int | None + line_end_col: int | None @staticmethod def from_call(path: pathlib.Path, c: ast.Call) -> 'FileLocation': @@ -213,50 +186,46 @@ class LangEntry: locations: list[FileLocation] string: str - comments: list[Optional[str]] + comments: list[str | None] def files(self) -> str: """Return a string representation of all the files this LangEntry is in, and its location therein.""" - out = '' - for loc in self.locations: - start = loc.line_start - end = loc.line_end - end_str = f':{end}' if end is not None and end != start else '' - out += f'{loc.path.name}:{start}{end_str}; ' + file_locations = [ + f'{loc.path.name}:{loc.line_start}' + + (f':{loc.line_end}' if loc.line_end is not None and loc.line_end != loc.line_start else '') + for loc in self.locations + ] - return out + return '; '.join(file_locations) def dedupe_lang_entries(entries: list[LangEntry]) -> list[LangEntry]: """ Deduplicate a list of lang entries. - This will coalesce LangEntries that have that same string but differing files and comments into a single - LangEntry that cotains all comments and FileLocations + This will coalesce LangEntries that have the same string but differing files and comments into a single + LangEntry that contains all comments and FileLocations. :param entries: The list to deduplicate :return: The deduplicated list """ - deduped: list[LangEntry] = [] + deduped: dict[str, LangEntry] = {} + for e in entries: - cont = False - for d in deduped: - if d.string == e.string: - cont = True - d.locations.append(e.locations[0]) - d.comments.extend(e.comments) + existing = deduped.get(e.string) + if existing: + existing.locations.extend(e.locations) + existing.comments.extend(e.comments) + else: + deduped[e.string] = LangEntry(locations=e.locations[:], string=e.string, comments=e.comments[:]) - if cont: - continue - - deduped.append(e) - - return deduped + return list(deduped.values()) def generate_lang_template(data: dict[pathlib.Path, list[ast.Call]]) -> str: """Generate a full en.template from the given data.""" entries: list[LangEntry] = [] + for path, calls in data.items(): for c in calls: entries.append(LangEntry([FileLocation.from_call(path, c)], get_arg(c), [getattr(c, 'comment')])) @@ -267,30 +236,72 @@ def generate_lang_template(data: dict[pathlib.Path, list[ast.Call]]) -> str: ''' print(f'Done Deduping entries {len(entries)=} {len(deduped)=}', file=sys.stderr) + for entry in deduped: assert len(entry.comments) == len(entry.locations) - comment = '' - files = 'In files: ' + entry.files() + comment_parts = [] string = f'"{entry.string}"' - for i in range(len(entry.comments)): - if entry.comments[i] is None: + for i, comment_text in enumerate(entry.comments): + if comment_text is None: continue loc = entry.locations[i] - to_append = f'{loc.path.name}: {entry.comments[i]}; ' - if to_append not in comment: - comment += to_append + comment_parts.append(f'{loc.path.name}: {comment_text};') - header = f'{comment.strip()} {files}'.strip() - out += f'/* {header} */\n' - out += f'{string} = {string};\n' - out += '\n' + if comment_parts: + header = ' '.join(comment_parts) + out += f'/* {header} */\n' + + out += f'{string} = {string};\n\n' return out -if __name__ == '__main__': +def compare_lang_with_template(template: set[str], res: dict[pathlib.Path, list[ast.Call]]) -> None: + """ + Compare language entries in source code with a given language template. + + :param template: A set of language entries from a language template. + :param res: A dictionary containing source code paths as keys and lists of ast.Call objects as values. + """ + seen = set() + + for file, calls in res.items(): + for c in calls: + arg = get_arg(c) + if arg in template: + seen.add(arg) + else: + print(f'NEW! {file}:{c.lineno}: {arg!r}') + + for old in set(template) ^ seen: + print(f'No longer used: {old}') + + +def print_json_output(res: dict[pathlib.Path, list[ast.Call]]) -> None: + """ + Print JSON output of extracted language entries. + + :param res: A dictionary containing source code paths as keys and lists of ast.Call objects as values. + """ + to_print_data = [ + { + 'path': str(path), + 'string': get_arg(c), + 'reconstructed': ast.unparse(c), # type: ignore + 'start_line': c.lineno, + 'start_offset': c.col_offset, + 'end_line': c.end_lineno, + 'end_offset': c.end_col_offset, + 'comment': getattr(c, 'comment', None) + } for (path, calls) in res.items() for c in calls + ] + + print(json.dumps(to_print_data, indent=2)) + + +if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--directory', help='Directory to search from', default='.') parser.add_argument('--ignore', action='append', help='directories to ignore', default=['venv', '.venv', '.git']) @@ -302,56 +313,32 @@ if __name__ == '__main__': args = parser.parse_args() directory = pathlib.Path(args.directory) - res = scan_directory(directory, [pathlib.Path(p) for p in args.ignore]) + skip = [pathlib.Path(p) for p in args.ignore] + res = scan_directory(directory, skip) - if args.compare_lang is not None and len(args.compare_lang) > 0: - seen = set() + if args.compare_lang: template = parse_template(args.compare_lang) - - for file, calls in res.items(): - for c in calls: - arg = get_arg(c) - if arg in template: - seen.add(arg) - else: - print(f'NEW! {file}:{c.lineno}: {arg!r}') - - for old in set(template) ^ seen: - print(f'No longer used: {old}') + compare_lang_with_template(template, res) elif args.json: - to_print_data = [ - { - 'path': str(path), - 'string': get_arg(c), - 'reconstructed': ast.unparse(c), - 'start_line': c.lineno, - 'start_offset': c.col_offset, - 'end_line': c.end_lineno, - 'end_offset': c.end_col_offset, - 'comment': getattr(c, 'comment', None) - } for (path, calls) in res.items() for c in calls - ] - - print(json.dumps(to_print_data, indent=2)) + print_json_output(res) elif args.lang: if args.lang == '-': print(generate_lang_template(res)) - else: with open(args.lang, mode='w+', newline='\n') as langfile: langfile.writelines(generate_lang_template(res)) else: for path, calls in res.items(): - if len(calls) == 0: + if not calls: continue print(path) for c in calls: print( - f' {c.lineno:4d}({c.col_offset:3d}):{c.end_lineno:4d}({c.end_col_offset:3d})\t', ast.unparse(c) + f' {c.lineno:4d}({c.col_offset:3d}):{c.end_lineno:4d}(' + f'{c.end_col_offset:3d})\t', ast.unparse(c) # type: ignore ) - print() diff --git a/scripts/killswitch_test.py b/scripts/killswitch_test.py index b85b341e..cef93c88 100644 --- a/scripts/killswitch_test.py +++ b/scripts/killswitch_test.py @@ -116,9 +116,14 @@ if __name__ == '__main__': if file_name == '-': file = sys.stdin else: - file = open(file_name) - - res = json.load(file) - file.close() + try: + with open(file_name) as file: + res = json.load(file) + except FileNotFoundError: + print(f"File '{file_name}' not found.") + sys.exit(1) + except json.JSONDecodeError: + print(f"Error decoding JSON in '{file_name}'.") + sys.exit(1) show_killswitch_set_info(KillSwitchSet(parse_kill_switches(res))) diff --git a/scripts/pip_rev_deps.py b/scripts/pip_rev_deps.py index b97757d5..8a187172 100644 --- a/scripts/pip_rev_deps.py +++ b/scripts/pip_rev_deps.py @@ -1,11 +1,10 @@ -#!/usr/bin/env python -"""Find the reverse dependencies of a package according to pip.""" +"""Search for dependencies given a package.""" import sys import pkg_resources -def find_reverse_deps(package_name: str): +def find_reverse_deps(package_name: str) -> list[str]: """ Find the packages that depend on the named one. @@ -19,4 +18,16 @@ def find_reverse_deps(package_name: str): if __name__ == '__main__': - print(find_reverse_deps(sys.argv[1])) + if len(sys.argv) != 2: + print("Usage: python reverse_deps.py ") + sys.exit(1) + + package_name = sys.argv[1] + reverse_deps = find_reverse_deps(package_name) + + if reverse_deps: + print(f"Reverse dependencies of '{package_name}':") + for dep in reverse_deps: + print(dep) + else: + print(f"No reverse dependencies found for '{package_name}'.")