From c8edce26e69cd8b14b4b84c203515b25a513b2d1 Mon Sep 17 00:00:00 2001 From: David Sangrey Date: Thu, 16 Nov 2023 16:11:14 -0500 Subject: [PATCH 1/6] [2051] Workflow and Requirement Update --- .github/workflows/windows-build.yml | 15 ++++----------- requirements-dev.txt | 10 +++++----- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index 7f465497..b50f868f 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -103,18 +103,11 @@ jobs: pip install wheel pip install -r requirements-dev.txt - - name: Download latest WinSparkle release + - name: Download winsparkle run: | - $url = "https://api.github.com/repos/vslavik/winsparkle/releases/latest" - $response = Invoke-RestMethod -Uri $url - $latestAsset = $response.assets | Where-Object { $_.name -match "WinSparkle.*\.zip" -and $_.name -notmatch "-src" } - - $downloadUrl = $latestAsset.browser_download_url - Invoke-WebRequest -Uri $downloadUrl -OutFile WinSparkle-Latest.zip - - Expand-Archive -Path WinSparkle-Latest.zip -DestinationPath . - $extractedFolder = Get-ChildItem -Filter "WinSparkle-*" -Directory - Move-Item -Path "$($extractedFolder.FullName)\Release\*" -Destination . + Invoke-Webrequest -UseBasicParsing https://github.com/vslavik/winsparkle/releases/download/v0.8.0/WinSparkle-0.8.0.zip -OutFile out.zip + Expand-Archive out.zip + Move-Item 'out\WinSparkle-0.8.0\Release\*' '.\' - name: Build EDMC run: | diff --git a/requirements-dev.txt b/requirements-dev.txt index a8b0815b..3ab5f9a3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -18,24 +18,24 @@ flake8-noqa==1.3.2 flake8-polyfill==1.0.2 flake8-use-fstring==1.4 -mypy==1.6.1 +mypy==1.7.0 pep8-naming==0.13.3 safety==2.3.5 -types-requests==2.31.0.2 +types-requests==2.31.0.10 types-pkg-resources==0.1.3 # Code formatting tools autopep8==2.0.4 # Git pre-commit checking -pre-commit==3.3.3 +pre-commit==3.5.0 # HTML changelogs -grip==4.6.1 +grip==4.6.2 # Packaging # We only need py2exe on windows. -py2exe==0.13.0.0; sys_platform == 'win32' +py2exe==0.13.0.1; sys_platform == 'win32' # Testing pytest==7.4.3 From 62cf621b2049de0c966d98b60d0afff999147899 Mon Sep 17 00:00:00 2001 From: David Sangrey Date: Thu, 16 Nov 2023 16:11:37 -0500 Subject: [PATCH 2/6] [2051] Docfile Brief Pass --- docs/examples/click_counter/load.py | 4 ++-- docs/examples/plugintest/load.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/examples/click_counter/load.py b/docs/examples/click_counter/load.py index 70c24f53..0c65fc29 100644 --- a/docs/examples/click_counter/load.py +++ b/docs/examples/click_counter/load.py @@ -79,7 +79,7 @@ class ClickCounter: """ # You need to cast to `int` here to store *as* an `int`, so that # `config.get_int()` will work for re-loading the value. - config.set('click_counter_count', int(self.click_count.get())) # type: ignore + config.set('click_counter_count', int(self.click_count.get())) def setup_main_ui(self, parent: tk.Frame) -> tk.Frame: """ @@ -95,7 +95,7 @@ class ClickCounter: button = tk.Button( frame, text="Count me", - command=lambda: self.click_count.set(str(int(self.click_count.get()) + 1)) # type: ignore + command=lambda: self.click_count.set(str(int(self.click_count.get()) + 1)) ) button.grid(row=current_row) current_row += 1 diff --git a/docs/examples/plugintest/load.py b/docs/examples/plugintest/load.py index 314eef19..4aca1689 100644 --- a/docs/examples/plugintest/load.py +++ b/docs/examples/plugintest/load.py @@ -44,7 +44,7 @@ class This: this = This() -class PluginTest(object): +class PluginTest: """Class that performs actual tests on bundled modules.""" def __init__(self, directory: str): @@ -83,7 +83,6 @@ class PluginTest(object): logger.debug(f'timestamp = "{timestamp}", cmdr = "{cmdrname}", system = "{system}", station = "{station}", event = "{event}"') # noqa: E501 self.sqlc.execute('INSERT INTO entries VALUES(?, ?, ?, ?, ?)', (timestamp, cmdrname, system, station, event)) self.sqlconn.commit() - return None def plugin_start3(plugin_dir: str) -> str: From c7c6d5eed7d9f4fa60ea1cd2b23c905b915f49ab Mon Sep 17 00:00:00 2001 From: David Sangrey Date: Thu, 16 Nov 2023 16:13:14 -0500 Subject: [PATCH 3/6] [2051] Scripts Update --- scripts/find_localised_strings.py | 269 ++++++++++++++---------------- scripts/killswitch_test.py | 13 +- scripts/pip_rev_deps.py | 19 ++- 3 files changed, 152 insertions(+), 149 deletions(-) diff --git a/scripts/find_localised_strings.py b/scripts/find_localised_strings.py index 74cfe1b4..b9d8a8b5 100644 --- a/scripts/find_localised_strings.py +++ b/scripts/find_localised_strings.py @@ -1,4 +1,6 @@ """Search all given paths recursively for localised string calls.""" +from __future__ import annotations + import argparse import ast import dataclasses @@ -6,9 +8,6 @@ import json import pathlib import re import sys -from typing import Optional - -# spell-checker: words dedupe deduping deduped def get_func_name(thing: ast.AST) -> str: @@ -16,11 +15,9 @@ def get_func_name(thing: ast.AST) -> str: if isinstance(thing, ast.Name): return thing.id - elif isinstance(thing, ast.Attribute): + if isinstance(thing, ast.Attribute): return get_func_name(thing.value) - - else: - return '' + return '' def get_arg(call: ast.Call) -> str: @@ -31,10 +28,9 @@ def get_arg(call: ast.Call) -> str: arg = call.args[0] if isinstance(arg, ast.Constant): return arg.value - elif isinstance(arg, ast.Name): + if isinstance(arg, ast.Name): return f'VARIABLE! CHECK CODE! {arg.id}' - else: - return f'Unknown! {type(arg)=} {ast.dump(arg)} ||| {ast.unparse(arg)}' + return f'Unknown! {type(arg)=} {ast.dump(arg)} ||| {ast.unparse(arg)}' # type: ignore def find_calls_in_stmt(statement: ast.AST) -> list[ast.Call]: @@ -62,7 +58,7 @@ COMMENT_SAME_LINE_RE = re.compile(r'^.*?(#.*)$') COMMENT_OWN_LINE_RE = re.compile(r'^\s*?(#.*)$') -def extract_comments(call: ast.Call, lines: list[str], file: pathlib.Path) -> Optional[str]: # noqa: CCR001 +def extract_comments(call: ast.Call, lines: list[str], file: pathlib.Path) -> str | None: """ Extract comments from source code based on the given call. @@ -74,51 +70,32 @@ def extract_comments(call: ast.Call, lines: list[str], file: pathlib.Path) -> Op :param file: The path to the file this call node came from :return: The first comment that matches the rules, or None """ - out: Optional[str] = None - above = call.lineno - 2 - current = call.lineno - 1 + above_line_number = call.lineno - 2 + current_line_number = call.lineno - 1 - above_line = lines[above].strip() if len(lines) >= above else None - above_comment: Optional[str] = None - current_line = lines[current].strip() - current_comment: Optional[str] = None + def extract_lang_comment(line: str) -> str | None: + """ + Extract a language comment from a given line. - bad_comment: Optional[str] = None - if above_line is not None: - match = COMMENT_OWN_LINE_RE.match(above_line) - if match: - above_comment = match.group(1).strip() - if not above_comment.startswith('# LANG:'): - bad_comment = f'Unknown comment for {file}:{call.lineno} {above_line}' - above_comment = None + :param line: The line to extract the language comment from. + :return: The extracted language comment, or None if no valid comment is found. + """ + match = COMMENT_OWN_LINE_RE.match(line) + if match and match.group(1).startswith('# LANG:'): + return match.group(1).replace('# LANG:', '').strip() + return None - else: - above_comment = above_comment.replace('# LANG:', '').strip() + above_comment = extract_lang_comment(lines[above_line_number]) if len(lines) >= above_line_number else None + current_comment = extract_lang_comment(lines[current_line_number]) - if current_line is not None: - match = COMMENT_SAME_LINE_RE.match(current_line) - if match: - current_comment = match.group(1).strip() - if not current_comment.startswith('# LANG:'): - bad_comment = f'Unknown comment for {file}:{call.lineno} {current_line}' - current_comment = None + if current_comment is None: + current_comment = above_comment - else: - current_comment = current_comment.replace('# LANG:', '').strip() + if current_comment is None: + print(f'No comment for {file}:{call.lineno} {lines[current_line_number]}', file=sys.stderr) + return None - if current_comment is not None: - out = current_comment - - elif above_comment is not None: - out = above_comment - - elif bad_comment is not None: - print(bad_comment, file=sys.stderr) - - if out is None: - print(f'No comment for {file}:{call.lineno} {current_line}', file=sys.stderr) - - return out + return current_comment def scan_file(path: pathlib.Path) -> list[ast.Call]: @@ -126,17 +103,19 @@ def scan_file(path: pathlib.Path) -> list[ast.Call]: data = path.read_text(encoding='utf-8') lines = data.splitlines() parsed = ast.parse(data) - out: list[ast.Call] = [] + calls: list[ast.Call] = [] for statement in parsed.body: - out.extend(find_calls_in_stmt(statement)) + calls.extend(find_calls_in_stmt(statement)) - # see if we can extract any comments - for call in out: - setattr(call, 'comment', extract_comments(call, lines, path)) + # Extract and assign comments to each call + for call in calls: + call.comment = extract_comments(call, lines, path) # type: ignore - out.sort(key=lambda c: c.lineno) - return out + # Sort the calls by line number + calls.sort(key=lambda c: c.lineno) + + return calls def scan_directory(path: pathlib.Path, skip: list[pathlib.Path] | None = None) -> dict[pathlib.Path, list[ast.Call]]: @@ -150,40 +129,34 @@ def scan_directory(path: pathlib.Path, skip: list[pathlib.Path] | None = None) - for thing in path.iterdir(): if skip is not None and any(s.name == thing.name for s in skip): continue - if thing.is_file(): if not thing.name.endswith('.py'): continue - out[thing] = scan_file(thing) - elif thing.is_dir(): - out |= scan_directory(thing) - + out |= scan_directory(thing) # type: ignore else: raise ValueError(type(thing), thing) - return out -def parse_template(path) -> set[str]: +def parse_template(path: pathlib.Path) -> set[str]: """ Parse a lang.template file. - The regexp this uses was extracted from l10n.py. + The regular expression used here was extracted from l10n.py. :param path: The path to the lang file """ - lang_re = re.compile(r'\s*"((?:[^"]|(?:\"))+)"\s*=\s*"((?:[^"]|(?:\"))+)"\s*;\s*$') - out = set() + lang_re = re.compile(r'\s*"([^"]+)"\s*=\s*"([^"]+)"\s*;\s*$') + result = set() + for line in pathlib.Path(path).read_text(encoding='utf-8').splitlines(): match = lang_re.match(line) - if not match: - continue - if match.group(1) != '!Language': - out.add(match.group(1)) + if match and match.group(1) != '!Language': + result.add(match.group(1)) - return out + return result @dataclasses.dataclass @@ -193,8 +166,8 @@ class FileLocation: path: pathlib.Path line_start: int line_start_col: int - line_end: Optional[int] - line_end_col: Optional[int] + line_end: int | None + line_end_col: int | None @staticmethod def from_call(path: pathlib.Path, c: ast.Call) -> 'FileLocation': @@ -213,50 +186,46 @@ class LangEntry: locations: list[FileLocation] string: str - comments: list[Optional[str]] + comments: list[str | None] def files(self) -> str: """Return a string representation of all the files this LangEntry is in, and its location therein.""" - out = '' - for loc in self.locations: - start = loc.line_start - end = loc.line_end - end_str = f':{end}' if end is not None and end != start else '' - out += f'{loc.path.name}:{start}{end_str}; ' + file_locations = [ + f'{loc.path.name}:{loc.line_start}' + + (f':{loc.line_end}' if loc.line_end is not None and loc.line_end != loc.line_start else '') + for loc in self.locations + ] - return out + return '; '.join(file_locations) def dedupe_lang_entries(entries: list[LangEntry]) -> list[LangEntry]: """ Deduplicate a list of lang entries. - This will coalesce LangEntries that have that same string but differing files and comments into a single - LangEntry that cotains all comments and FileLocations + This will coalesce LangEntries that have the same string but differing files and comments into a single + LangEntry that contains all comments and FileLocations. :param entries: The list to deduplicate :return: The deduplicated list """ - deduped: list[LangEntry] = [] + deduped: dict[str, LangEntry] = {} + for e in entries: - cont = False - for d in deduped: - if d.string == e.string: - cont = True - d.locations.append(e.locations[0]) - d.comments.extend(e.comments) + existing = deduped.get(e.string) + if existing: + existing.locations.extend(e.locations) + existing.comments.extend(e.comments) + else: + deduped[e.string] = LangEntry(locations=e.locations[:], string=e.string, comments=e.comments[:]) - if cont: - continue - - deduped.append(e) - - return deduped + return list(deduped.values()) def generate_lang_template(data: dict[pathlib.Path, list[ast.Call]]) -> str: """Generate a full en.template from the given data.""" entries: list[LangEntry] = [] + for path, calls in data.items(): for c in calls: entries.append(LangEntry([FileLocation.from_call(path, c)], get_arg(c), [getattr(c, 'comment')])) @@ -267,30 +236,72 @@ def generate_lang_template(data: dict[pathlib.Path, list[ast.Call]]) -> str: ''' print(f'Done Deduping entries {len(entries)=} {len(deduped)=}', file=sys.stderr) + for entry in deduped: assert len(entry.comments) == len(entry.locations) - comment = '' - files = 'In files: ' + entry.files() + comment_parts = [] string = f'"{entry.string}"' - for i in range(len(entry.comments)): - if entry.comments[i] is None: + for i, comment_text in enumerate(entry.comments): + if comment_text is None: continue loc = entry.locations[i] - to_append = f'{loc.path.name}: {entry.comments[i]}; ' - if to_append not in comment: - comment += to_append + comment_parts.append(f'{loc.path.name}: {comment_text};') - header = f'{comment.strip()} {files}'.strip() - out += f'/* {header} */\n' - out += f'{string} = {string};\n' - out += '\n' + if comment_parts: + header = ' '.join(comment_parts) + out += f'/* {header} */\n' + + out += f'{string} = {string};\n\n' return out -if __name__ == '__main__': +def compare_lang_with_template(template: set[str], res: dict[pathlib.Path, list[ast.Call]]) -> None: + """ + Compare language entries in source code with a given language template. + + :param template: A set of language entries from a language template. + :param res: A dictionary containing source code paths as keys and lists of ast.Call objects as values. + """ + seen = set() + + for file, calls in res.items(): + for c in calls: + arg = get_arg(c) + if arg in template: + seen.add(arg) + else: + print(f'NEW! {file}:{c.lineno}: {arg!r}') + + for old in set(template) ^ seen: + print(f'No longer used: {old}') + + +def print_json_output(res: dict[pathlib.Path, list[ast.Call]]) -> None: + """ + Print JSON output of extracted language entries. + + :param res: A dictionary containing source code paths as keys and lists of ast.Call objects as values. + """ + to_print_data = [ + { + 'path': str(path), + 'string': get_arg(c), + 'reconstructed': ast.unparse(c), # type: ignore + 'start_line': c.lineno, + 'start_offset': c.col_offset, + 'end_line': c.end_lineno, + 'end_offset': c.end_col_offset, + 'comment': getattr(c, 'comment', None) + } for (path, calls) in res.items() for c in calls + ] + + print(json.dumps(to_print_data, indent=2)) + + +if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--directory', help='Directory to search from', default='.') parser.add_argument('--ignore', action='append', help='directories to ignore', default=['venv', '.venv', '.git']) @@ -302,56 +313,32 @@ if __name__ == '__main__': args = parser.parse_args() directory = pathlib.Path(args.directory) - res = scan_directory(directory, [pathlib.Path(p) for p in args.ignore]) + skip = [pathlib.Path(p) for p in args.ignore] + res = scan_directory(directory, skip) - if args.compare_lang is not None and len(args.compare_lang) > 0: - seen = set() + if args.compare_lang: template = parse_template(args.compare_lang) - - for file, calls in res.items(): - for c in calls: - arg = get_arg(c) - if arg in template: - seen.add(arg) - else: - print(f'NEW! {file}:{c.lineno}: {arg!r}') - - for old in set(template) ^ seen: - print(f'No longer used: {old}') + compare_lang_with_template(template, res) elif args.json: - to_print_data = [ - { - 'path': str(path), - 'string': get_arg(c), - 'reconstructed': ast.unparse(c), - 'start_line': c.lineno, - 'start_offset': c.col_offset, - 'end_line': c.end_lineno, - 'end_offset': c.end_col_offset, - 'comment': getattr(c, 'comment', None) - } for (path, calls) in res.items() for c in calls - ] - - print(json.dumps(to_print_data, indent=2)) + print_json_output(res) elif args.lang: if args.lang == '-': print(generate_lang_template(res)) - else: with open(args.lang, mode='w+', newline='\n') as langfile: langfile.writelines(generate_lang_template(res)) else: for path, calls in res.items(): - if len(calls) == 0: + if not calls: continue print(path) for c in calls: print( - f' {c.lineno:4d}({c.col_offset:3d}):{c.end_lineno:4d}({c.end_col_offset:3d})\t', ast.unparse(c) + f' {c.lineno:4d}({c.col_offset:3d}):{c.end_lineno:4d}(' + f'{c.end_col_offset:3d})\t', ast.unparse(c) # type: ignore ) - print() diff --git a/scripts/killswitch_test.py b/scripts/killswitch_test.py index b85b341e..cef93c88 100644 --- a/scripts/killswitch_test.py +++ b/scripts/killswitch_test.py @@ -116,9 +116,14 @@ if __name__ == '__main__': if file_name == '-': file = sys.stdin else: - file = open(file_name) - - res = json.load(file) - file.close() + try: + with open(file_name) as file: + res = json.load(file) + except FileNotFoundError: + print(f"File '{file_name}' not found.") + sys.exit(1) + except json.JSONDecodeError: + print(f"Error decoding JSON in '{file_name}'.") + sys.exit(1) show_killswitch_set_info(KillSwitchSet(parse_kill_switches(res))) diff --git a/scripts/pip_rev_deps.py b/scripts/pip_rev_deps.py index b97757d5..8a187172 100644 --- a/scripts/pip_rev_deps.py +++ b/scripts/pip_rev_deps.py @@ -1,11 +1,10 @@ -#!/usr/bin/env python -"""Find the reverse dependencies of a package according to pip.""" +"""Search for dependencies given a package.""" import sys import pkg_resources -def find_reverse_deps(package_name: str): +def find_reverse_deps(package_name: str) -> list[str]: """ Find the packages that depend on the named one. @@ -19,4 +18,16 @@ def find_reverse_deps(package_name: str): if __name__ == '__main__': - print(find_reverse_deps(sys.argv[1])) + if len(sys.argv) != 2: + print("Usage: python reverse_deps.py ") + sys.exit(1) + + package_name = sys.argv[1] + reverse_deps = find_reverse_deps(package_name) + + if reverse_deps: + print(f"Reverse dependencies of '{package_name}':") + for dep in reverse_deps: + print(dep) + else: + print(f"No reverse dependencies found for '{package_name}'.") From 8693c1a66798d47700c0d05d4b9ba7b5e9cd3536 Mon Sep 17 00:00:00 2001 From: David Sangrey Date: Thu, 16 Nov 2023 16:14:59 -0500 Subject: [PATCH 4/6] [2051] Utilities Quick Pass --- util/text.py | 8 +++++++- util_ships.py | 12 +++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/util/text.py b/util/text.py index 0755f0f3..84af758b 100644 --- a/util/text.py +++ b/util/text.py @@ -1,4 +1,10 @@ -"""Utilities for dealing with text (and byte representations thereof).""" +""" +text.py - Dealing with Text and Bytes. + +Copyright (c) EDCD, All Rights Reserved +Licensed under the GNU General Public License. +See LICENSE file. +""" from __future__ import annotations from gzip import compress diff --git a/util_ships.py b/util_ships.py index bd3ae3df..8bbfd813 100644 --- a/util_ships.py +++ b/util_ships.py @@ -1,4 +1,10 @@ -"""Utility functions relating to ships.""" +""" +util_ships.py - Ship Utilities. + +Copyright (c) EDCD, All Rights Reserved +Licensed under the GNU General Public License. +See LICENSE file. +""" from edmc_data import ship_name_map @@ -11,6 +17,6 @@ def ship_file_name(ship_name: str, ship_type: str) -> str: if name.lower() in ('con', 'prn', 'aux', 'nul', 'com0', 'com2', 'com3', 'com4', 'com5', 'com6', 'com7', 'com8', 'com9', 'lpt0', 'lpt2', 'lpt3', 'lpt4', 'lpt5', 'lpt6', 'lpt7', 'lpt8', 'lpt9'): - name = name + '_' + name += '_' - return name.translate({ord(x): u'_' for x in ('\0', '<', '>', ':', '"', '/', '\\', '|', '?', '*')}) + return name.translate({ord(x): '_' for x in ('\0', '<', '>', ':', '"', '/', '\\', '|', '?', '*')}) From b741047fd589a1add15a2de4276dac1903dd0bee Mon Sep 17 00:00:00 2001 From: David Sangrey Date: Thu, 16 Nov 2023 18:16:25 -0500 Subject: [PATCH 5/6] [2051] Fix Localized Strings --- plugins/edsm.py | 4 +- scripts/find_localised_strings.py | 235 ++++++++++++++---------------- 2 files changed, 113 insertions(+), 126 deletions(-) diff --git a/plugins/edsm.py b/plugins/edsm.py index 33af692b..4827dbf4 100644 --- a/plugins/edsm.py +++ b/plugins/edsm.py @@ -309,7 +309,7 @@ def plugin_prefs(parent: ttk.Notebook, cmdr: str | None, is_beta: bool) -> tk.Fr this.log = tk.IntVar(value=config.get_int('edsm_out') and 1) this.log_button = nb.Checkbutton( frame, - text=_('Send flight log and Cmdr status to EDSM'), + text=_('Send flight log and CMDR status to EDSM'), # LANG: Send flight log and CMDR Status to EDSM variable=this.log, command=prefsvarchanged ) @@ -320,7 +320,7 @@ def plugin_prefs(parent: ttk.Notebook, cmdr: str | None, is_beta: bool) -> tk.Fr this.label = HyperlinkLabel( frame, - text=_('Elite Dangerous Star Map credentials'), + text=_('Elite Dangerous Star Map credentials'), # LANG: Elite Dangerous Star Map credentials background=nb.Label().cget('background'), url='https://www.edsm.net/settings/api', underline=True diff --git a/scripts/find_localised_strings.py b/scripts/find_localised_strings.py index b9d8a8b5..cdc95686 100644 --- a/scripts/find_localised_strings.py +++ b/scripts/find_localised_strings.py @@ -30,7 +30,7 @@ def get_arg(call: ast.Call) -> str: return arg.value if isinstance(arg, ast.Name): return f'VARIABLE! CHECK CODE! {arg.id}' - return f'Unknown! {type(arg)=} {ast.dump(arg)} ||| {ast.unparse(arg)}' # type: ignore + return f'Unknown! {type(arg)=} {ast.dump(arg)} ||| {ast.unparse(arg)}' def find_calls_in_stmt(statement: ast.AST) -> list[ast.Call]: @@ -39,9 +39,7 @@ def find_calls_in_stmt(statement: ast.AST) -> list[ast.Call]: for n in ast.iter_child_nodes(statement): out.extend(find_calls_in_stmt(n)) if isinstance(statement, ast.Call) and get_func_name(statement.func) == '_': - out.append(statement) - return out @@ -70,32 +68,48 @@ def extract_comments(call: ast.Call, lines: list[str], file: pathlib.Path) -> st :param file: The path to the file this call node came from :return: The first comment that matches the rules, or None """ - above_line_number = call.lineno - 2 - current_line_number = call.lineno - 1 + out: str | None = None + above = call.lineno - 2 + current = call.lineno - 1 - def extract_lang_comment(line: str) -> str | None: - """ - Extract a language comment from a given line. + above_line = lines[above].strip() if len(lines) >= above else None + above_comment: str | None = None + current_line = lines[current].strip() + current_comment: str | None = None - :param line: The line to extract the language comment from. - :return: The extracted language comment, or None if no valid comment is found. - """ - match = COMMENT_OWN_LINE_RE.match(line) - if match and match.group(1).startswith('# LANG:'): - return match.group(1).replace('# LANG:', '').strip() - return None + bad_comment: str | None = None + if above_line is not None: + match = COMMENT_OWN_LINE_RE.match(above_line) + if match: + above_comment = match.group(1).strip() + if not above_comment.startswith('# LANG:'): + bad_comment = f'Unknown comment for {file}:{call.lineno} {above_line}' + above_comment = None - above_comment = extract_lang_comment(lines[above_line_number]) if len(lines) >= above_line_number else None - current_comment = extract_lang_comment(lines[current_line_number]) + else: + above_comment = above_comment.replace('# LANG:', '').strip() - if current_comment is None: - current_comment = above_comment + if current_line is not None: + match = COMMENT_SAME_LINE_RE.match(current_line) + if match: + current_comment = match.group(1).strip() + if not current_comment.startswith('# LANG:'): + bad_comment = f'Unknown comment for {file}:{call.lineno} {current_line}' + current_comment = None - if current_comment is None: - print(f'No comment for {file}:{call.lineno} {lines[current_line_number]}', file=sys.stderr) - return None + else: + current_comment = current_comment.replace('# LANG:', '').strip() - return current_comment + if current_comment is not None: + out = current_comment + elif above_comment is not None: + out = above_comment + elif bad_comment is not None: + print(bad_comment, file=sys.stderr) + + if out is None: + print(f'No comment for {file}:{call.lineno} {current_line}', file=sys.stderr) + return out def scan_file(path: pathlib.Path) -> list[ast.Call]: @@ -103,19 +117,17 @@ def scan_file(path: pathlib.Path) -> list[ast.Call]: data = path.read_text(encoding='utf-8') lines = data.splitlines() parsed = ast.parse(data) - calls: list[ast.Call] = [] + out: list[ast.Call] = [] for statement in parsed.body: - calls.extend(find_calls_in_stmt(statement)) + out.extend(find_calls_in_stmt(statement)) - # Extract and assign comments to each call - for call in calls: - call.comment = extract_comments(call, lines, path) # type: ignore + # see if we can extract any comments + for call in out: + setattr(call, 'comment', extract_comments(call, lines, path)) - # Sort the calls by line number - calls.sort(key=lambda c: c.lineno) - - return calls + out.sort(key=lambda c: c.lineno) + return out def scan_directory(path: pathlib.Path, skip: list[pathlib.Path] | None = None) -> dict[pathlib.Path, list[ast.Call]]: @@ -125,38 +137,38 @@ def scan_directory(path: pathlib.Path, skip: list[pathlib.Path] | None = None) - :param path: path to scan :param skip: paths to skip, if any, defaults to None """ + if skip is None: + skip = [] out = {} for thing in path.iterdir(): - if skip is not None and any(s.name == thing.name for s in skip): + if any(same_path.name == thing.name for same_path in skip): continue - if thing.is_file(): - if not thing.name.endswith('.py'): - continue + + if thing.is_file() and thing.suffix == '.py': out[thing] = scan_file(thing) elif thing.is_dir(): - out |= scan_directory(thing) # type: ignore - else: - raise ValueError(type(thing), thing) + out.update(scan_directory(thing, skip)) + return out -def parse_template(path: pathlib.Path) -> set[str]: +def parse_template(path) -> set[str]: """ Parse a lang.template file. - The regular expression used here was extracted from l10n.py. + The regexp this uses was extracted from l10n.py. :param path: The path to the lang file """ lang_re = re.compile(r'\s*"([^"]+)"\s*=\s*"([^"]+)"\s*;\s*$') - result = set() + out = set() + with open(path, encoding='utf-8') as file: + for line in file: + match = lang_re.match(line.strip()) + if match and match.group(1) != '!Language': + out.add(match.group(1)) - for line in pathlib.Path(path).read_text(encoding='utf-8').splitlines(): - match = lang_re.match(line) - if match and match.group(1) != '!Language': - result.add(match.group(1)) - - return result + return out @dataclasses.dataclass @@ -191,41 +203,38 @@ class LangEntry: def files(self) -> str: """Return a string representation of all the files this LangEntry is in, and its location therein.""" file_locations = [ - f'{loc.path.name}:{loc.line_start}' + - (f':{loc.line_end}' if loc.line_end is not None and loc.line_end != loc.line_start else '') + f"{loc.path.name}:{loc.line_start}:{loc.line_end or ''}" for loc in self.locations ] - - return '; '.join(file_locations) + return "; ".join(file_locations) def dedupe_lang_entries(entries: list[LangEntry]) -> list[LangEntry]: """ Deduplicate a list of lang entries. - This will coalesce LangEntries that have the same string but differing files and comments into a single - LangEntry that contains all comments and FileLocations. + This will coalesce LangEntries that have that same string but differing files and comments into a single + LangEntry that cotains all comments and FileLocations :param entries: The list to deduplicate :return: The deduplicated list """ deduped: dict[str, LangEntry] = {} - for e in entries: existing = deduped.get(e.string) if existing: existing.locations.extend(e.locations) existing.comments.extend(e.comments) else: - deduped[e.string] = LangEntry(locations=e.locations[:], string=e.string, comments=e.comments[:]) - + deduped[e.string] = LangEntry( + locations=e.locations[:], string=e.string, comments=e.comments[:] + ) return list(deduped.values()) def generate_lang_template(data: dict[pathlib.Path, list[ast.Call]]) -> str: """Generate a full en.template from the given data.""" entries: list[LangEntry] = [] - for path, calls in data.items(): for c in calls: entries.append(LangEntry([FileLocation.from_call(path, c)], get_arg(c), [getattr(c, 'comment')])) @@ -236,72 +245,26 @@ def generate_lang_template(data: dict[pathlib.Path, list[ast.Call]]) -> str: ''' print(f'Done Deduping entries {len(entries)=} {len(deduped)=}', file=sys.stderr) - for entry in deduped: assert len(entry.comments) == len(entry.locations) - comment_parts = [] + + comment_set = set() + for comment, loc in zip(entry.comments, entry.locations): + if comment: + comment_set.add(f'{loc.path.name}: {comment};') + + files = 'In files: ' + entry.files() + comment = ' '.join(comment_set).strip() + + header = f'{comment} {files}'.strip() string = f'"{entry.string}"' - - for i, comment_text in enumerate(entry.comments): - if comment_text is None: - continue - - loc = entry.locations[i] - comment_parts.append(f'{loc.path.name}: {comment_text};') - - if comment_parts: - header = ' '.join(comment_parts) - out += f'/* {header} */\n' - + out += f'/* {header} */\n' out += f'{string} = {string};\n\n' return out -def compare_lang_with_template(template: set[str], res: dict[pathlib.Path, list[ast.Call]]) -> None: - """ - Compare language entries in source code with a given language template. - - :param template: A set of language entries from a language template. - :param res: A dictionary containing source code paths as keys and lists of ast.Call objects as values. - """ - seen = set() - - for file, calls in res.items(): - for c in calls: - arg = get_arg(c) - if arg in template: - seen.add(arg) - else: - print(f'NEW! {file}:{c.lineno}: {arg!r}') - - for old in set(template) ^ seen: - print(f'No longer used: {old}') - - -def print_json_output(res: dict[pathlib.Path, list[ast.Call]]) -> None: - """ - Print JSON output of extracted language entries. - - :param res: A dictionary containing source code paths as keys and lists of ast.Call objects as values. - """ - to_print_data = [ - { - 'path': str(path), - 'string': get_arg(c), - 'reconstructed': ast.unparse(c), # type: ignore - 'start_line': c.lineno, - 'start_offset': c.col_offset, - 'end_line': c.end_lineno, - 'end_offset': c.end_col_offset, - 'comment': getattr(c, 'comment', None) - } for (path, calls) in res.items() for c in calls - ] - - print(json.dumps(to_print_data, indent=2)) - - -if __name__ == "__main__": +if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--directory', help='Directory to search from', default='.') parser.add_argument('--ignore', action='append', help='directories to ignore', default=['venv', '.venv', '.git']) @@ -313,32 +276,56 @@ if __name__ == "__main__": args = parser.parse_args() directory = pathlib.Path(args.directory) - skip = [pathlib.Path(p) for p in args.ignore] - res = scan_directory(directory, skip) + res = scan_directory(directory, [pathlib.Path(p) for p in args.ignore]) - if args.compare_lang: + if args.compare_lang is not None and len(args.compare_lang) > 0: + seen = set() template = parse_template(args.compare_lang) - compare_lang_with_template(template, res) + + for file, calls in res.items(): + for c in calls: + arg = get_arg(c) + if arg in template: + seen.add(arg) + else: + print(f'NEW! {file}:{c.lineno}: {arg!r}') + + for old in set(template) ^ seen: + print(f'No longer used: {old}') elif args.json: - print_json_output(res) + to_print_data = [ + { + 'path': str(path), + 'string': get_arg(c), + 'reconstructed': ast.unparse(c), + 'start_line': c.lineno, + 'start_offset': c.col_offset, + 'end_line': c.end_lineno, + 'end_offset': c.end_col_offset, + 'comment': getattr(c, 'comment', None) + } for (path, calls) in res.items() for c in calls + ] + + print(json.dumps(to_print_data, indent=2)) elif args.lang: if args.lang == '-': print(generate_lang_template(res)) + else: with open(args.lang, mode='w+', newline='\n') as langfile: langfile.writelines(generate_lang_template(res)) else: for path, calls in res.items(): - if not calls: + if len(calls) == 0: continue print(path) for c in calls: print( - f' {c.lineno:4d}({c.col_offset:3d}):{c.end_lineno:4d}(' - f'{c.end_col_offset:3d})\t', ast.unparse(c) # type: ignore + f' {c.lineno:4d}({c.col_offset:3d}):{c.end_lineno:4d}({c.end_col_offset:3d})\t', ast.unparse(c) ) + print() From 93619e456560db5891aedf078bb0528345376c5b Mon Sep 17 00:00:00 2001 From: David Sangrey Date: Thu, 16 Nov 2023 18:16:25 -0500 Subject: [PATCH 6/6] [2051] Fix Localized Strings --- plugins/edsm.py | 4 +- scripts/find_localised_strings.py | 237 ++++++++++++++---------------- 2 files changed, 114 insertions(+), 127 deletions(-) diff --git a/plugins/edsm.py b/plugins/edsm.py index 33af692b..4827dbf4 100644 --- a/plugins/edsm.py +++ b/plugins/edsm.py @@ -309,7 +309,7 @@ def plugin_prefs(parent: ttk.Notebook, cmdr: str | None, is_beta: bool) -> tk.Fr this.log = tk.IntVar(value=config.get_int('edsm_out') and 1) this.log_button = nb.Checkbutton( frame, - text=_('Send flight log and Cmdr status to EDSM'), + text=_('Send flight log and CMDR status to EDSM'), # LANG: Send flight log and CMDR Status to EDSM variable=this.log, command=prefsvarchanged ) @@ -320,7 +320,7 @@ def plugin_prefs(parent: ttk.Notebook, cmdr: str | None, is_beta: bool) -> tk.Fr this.label = HyperlinkLabel( frame, - text=_('Elite Dangerous Star Map credentials'), + text=_('Elite Dangerous Star Map credentials'), # LANG: Elite Dangerous Star Map credentials background=nb.Label().cget('background'), url='https://www.edsm.net/settings/api', underline=True diff --git a/scripts/find_localised_strings.py b/scripts/find_localised_strings.py index b9d8a8b5..1d1da826 100644 --- a/scripts/find_localised_strings.py +++ b/scripts/find_localised_strings.py @@ -30,7 +30,7 @@ def get_arg(call: ast.Call) -> str: return arg.value if isinstance(arg, ast.Name): return f'VARIABLE! CHECK CODE! {arg.id}' - return f'Unknown! {type(arg)=} {ast.dump(arg)} ||| {ast.unparse(arg)}' # type: ignore + return f'Unknown! {type(arg)=} {ast.dump(arg)} ||| {ast.unparse(arg)}' def find_calls_in_stmt(statement: ast.AST) -> list[ast.Call]: @@ -39,9 +39,7 @@ def find_calls_in_stmt(statement: ast.AST) -> list[ast.Call]: for n in ast.iter_child_nodes(statement): out.extend(find_calls_in_stmt(n)) if isinstance(statement, ast.Call) and get_func_name(statement.func) == '_': - out.append(statement) - return out @@ -58,7 +56,7 @@ COMMENT_SAME_LINE_RE = re.compile(r'^.*?(#.*)$') COMMENT_OWN_LINE_RE = re.compile(r'^\s*?(#.*)$') -def extract_comments(call: ast.Call, lines: list[str], file: pathlib.Path) -> str | None: +def extract_comments(call: ast.Call, lines: list[str], file: pathlib.Path) -> str | None: # noqa: CCR001 """ Extract comments from source code based on the given call. @@ -70,32 +68,48 @@ def extract_comments(call: ast.Call, lines: list[str], file: pathlib.Path) -> st :param file: The path to the file this call node came from :return: The first comment that matches the rules, or None """ - above_line_number = call.lineno - 2 - current_line_number = call.lineno - 1 + out: str | None = None + above = call.lineno - 2 + current = call.lineno - 1 - def extract_lang_comment(line: str) -> str | None: - """ - Extract a language comment from a given line. + above_line = lines[above].strip() if len(lines) >= above else None + above_comment: str | None = None + current_line = lines[current].strip() + current_comment: str | None = None - :param line: The line to extract the language comment from. - :return: The extracted language comment, or None if no valid comment is found. - """ - match = COMMENT_OWN_LINE_RE.match(line) - if match and match.group(1).startswith('# LANG:'): - return match.group(1).replace('# LANG:', '').strip() - return None + bad_comment: str | None = None + if above_line is not None: + match = COMMENT_OWN_LINE_RE.match(above_line) + if match: + above_comment = match.group(1).strip() + if not above_comment.startswith('# LANG:'): + bad_comment = f'Unknown comment for {file}:{call.lineno} {above_line}' + above_comment = None - above_comment = extract_lang_comment(lines[above_line_number]) if len(lines) >= above_line_number else None - current_comment = extract_lang_comment(lines[current_line_number]) + else: + above_comment = above_comment.replace('# LANG:', '').strip() - if current_comment is None: - current_comment = above_comment + if current_line is not None: + match = COMMENT_SAME_LINE_RE.match(current_line) + if match: + current_comment = match.group(1).strip() + if not current_comment.startswith('# LANG:'): + bad_comment = f'Unknown comment for {file}:{call.lineno} {current_line}' + current_comment = None - if current_comment is None: - print(f'No comment for {file}:{call.lineno} {lines[current_line_number]}', file=sys.stderr) - return None + else: + current_comment = current_comment.replace('# LANG:', '').strip() - return current_comment + if current_comment is not None: + out = current_comment + elif above_comment is not None: + out = above_comment + elif bad_comment is not None: + print(bad_comment, file=sys.stderr) + + if out is None: + print(f'No comment for {file}:{call.lineno} {current_line}', file=sys.stderr) + return out def scan_file(path: pathlib.Path) -> list[ast.Call]: @@ -103,19 +117,17 @@ def scan_file(path: pathlib.Path) -> list[ast.Call]: data = path.read_text(encoding='utf-8') lines = data.splitlines() parsed = ast.parse(data) - calls: list[ast.Call] = [] + out: list[ast.Call] = [] for statement in parsed.body: - calls.extend(find_calls_in_stmt(statement)) + out.extend(find_calls_in_stmt(statement)) - # Extract and assign comments to each call - for call in calls: - call.comment = extract_comments(call, lines, path) # type: ignore + # see if we can extract any comments + for call in out: + setattr(call, 'comment', extract_comments(call, lines, path)) - # Sort the calls by line number - calls.sort(key=lambda c: c.lineno) - - return calls + out.sort(key=lambda c: c.lineno) + return out def scan_directory(path: pathlib.Path, skip: list[pathlib.Path] | None = None) -> dict[pathlib.Path, list[ast.Call]]: @@ -125,38 +137,38 @@ def scan_directory(path: pathlib.Path, skip: list[pathlib.Path] | None = None) - :param path: path to scan :param skip: paths to skip, if any, defaults to None """ + if skip is None: + skip = [] out = {} for thing in path.iterdir(): - if skip is not None and any(s.name == thing.name for s in skip): + if any(same_path.name == thing.name for same_path in skip): continue - if thing.is_file(): - if not thing.name.endswith('.py'): - continue + + if thing.is_file() and thing.suffix == '.py': out[thing] = scan_file(thing) elif thing.is_dir(): - out |= scan_directory(thing) # type: ignore - else: - raise ValueError(type(thing), thing) + out.update(scan_directory(thing, skip)) + return out -def parse_template(path: pathlib.Path) -> set[str]: +def parse_template(path) -> set[str]: """ Parse a lang.template file. - The regular expression used here was extracted from l10n.py. + The regexp this uses was extracted from l10n.py. :param path: The path to the lang file """ lang_re = re.compile(r'\s*"([^"]+)"\s*=\s*"([^"]+)"\s*;\s*$') - result = set() + out = set() + with open(path, encoding='utf-8') as file: + for line in file: + match = lang_re.match(line.strip()) + if match and match.group(1) != '!Language': + out.add(match.group(1)) - for line in pathlib.Path(path).read_text(encoding='utf-8').splitlines(): - match = lang_re.match(line) - if match and match.group(1) != '!Language': - result.add(match.group(1)) - - return result + return out @dataclasses.dataclass @@ -191,41 +203,38 @@ class LangEntry: def files(self) -> str: """Return a string representation of all the files this LangEntry is in, and its location therein.""" file_locations = [ - f'{loc.path.name}:{loc.line_start}' + - (f':{loc.line_end}' if loc.line_end is not None and loc.line_end != loc.line_start else '') + f"{loc.path.name}:{loc.line_start}:{loc.line_end or ''}" for loc in self.locations ] - - return '; '.join(file_locations) + return "; ".join(file_locations) def dedupe_lang_entries(entries: list[LangEntry]) -> list[LangEntry]: """ Deduplicate a list of lang entries. - This will coalesce LangEntries that have the same string but differing files and comments into a single - LangEntry that contains all comments and FileLocations. + This will coalesce LangEntries that have that same string but differing files and comments into a single + LangEntry that cotains all comments and FileLocations :param entries: The list to deduplicate :return: The deduplicated list """ deduped: dict[str, LangEntry] = {} - for e in entries: existing = deduped.get(e.string) if existing: existing.locations.extend(e.locations) existing.comments.extend(e.comments) else: - deduped[e.string] = LangEntry(locations=e.locations[:], string=e.string, comments=e.comments[:]) - + deduped[e.string] = LangEntry( + locations=e.locations[:], string=e.string, comments=e.comments[:] + ) return list(deduped.values()) def generate_lang_template(data: dict[pathlib.Path, list[ast.Call]]) -> str: """Generate a full en.template from the given data.""" entries: list[LangEntry] = [] - for path, calls in data.items(): for c in calls: entries.append(LangEntry([FileLocation.from_call(path, c)], get_arg(c), [getattr(c, 'comment')])) @@ -236,72 +245,26 @@ def generate_lang_template(data: dict[pathlib.Path, list[ast.Call]]) -> str: ''' print(f'Done Deduping entries {len(entries)=} {len(deduped)=}', file=sys.stderr) - for entry in deduped: assert len(entry.comments) == len(entry.locations) - comment_parts = [] + + comment_set = set() + for comment, loc in zip(entry.comments, entry.locations): + if comment: + comment_set.add(f'{loc.path.name}: {comment};') + + files = 'In files: ' + entry.files() + comment = ' '.join(comment_set).strip() + + header = f'{comment} {files}'.strip() string = f'"{entry.string}"' - - for i, comment_text in enumerate(entry.comments): - if comment_text is None: - continue - - loc = entry.locations[i] - comment_parts.append(f'{loc.path.name}: {comment_text};') - - if comment_parts: - header = ' '.join(comment_parts) - out += f'/* {header} */\n' - + out += f'/* {header} */\n' out += f'{string} = {string};\n\n' return out -def compare_lang_with_template(template: set[str], res: dict[pathlib.Path, list[ast.Call]]) -> None: - """ - Compare language entries in source code with a given language template. - - :param template: A set of language entries from a language template. - :param res: A dictionary containing source code paths as keys and lists of ast.Call objects as values. - """ - seen = set() - - for file, calls in res.items(): - for c in calls: - arg = get_arg(c) - if arg in template: - seen.add(arg) - else: - print(f'NEW! {file}:{c.lineno}: {arg!r}') - - for old in set(template) ^ seen: - print(f'No longer used: {old}') - - -def print_json_output(res: dict[pathlib.Path, list[ast.Call]]) -> None: - """ - Print JSON output of extracted language entries. - - :param res: A dictionary containing source code paths as keys and lists of ast.Call objects as values. - """ - to_print_data = [ - { - 'path': str(path), - 'string': get_arg(c), - 'reconstructed': ast.unparse(c), # type: ignore - 'start_line': c.lineno, - 'start_offset': c.col_offset, - 'end_line': c.end_lineno, - 'end_offset': c.end_col_offset, - 'comment': getattr(c, 'comment', None) - } for (path, calls) in res.items() for c in calls - ] - - print(json.dumps(to_print_data, indent=2)) - - -if __name__ == "__main__": +if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--directory', help='Directory to search from', default='.') parser.add_argument('--ignore', action='append', help='directories to ignore', default=['venv', '.venv', '.git']) @@ -313,32 +276,56 @@ if __name__ == "__main__": args = parser.parse_args() directory = pathlib.Path(args.directory) - skip = [pathlib.Path(p) for p in args.ignore] - res = scan_directory(directory, skip) + res = scan_directory(directory, [pathlib.Path(p) for p in args.ignore]) - if args.compare_lang: + if args.compare_lang is not None and len(args.compare_lang) > 0: + seen = set() template = parse_template(args.compare_lang) - compare_lang_with_template(template, res) + + for file, calls in res.items(): + for c in calls: + arg = get_arg(c) + if arg in template: + seen.add(arg) + else: + print(f'NEW! {file}:{c.lineno}: {arg!r}') + + for old in set(template) ^ seen: + print(f'No longer used: {old}') elif args.json: - print_json_output(res) + to_print_data = [ + { + 'path': str(path), + 'string': get_arg(c), + 'reconstructed': ast.unparse(c), + 'start_line': c.lineno, + 'start_offset': c.col_offset, + 'end_line': c.end_lineno, + 'end_offset': c.end_col_offset, + 'comment': getattr(c, 'comment', None) + } for (path, calls) in res.items() for c in calls + ] + + print(json.dumps(to_print_data, indent=2)) elif args.lang: if args.lang == '-': print(generate_lang_template(res)) + else: with open(args.lang, mode='w+', newline='\n') as langfile: langfile.writelines(generate_lang_template(res)) else: for path, calls in res.items(): - if not calls: + if len(calls) == 0: continue print(path) for c in calls: print( - f' {c.lineno:4d}({c.col_offset:3d}):{c.end_lineno:4d}(' - f'{c.end_col_offset:3d})\t', ast.unparse(c) # type: ignore + f' {c.lineno:4d}({c.col_offset:3d}):{c.end_lineno:4d}({c.end_col_offset:3d})\t', ast.unparse(c) ) + print()