Merge branch 'develop' into enhancement/519/add-beta

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,22 +7,19 @@ assignees: ''
 
 ---
 
-**Please check the [Known Issues](https://github.com/EDCD/EDMarketConnector/issues/618) in case this has already been reported.**
-
-**Please also check if the issue is covered in our [Troubleshooting Guide](https://github.com/EDCD/EDMarketConnector/wiki/Troubleshooting).** It might be something with a known work around, or where a third party (such as EDSM) is causing logging that is harmless.
-
 **Please complete the following information:**
- - Version: [e.g. 4.0.6 - See 'Help > About E:D Market Connector'.  If running from source using git then please paste the output of `git log --decorate=full | head -1`]
+
+[//]: # (You can gather most of this information with the EDMC System Profiler)
+ 
+ - Version: [e.g. 5.10.4+39af6c34`]
  - Game Version: [e.g. 'Live' or 'Odyssey']
  - OS: [e.g. Windows 10, Linux Debian 10.6, etc.]
  - OS Locale: [e.g. English, French, Serbian...]
  - If applicable: Browser [e.g. chrome, safari]
- - Please attach **BOTH** log files, by dragging and dropping them into this input:
-     1. `%TEMP%\EDMarketConnector.log` from *immediately* after the bug occurs (re-running the application overwrites this file).
-     1. `%TEMP%\EDMarketConnector\EDMarketConnector-debug.log`.  See [Debug Log File](https://github.com/EDCD/EDMarketConnector/wiki/Troubleshooting#debug-log-files).  NB: If you don't have this log file then you're not running the latest version of the application and should update first to see if we already fixed the bug you're reporting.
 
 **Describe the bug**
-A clear and concise description of what the bug is.
+
+[//]: # (A clear and concise description of what the bug is.)
 
 **To Reproduce**
 Steps to reproduce the behavior:
@@ -32,10 +29,24 @@ Steps to reproduce the behavior:
 4. See error
 
 **Expected behavior**
-A clear and concise description of what you expected to happen.
+
+[//]: # (A clear and concise description of what you expected to happen.)
 
 **Screenshots**
-If applicable, add screenshots to help explain your problem.
+
+[//]: # (If applicable, add screenshots to help explain your problem.)
 
 **Additional context**
-Add any other context about the problem here.
+
+**Please Confirm the Following...**
+
+[//]: # (Add any other context about the problem here.)
+- [ ] I have checked the [Known Issues](https://github.com/EDCD/EDMarketConnector/issues/618) list to ensure this is not a duplicate
+- [ ] I have checked the [Troubleshooting Guide](https://github.com/EDCD/EDMarketConnector/wiki/Troubleshooting) to check for known workarounds
+
+**Logs**
+Please attach both the EDMarketConnector.log and EDMarketConnector-debug.log if available.
+
+You can find these logs at `%TEMP%\EDMarketConnector.log` and `%TEMP%\EDMarketConnector\EDMarketConnector-debug.log`
+
+See [Debug Log File](https://github.com/EDCD/EDMarketConnector/wiki/Troubleshooting#debug-log-files) for information on the Debug Log files

.gitignore

@@ -56,3 +56,4 @@ pylint.txt
 
 # Ignore Submodule data directory
 coriolis-data/
+FDevIDs/

.gitmodules

@@ -1,6 +1,3 @@
 [submodule "coriolis-data"]
 	path = coriolis-data
 	url = https://github.com/EDCD/coriolis-data.git
-[submodule "FDevIDs"]
-	path = FDevIDs
-	url = https://github.com/EDCD/FDevIDs.git

EDMC.py

@@ -42,7 +42,7 @@ import stats
 from commodity import COMMODITY_DEFAULT
 from config import appcmdname, appversion, config
 from monitor import monitor
-from update import EDMCVersion, Updater
+from update import EDMCVersion, Updater, check_for_fdev_updates
 
 sys.path.append(config.internal_plugin_dir)
 # This import must be after the sys.path.append.
@@ -497,6 +497,10 @@ def main():  # noqa: C901, CCR001
 
 
 if __name__ == '__main__':
-    main()
+    try:
+        check_for_fdev_updates(silent=True)
+        main()
+    except KeyboardInterrupt:
+        logger.info("Ctrl+C Detected, Attempting Clean Shutdown")
     logger.debug('Exiting')
     sys.exit(EXIT_SUCCESS)

EDMCSystemProfiler.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+"""
+EDMCSystemProfiler.py - GUI or Command-Line Tool to Print Diagnostic Information about EDMC.
+
+Copyright (c) EDCD, All Rights Reserved
+Licensed under the GNU General Public License.
+See LICENSE file.
+"""
+import argparse
+import locale
+import webbrowser
+import platform
+import sys
+from os import chdir, environ, path
+import pathlib
+import logging
+from journal_lock import JournalLock
+
+if getattr(sys, "frozen", False):
+    # Under py2exe sys.path[0] is the executable name
+    if sys.platform == "win32":
+        chdir(path.dirname(sys.path[0]))
+        # Allow executable to be invoked from any cwd
+        environ["TCL_LIBRARY"] = path.join(path.dirname(sys.path[0]), "lib", "tcl")
+        environ["TK_LIBRARY"] = path.join(path.dirname(sys.path[0]), "lib", "tk")
+
+else:
+    # We still want to *try* to have CWD be where the main script is, even if
+    # not frozen.
+    chdir(pathlib.Path(__file__).parent)
+
+import config
+from config import appversion, appname
+import tkinter as tk
+from tkinter import ttk
+from tkinter import messagebox
+from monitor import monitor
+from EDMCLogging import get_main_logger
+
+
+def get_sys_report(config: config.AbstractConfig) -> str:
+    """Gather system information about Elite, the Host Computer, and EDMC."""
+    # Calculate Requested Information
+    plt = platform.uname()
+    locale.setlocale(locale.LC_ALL, "")
+    lcl = locale.getlocale()
+    monitor.currentdir = config.get_str(
+        "journaldir", default=config.default_journal_dir
+    )
+    if not monitor.currentdir:
+        monitor.currentdir = config.default_journal_dir
+    try:
+        logfile = monitor.journal_newest_filename(monitor.currentdir)
+        if logfile is None:
+            raise ValueError("None from monitor.journal_newest_filename")
+
+        with open(logfile, "rb", 0) as loghandle:
+            for line in loghandle:
+                try:
+                    monitor.parse_entry(line)
+                except Exception as e:
+                    exception_type = e.__class__.__name__
+                    monitor.state["GameVersion"] = (
+                        exception_type
+                        if not monitor.state["GameVersion"]
+                        else monitor.state["GameVersion"]
+                    )
+                    monitor.state["GameBuild"] = (
+                        exception_type
+                        if not monitor.state["GameBuild"]
+                        else monitor.state["GameBuild"]
+                    )
+                    monitor.state["Odyssey"] = (
+                        exception_type
+                        if not monitor.state["Odyssey"]
+                        else monitor.state["Odyssey"]
+                    )
+    except Exception as e:
+        exception_type = e.__class__.__name__
+        monitor.state["GameVersion"] = exception_type
+        monitor.state["GameBuild"] = exception_type
+        monitor.state["Odyssey"] = exception_type
+
+    journal_lock = JournalLock()
+    lockable = journal_lock.open_journal_dir_lockfile()
+
+    report = f"EDMC Version: \n - {appversion()}\n\n"
+    report += "OS Details:\n"
+    report += f"- Operating System: {plt.system} {plt.release}\n"
+    report += f"- Version: {plt.version}\n"
+    report += f"- Machine: {plt.machine}\n"
+    report += f"- Python Version: {platform.python_version()}\n"
+    report += "\nEnvironment Details\n"
+    report += f"- Detected Locale: {lcl[0]}\n"
+    report += f"- Detected Encoding: {lcl[1]}\n"
+    report += f"- Journal Directory: {monitor.currentdir}\n"
+    report += f"- Game Version: {monitor.state['GameVersion']}\n"
+    report += f"- Game Build: {monitor.state['GameBuild']}\n"
+    report += f"- Using Odyssey: {monitor.state['Odyssey']}\n"
+    report += f"- Journal Dir Lockable: {lockable}\n"
+    return report
+
+
+def copy_sys_report(root: tk.Tk, report: str) -> None:
+    """Copy the system info to the keyboard."""
+    root.clipboard_clear()
+    root.clipboard_append(report)
+    messagebox.showinfo("System Profiler", "System Report copied to Clipboard")
+
+
+def main() -> None:
+    """Entry Point for the System Profiler."""
+    # Now Let's Begin
+    root: tk.Tk = tk.Tk()
+    root.withdraw()  # Hide the window initially to calculate the dimensions
+    try:
+        icon_image = tk.PhotoImage(
+            file=path.join(cur_config.respath_path, "io.edcd.EDMarketConnector.png")
+        )
+
+        root.iconphoto(True, icon_image)
+    except tk.TclError:
+        root.iconbitmap(path.join(cur_config.respath_path, "EDMarketConnector.ico"))
+
+    sys_report = get_sys_report(cur_config)
+
+    # Set up styling
+    style = ttk.Style(root)
+    style.configure("Title.TLabel", font=("Helvetica", 10, "bold"), foreground="#333")
+    style.configure("Subtitle.TLabel", font=("Helvetica", 8), foreground="#555")
+    style.configure("Details.TLabel", font=("Helvetica", 8), foreground="#222")
+
+    # Build UI
+    title_lbl = ttk.Label(
+        root, text="EDMarketConnector System Profiler", style="Title.TLabel"
+    )
+    title_lbl.grid(row=0, column=0, padx=20, pady=10)
+
+    system_details_lbl = ttk.Label(
+        root, text="System Details:", style="Subtitle.TLabel"
+    )
+    system_details_lbl.grid(row=1, column=0, padx=20, pady=0, sticky="w")
+
+    details_lbl = ttk.Label(
+        root, text=sys_report, style="Details.TLabel", justify="left"
+    )
+    details_lbl.grid(row=2, column=0, padx=20, pady=5, sticky="w")
+
+    # Buttons
+    sys_report_btn = ttk.Button(
+        root,
+        text="Copy System Report",
+        command=lambda: copy_sys_report(root, sys_report),
+    )
+    sys_report_btn.grid(row=3, column=0, padx=20, pady=10, sticky="w")
+
+    github_btn = ttk.Button(
+        root,
+        text="Open GitHub Bug Report",
+        command=lambda: webbrowser.open(
+            "https://github.com/EDCD/EDMarketConnector/issues/new?assignees="
+            "&labels=bug%2C+unconfirmed&projects=&template=bug_report.md&title="
+        ),
+    )
+    github_btn.grid(row=3, column=0, padx=20, pady=10, sticky="e")
+
+    # Update and get window dimensions
+    root.update()
+    width = root.winfo_reqwidth() + 20
+    height = root.winfo_reqheight() + 20
+
+    # Set window size and show
+    root.geometry(f"{width}x{height}")
+    root.title("EDMarketConnector")
+    root.deiconify()
+    root.resizable(False, False)
+
+    root.mainloop()
+
+
+if __name__ == "__main__":
+    # Args: Only work if not frozen
+    parser = argparse.ArgumentParser(
+        prog=appname,
+        description="Prints diagnostic and debugging information about the current EDMC configuration.",
+    )
+    parser.add_argument(
+        "--out-console",
+        help="write the system information to the console",
+        action="store_true",
+    )
+    args = parser.parse_args()
+
+    # Suppress Logger
+    logger = get_main_logger()
+    logger.setLevel(logging.CRITICAL)
+    if getattr(sys, "frozen", False):
+        sys.stderr._error = "inhibit log creation"  # type: ignore
+
+    cur_config = config.get_config()
+    if args.out_console:
+        sys_report = get_sys_report(cur_config)
+        print(sys_report)
+        sys.exit(0)
+
+    main()

EDMarketConnector.py

@@ -620,6 +620,7 @@ class AppWindow:
         self.help_menu.add_command(command=lambda: not self.HelpAbout.showing and self.HelpAbout(self.w))
         logfile_loc = pathlib.Path(tempfile.gettempdir()) / appname
         self.help_menu.add_command(command=lambda: prefs.open_folder(logfile_loc))  # Open Log Folder
+        self.help_menu.add_command(command=prefs.help_open_system_profiler)  # Open Log Folde
 
         self.menubar.add_cascade(menu=self.help_menu)
         if sys.platform == 'win32':
@@ -878,6 +879,7 @@ class AppWindow:
         self.help_menu.entryconfigure(5, label=tr.tl('Check for Updates...'))  # LANG: Help > Check for Updates...
         self.help_menu.entryconfigure(6, label=tr.tl("About {APP}").format(APP=applongname))  # LANG: Help > About App
         self.help_menu.entryconfigure(7, label=tr.tl('Open Log Folder'))  # LANG: Help > Open Log Folder
+        self.help_menu.entryconfigure(8, label=tr.tl('Open System Profiler'))  # LANG: Help > Open System Profiler
 
         # Edit menu
         self.edit_menu.entryconfigure(0, label=tr.tl('Copy'))  # LANG: Label for 'Copy' as in 'Copy and Paste'
@@ -2303,35 +2305,6 @@ sys.path: {sys.path}'''
             )
             config.set('plugins_not_py3_last', int(time()))
 
-    def check_fdev_ids():
-        """Display message about missing FDEVID files."""
-        fdev_files = {'commodity.csv', 'rare_commodity.csv'}
-        for file in fdev_files:
-            fdevid_file = pathlib.Path(config.respath_path / 'FDevIDs' / file)
-            if fdevid_file.is_file():
-                continue
-            # LANG: Popup-text about missing FDEVID Files
-            popup_text = tr.tl(
-                "FDevID Files not found! Some functionality regarding commodities "
-                r"may be disabled.\r\n\r\n Do you want to open the Wiki page on "
-                "how to set up submodules?"
-            )
-            # And now we do need these to be actual \r\n
-            popup_text = popup_text.replace('\\n', '\n')
-            popup_text = popup_text.replace('\\r', '\r')
-
-            openwikipage = tk.messagebox.askquestion(
-                # LANG: Popup window title for missing FDEVID files
-                tr.tl('FDevIDs: Missing Commodity Files'),
-                popup_text
-            )
-            if openwikipage == "yes":
-                webbrowser.open(
-                    "https://github.com/EDCD/EDMarketConnector/wiki/Running-from-source"
-                    "#obtain-a-copy-of-the-application-source"
-                )
-            break
-
     # UI Transparency
     ui_transparency = config.get_int('ui_transparency')
     if ui_transparency == 0:
@@ -2344,8 +2317,6 @@ sys.path: {sys.path}'''
     root.after(1, messagebox_not_py3)
     # Show warning popup for killswitches matching current version
     root.after(2, show_killswitch_poppup, root)
-    # Check for FDEV IDs
-    root.after(3, check_fdev_ids)
     # Start the main event loop
     try:
         root.mainloop()

FDevIDs

@@ -1 +0,0 @@
-Subproject commit 9b3f40612017b43a8b826017e1e2befebd9074f2

L10n/en.template

@@ -93,6 +93,9 @@
 /* EDMarketConnector.py: Help > Check for Updates...; In files: EDMarketConnector.py:930; EDMarketConnector.py:958; */
 "Check for Updates..." = "Check for Updates...";
 
+/* EDMarketConnector.py: Help > Open System Profiler; In files: EDMarketConnector.py:888; */
+"Open System Profiler" = "Open System Profiler";
+
 /* EDMarketConnector.py: File > Save Raw Data...; In files: EDMarketConnector.py:931; EDMarketConnector.py:948; */
 "Save Raw Data..." = "Save Raw Data...";
 
@@ -213,12 +216,6 @@
 /* EDMarketConnector.py: Popup-text about 'active' plugins without Python 3.x support; In files: EDMarketConnector.py:2253:2259; */
 "One or more of your enabled plugins do not yet have support for Python 3.x. Please see the list on the '{PLUGINS}' tab of '{FILE}' > '{SETTINGS}'. You should check if there is an updated version available, else alert the developer that they need to update the code for Python 3.x.\r\n\r\nYou can disable a plugin by renaming its folder to have '{DISABLED}' on the end of the name." = "One or more of your enabled plugins do not yet have support for Python 3.x. Please see the list on the '{PLUGINS}' tab of '{FILE}' > '{SETTINGS}'. You should check if there is an updated version available, else alert the developer that they need to update the code for Python 3.x.\r\n\r\nYou can disable a plugin by renaming its folder to have '{DISABLED}' on the end of the name.";
 
-/* EDMarketConnector.py: Popup-text about missing FDEVID Files; In files: EDMarketConnector.py:2329; */
-"FDevID Files not found! Some functionality regarding commodities may be disabled.\r\n\r\n Do you want to open the Wiki page on how to set up submodules?" = "FDevID Files not found! Some functionality regarding commodities may be disabled.\r\n\r\n Do you want to open the Wiki page on how to set up submodules?";
-
-/* EDMarketConnector.py: Popup window title for missing FDEVID files; In files: EDMarketConnector.py:2340; */
-"FDevIDs: Missing Commodity Files" = "FDevIDs: Missing Commodity Files";
-
 /* EDMarketConnector.py: Settings > Plugins tab; prefs.py: Label on Settings > Plugins tab; In files: EDMarketConnector.py:2263; prefs.py:986; */
 "Plugins" = "Plugins";
 

build.py

@@ -21,6 +21,7 @@ from config import (
     _static_appversion,
     update_interval
 )
+from update import check_for_fdev_updates
 
 
 def iss_build(template_path: str, output_file: str) -> None:
@@ -175,10 +176,19 @@ def build() -> None:
         ],
     }
 
+    checker_config: dict = {
+        "dest_base": "EDMCSystemProfiler",
+        "script": "EDMCSystemProfiler.py",
+        "icon_resources": [(0, f"{appname}.ico")],
+        "other_resources": [
+            (24, 1, pathlib.Path(f"resources/{appname}.manifest").read_text(encoding="UTF8"))
+        ],
+    }
+
     try:
         py2exe.freeze(
             version_info=version_info,
-            windows=[windows_config],
+            windows=[windows_config, checker_config],
             console=[console_config],
             data_files=data_files,
             options=options,
@@ -198,4 +208,5 @@ def build() -> None:
 
 
 if __name__ == "__main__":
+    check_for_fdev_updates()
     build()

plugins/edsm.py

@@ -491,6 +491,20 @@ def credentials(cmdr: str) -> tuple[str, str] | None:
     edsm_usernames = config.get_list('edsm_usernames')
     edsm_apikeys = config.get_list('edsm_apikeys')
 
+    if not edsm_usernames:  # https://github.com/EDCD/EDMarketConnector/issues/2232
+        edsm_usernames = ["" for _ in range(len(cmdrs))]
+    else:  # Check for Mismatched Length - fill with null values.
+        if len(edsm_usernames) < len(cmdrs):
+            edsm_usernames.extend(["" for _ in range(len(cmdrs) - len(edsm_usernames))])
+    config.set('edsm_usernames', edsm_usernames)
+
+    if not edsm_apikeys:
+        edsm_apikeys = ["" for _ in range(len(cmdrs))]
+    else:  # Check for Mismatched Length - fill with null values.
+        if len(edsm_apikeys) < len(cmdrs):
+            edsm_apikeys.extend(["" for _ in range(len(cmdrs) - len(edsm_apikeys))])
+    config.set('edsm_apikeys', edsm_apikeys)
+
     if cmdr in cmdrs and len(cmdrs) == len(edsm_usernames) == len(edsm_apikeys):
         idx = cmdrs.index(cmdr)
         if idx < len(edsm_usernames) and idx < len(edsm_apikeys):

prefs.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import contextlib
 import logging
 import pathlib
+import subprocess
 import sys
 import tempfile
 import tkinter as tk
@@ -56,6 +57,16 @@ def open_folder(file: pathlib.Path) -> None:
         system(f'xdg-open "{file}"')
 
 
+def help_open_system_profiler() -> None:
+    """Open the EDMC System Profiler."""
+    profiler_path = pathlib.Path(config.respath_path)
+    if getattr(sys, 'frozen', False):
+        profiler_path /= 'EDMCSystemProfiler.exe'
+        subprocess.run(profiler_path)
+    else:
+        subprocess.run(['python', "EDMCSystemProfiler.py"], shell=True)
+
+
 class PrefsVersion:
     """
     PrefsVersion contains versioned preferences.

update.py

@@ -7,6 +7,7 @@ See LICENSE file.
 """
 from __future__ import annotations
 
+import pathlib
 import sys
 import threading
 from traceback import print_exc
@@ -25,6 +26,38 @@ if TYPE_CHECKING:
 logger = get_main_logger()
 
 
+def check_for_fdev_updates(silent: bool = False) -> None:  # noqa: CCR001
+    """Check for and download FDEV ID file updates."""
+    files_urls = [
+        ('commodity.csv', 'https://raw.githubusercontent.com/EDCD/FDevIDs/master/commodity.csv'),
+        ('rare_commodity.csv', 'https://raw.githubusercontent.com/EDCD/FDevIDs/master/rare_commodity.csv')
+    ]
+
+    for file, url in files_urls:
+        fdevid_file = pathlib.Path(config.respath_path / 'FDevIDs' / file)
+        fdevid_file.parent.mkdir(parents=True, exist_ok=True)
+        try:
+            with open(fdevid_file, newline='', encoding='utf-8') as f:
+                local_content = f.read()
+        except FileNotFoundError:
+            local_content = None
+
+        response = requests.get(url)
+        if response.status_code != 200:
+            if not silent:
+                logger.error(f'Failed to download {file}! Unable to continue.')
+            continue
+
+        if local_content == response.text:
+            if not silent:
+                logger.info(f'FDEV ID file {file} already up to date.')
+        else:
+            if not silent:
+                logger.info(f'FDEV ID file {file} not up to date. Downloading...')
+            with open(fdevid_file, 'w', newline='', encoding='utf-8') as csvfile:
+                csvfile.write(response.text)
+
+
 class EDMCVersion:
     """
     Hold all the information about an EDMC version.
@@ -135,6 +168,8 @@ class Updater:
         elif sys.platform == 'win32' and self.updater:
             self.updater.win_sparkle_check_update_with_ui()
 
+        check_for_fdev_updates()
+
     def check_appcast(self) -> EDMCVersion | None:
         """
         Manually (no Sparkle or WinSparkle) check the update_feed appcast file.