Merge branch 'release-5.4.1' into releases

.github/workflows/pr-checks.yml

@@ -52,10 +52,10 @@ jobs:
     ####################################################################
     # Get Python set up
     ####################################################################
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
       with:
-        python-version: "3.10"
+        python-version-file: '.python-version'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/push-checks.yml

@@ -20,10 +20,10 @@ jobs:
     - uses: actions/checkout@v3
       with:
         fetch-depth: 0
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v3
+    - name: Set up Python
+      uses: actions/setup-python@v4
       with:
-        python-version: "3.10"
+        python-version-file: '.python-version'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/windows-build.yml

@@ -20,9 +20,9 @@ jobs:
         with:
           submodules: true
 
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
-          python-version: "3.10.4"
+          python-version-file: '.python-version'
           architecture: "x86"
 
       - name: Install python tools

.python-version

@@ -1 +1 @@
-3.10.4
+3.10.5

ChangeLog.md

@@ -9,7 +9,7 @@ produce the Windows executables and installer.
 
 ---
 
-* We now test against, and package with, Python 3.10.4.
+* We now test against, and package with, Python 3.10.5.
 
   **As a consequence of this we no longer support Windows 7.  
   This is due to
@@ -27,6 +27,30 @@ produce the Windows executables and installer.
 
 ---
 
+Release 5.4.1
+===
+
+* We now test against, and package with, Python 3.10.5.
+* If for any reason `EDMarketConnector.exe` fails to shutdown and exit when
+  asked to by the upgrade process this should no longer result in a spontaneous
+  system reboot.  Closes [#1492](https://github.com/EDCD/EDMarketConnector/issues/1492).
+
+  A manual reboot will still be required to complete the EDMarketConnector
+  upgrade process and we make no guarantees about the stability of the
+  application until this is done.
+* The new EDDN `fsssignaldiscovered/1` schema has been implemented.
+* EDSM trace level logging will no longer log API credentials unless explicitly
+  asked to, separately from other EDSM API trace logging.
+
+Bug Fixes
+---
+
+* EDDN: Ensure we always remove all `_Localised` suffix keys in data.  This
+  was missed in some recent new schemas and turned out to be an issue for at
+  least `approachsettlement/1`.
+
+---
+
 Release 5.4.0
 ===
 

config/__init__.py

@@ -52,7 +52,7 @@ appcmdname = 'EDMC'
 # <https://semver.org/#semantic-versioning-specification-semver>
 # Major.Minor.Patch(-prerelease)(+buildmetadata)
 # NB: Do *not* import this, use the functions appversion() and appversion_nobuild()
-_static_appversion = '5.4.0'
+_static_appversion = '5.4.1'
 _cached_version: Optional[semantic_version.Version] = None
 copyright = '© 2015-2019 Jonathan Harris, 2020-2022 EDCD'
 

docs/Releasing.md

@@ -433,13 +433,13 @@ If you are making a pre-release then:
 
 When changing the Python version (Major.Minor.Patch) used:
 
-1. Change the contents of `.python-version` so that pyenv notices.
+1. Change the contents of `.python-version` so that pyenv notices.  All of
+  the GitHub workflows now reference this via the `setup-python`
+  `python-version-file` directive.
 
 1. Any version change:
 
-   1. `.github/workflows/windows-build.yml` needs updating to have the GitHub
-   based build use the correct version.
-   2. `ChangeLog.md` - The `We now test against, and package with, Python
+   1. `ChangeLog.md` - The `We now test against, and package with, Python
        M.m.P.` line.
 
 1. Major or Minor level changes:
@@ -449,5 +449,3 @@ When changing the Python version (Major.Minor.Patch) used:
        pythonXX.dll file.
     3. `.pre-commit-config.yaml` will need the `default_language_version`
        section updated to the appropriate version.
-    4. All `.github/workflows/` files will need to be citing the correct
-       version in any `uses: actions/setup-python` section.

plugins/eddn.py

@@ -152,6 +152,7 @@ class EDDN:
         self.session.headers['User-Agent'] = user_agent
         self.replayfile: Optional[TextIO] = None  # For delayed messages
         self.replaylog: List[str] = []
+        self.fss_signals: List[Mapping[str, Any]] = []
 
         if config.eddn_url is not None:
             self.eddn_url = config.eddn_url
@@ -1124,6 +1125,9 @@ class EDDN:
             del entry['horizons']
 
         # END WORKAROUND
+
+        # In case Frontier ever add any
+        entry = filter_localised(entry)
         #######################################################################
 
         #######################################################################
@@ -1184,6 +1188,13 @@ class EDDN:
         # WORKAROUND END
         #######################################################################
 
+        #######################################################################
+        # Elisions
+        #######################################################################
+        # In case Frontier ever add any
+        entry = filter_localised(entry)
+        #######################################################################
+
         #######################################################################
         # Augmentations
         #######################################################################
@@ -1227,6 +1238,13 @@ class EDDN:
         #   "event": "FSSAllBodiesFound",
         #   "timestamp": "2022-02-09T18:15:14Z"
         # }
+        #######################################################################
+        # Elisions
+        #######################################################################
+        # In case Frontier ever add any
+        entry = filter_localised(entry)
+        #######################################################################
+
         #######################################################################
         # Augmentations
         #######################################################################
@@ -1307,6 +1325,106 @@ class EDDN:
         this.eddn.export_journal_entry(cmdr, entry, msg)
         return None
 
+    def enqueue_journal_fsssignaldiscovered(self, entry: MutableMapping[str, Any]) -> None:
+        """
+        Queue up an FSSSignalDiscovered journal event for later sending.
+
+        :param entry: the journal entry to batch
+        """
+        if entry is None or entry == "":
+            logger.warning(f"Supplied event was empty: {entry!r}")
+            return
+
+        logger.trace_if("plugin.eddn.fsssignaldiscovered", f"Appending FSSSignalDiscovered entry:\n"
+                        f" {json.dumps(entry)}")
+        self.fss_signals.append(entry)
+
+    def export_journal_fsssignaldiscovered(
+        self, cmdr: str, system_name: str, system_starpos: list, is_beta: bool, entry: MutableMapping[str, Any]
+    ) -> Optional[str]:
+        """
+        Send an FSSSignalDiscovered message to EDDN on the correct schema.
+
+        :param cmdr: the commander under which this upload is made
+        :param system_name: Name of current star system
+        :param system_starpos: Coordinates of current star system
+        :param is_beta: whether or not we are in beta mode
+        :param entry: the non-FSSSignalDiscovered journal entry that triggered this batch send
+        """
+        logger.trace_if("plugin.eddn.fsssignaldiscovered", f"This other event is: {json.dumps(entry)}")
+        #######################################################################
+        # Location cross-check and augmentation setup
+        #######################################################################
+        # Determine if this is Horizons order or Odyssey order
+        if entry['event'] in ('Location', 'FSDJump', 'CarrierJump'):
+            # Odyssey order, use this new event's data for cross-check
+            aug_systemaddress = entry['SystemAddress']
+            aug_starsystem = entry['StarSystem']
+            aug_starpos = entry['StarPos']
+
+        else:
+            # Horizons order, so use tracked data for cross-check
+            aug_systemaddress = this.systemaddress
+            aug_starsystem = system_name
+            aug_starpos = system_starpos
+
+        if aug_systemaddress != self.fss_signals[0]['SystemAddress']:
+            logger.warning("First signal's SystemAddress doesn't match current location: "
+                           f"{self.fss_signals[0]['SystemAddress']} != {aug_systemaddress}")
+            self.fss_signals = []
+            return 'Wrong System! Missed jump ?'
+        #######################################################################
+
+        # Build basis of message
+        msg: Dict = {
+            '$schemaRef': f'https://eddn.edcd.io/schemas/fsssignaldiscovered/1{"/test" if is_beta else ""}',
+            'message': {
+                "event": "FSSSignalDiscovered",
+                "timestamp": self.fss_signals[0]['timestamp'],
+                "SystemAddress": aug_systemaddress,
+                "StarSystem": aug_starsystem,
+                "StarPos": aug_starpos,
+                "signals": [],
+            }
+        }
+
+        # Now add the signals, checking each is for the correct system, dropping
+        # any that aren't, and applying necessary elisions.
+        for s in self.fss_signals:
+            if s['SystemAddress'] != aug_systemaddress:
+                logger.warning("Signal's SystemAddress not current system, dropping: "
+                               f"{s['SystemAddress']} != {aug_systemaddress}")
+                continue
+
+            # Remove any _Localised keys (would only be in a USS signal)
+            s = filter_localised(s)
+
+            # Drop Mission USS signals.
+            if "USSType" in s and s["USSType"] == "$USS_Type_MissionTarget;":
+                logger.trace_if("plugin.eddn.fsssignaldiscovered", "USSType is $USS_Type_MissionTarget;, dropping")
+                continue
+
+            # Remove any key/values that shouldn't be there per signal
+            s.pop('event', None)
+            s.pop('horizons', None)
+            s.pop('odyssey', None)
+            s.pop('TimeRemaining', None)
+            s.pop('SystemAddress', None)
+
+            msg['message']['signals'].append(s)
+
+        # `horizons` and `odyssey` augmentations
+        msg['message']['horizons'] = entry['horizons']
+        msg['message']['odyssey'] = entry['odyssey']
+
+        logger.trace_if("plugin.eddn.fsssignaldiscovered", f"FSSSignalDiscovered batch is {json.dumps(msg)}")
+
+        # Fake an 'entry' as it's only there for some "should we send replay?" checks in the called function.
+        this.eddn.export_journal_entry(cmdr, {'event': 'send_fsssignaldiscovered'}, msg)
+        self.fss_signals = []
+
+        return None
+
     def canonicalise(self, item: str) -> str:
         """
         Canonicalise the given commodity name.
@@ -1557,6 +1675,18 @@ def journal_entry(  # noqa: C901, CCR001
     this.horizons = entry['horizons'] = state['Horizons']
     this.odyssey = entry['odyssey'] = state['Odyssey']
 
+    # Simple queue: send batched FSSSignalDiscovered once a non-FSSSignalDiscovered is observed
+    if event_name != 'fsssignaldiscovered' and this.eddn.fss_signals:
+        # We can't return here, we still might need to otherwise process this event,
+        # so errors will never be shown to the user.
+        this.eddn.export_journal_fsssignaldiscovered(
+            cmdr,
+            system,
+            state['StarPos'],
+            is_beta,
+            entry
+        )
+
     # Track location
     if event_name == 'supercruiseexit':
         # For any orbital station we have no way of determining the body
@@ -1649,13 +1779,8 @@ def journal_entry(  # noqa: C901, CCR001
                 entry
             )
 
-        # NB: If adding FSSSignalDiscovered these absolutely come in at login
-        #     time **BEFORE** the `Location` event, so we won't yet know things
-        #     like SystemName, or StarPos.
-        #     We can either have the "now send the batch" code add such (but
-        #     that has corner cases around changing systems in the meantime),
-        #     drop those events, or if the schema allows, send without those
-        #     augmentations.
+        elif event_name == 'fsssignaldiscovered':
+            this.eddn.enqueue_journal_fsssignaldiscovered(entry)
 
         elif event_name == 'fssallbodiesfound':
             return this.eddn.export_journal_fssallbodiesfound(

plugins/edsm.py

@@ -62,6 +62,7 @@ DISCARDED_EVENTS_SLEEP = 10
 
 # trace-if events
 CMDR_EVENTS = 'plugin.edsm.cmdr-events'
+CMDR_CREDS = 'plugin.edsm.cmdr-credentials'
 
 
 class This:
@@ -386,7 +387,7 @@ def credentials(cmdr: str) -> Optional[Tuple[str, str]]:
     :param cmdr: The commander to get credentials for
     :return: The credentials, or None
     """
-    logger.trace_if(CMDR_EVENTS, f'{cmdr=}')
+    logger.trace_if(CMDR_CREDS, f'{cmdr=}')
 
     # Credentials for cmdr
     if not cmdr:
@@ -405,12 +406,12 @@ def credentials(cmdr: str) -> Optional[Tuple[str, str]]:
         if idx >= len(edsm_usernames) or idx >= len(edsm_apikeys):
             return None
 
-        logger.trace_if(CMDR_EVENTS, f'{cmdr=}: returning ({edsm_usernames[idx]=}, {edsm_apikeys[idx]=})')
+        logger.trace_if(CMDR_CREDS, f'{cmdr=}: returning ({edsm_usernames[idx]=}, {edsm_apikeys[idx]=})')
 
         return (edsm_usernames[idx], edsm_apikeys[idx])
 
     else:
-        logger.trace_if(CMDR_EVENTS, f'{cmdr=}: returning None')
+        logger.trace_if(CMDR_CREDS, f'{cmdr=}: returning None')
         return None
 
 

requirements-dev.txt

@@ -18,11 +18,11 @@ flake8-docstrings==1.6.0
 isort==5.10.1
 flake8-isort==4.1.1
 flake8-json==21.7.0
-flake8-noqa==1.2.2
+flake8-noqa==1.2.5
 flake8-polyfill==1.0.2
 flake8-use-fstring==1.3
 
-mypy==0.960
+mypy==0.961
 pep8-naming==0.13.0
 safety==1.10.3
 types-requests==2.27.30

requirements.txt

@@ -1,6 +1,6 @@
-certifi==2022.5.18.1
-requests==2.27.1
-watchdog==2.1.8
+certifi==2022.6.15
+requests==2.28.0
+watchdog==2.1.9
 # Commented out because this doesn't package well with py2exe
 infi.systray==0.1.12; sys_platform == 'win32'
 # argh==0.26.2 watchdog dep

wix/template.wxs

@@ -44,6 +44,11 @@
                             Name="InstallLocation"
                             Type="raw" />
         </Property>
+
+        <!-- Attempt to **NEVER** cause a reboot, e.g. if EDMarketConnector.exe
+             doesn't exit, and thus the installer can't properly replace it. -->
+        <Property Id="REBOOT" Value="ReallySuppress" />
+
         <CustomAction Id="SetINSTALLDIR" Property="INSTALLDIR" Value="[ARPINSTALLLOCATION]" />
         <InstallUISequence>
             <Custom Action="SetINSTALLDIR" After="AppSearch">