diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 019facb4..00000000 --- a/.coveragerc +++ /dev/null @@ -1,6 +0,0 @@ -[run] -omit = - # The tests themselves - tests/* - # Any venv files - venv/* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 20a49627..6737328d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,7 +21,7 @@ repos: # - id: autopep8 ### # flake8 --show-source -### - repo: https://gitlab.com/pycqa/flake8 +### - repo: https://github.com/PyCQA/flake8 ### rev: '' ### hooks: ### - id: flake8 diff --git a/.python-version b/.python-version index 1281604a..36435ac6 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.10.7 +3.10.8 diff --git a/ChangeLog.md b/ChangeLog.md index efb069cc..283a09be 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -9,7 +9,7 @@ produce the Windows executables and installer. --- -* We now test against, and package with, Python 3.10.7. +* We now test against, and package with, Python 3.10.8. **As a consequence of this we no longer support Windows 7. This is due to @@ -27,6 +27,136 @@ produce the Windows executables and installer. --- +Release 5.6.0 +=== + +Tha major reason for this release is to address the Live versus Legacy galaxy +split [coming in Update 14 of the game](https://www.elitedangerous.com/news/elite-dangerous-update-14-and-beyond-live-and-legacy-modes). +See the section "Update 14 and the Galaxy Split" below for how this might +impact you. + +Changes +--- + +* We now test against, and package with, Python 3.10.8. +* The code for sending data to EDDN has been reworked. This changes the + 'replay log' from utilising an internal array, backed by a flat file + (`replay.jsonl`), to an sqlite3 database. + + As a result: + 1. Any messages stored in the old `replay.jsonl` are converted at startup, + if that file is present, and then the file removed. + 2. All new messages are stored in this new sqlite3 queue before any attempt + is made to send them. An immediate attempt is then made to send any + message not affected by "Delay sending until docked". + 3. Sending of queued messages will be attempted every 5 minutes, unless + "Delay sending until docked" is active and the Cmdr is not docked in + their own ship. This is in case a message failed to send due to an issue + communicating with the EDDN Gateway. + 4. When you dock in your own ship an immediate attempt to send all queued + messages will be initiated. + 5. When processing queued messages the same 0.4-second inter-message delay + as with the old code has been implemented. This serves to not suddenly + flood the EDDN Gateway. If any message fails to send for Gateway reasons, + i.e. not a bad message, then this processing is abandoned to wait for + the next invocation. + + The 5-minute timer in point 3 differs from the old code, where almost any + new message sending attempt could initiate processing of the queue. At + application startup this delay is only 10 seconds. + + Currently, the feedback of "Sending data to EDDN..." in the UI status line + has been removed. + + **If you do not have "Delay sending until docked" active, then the only + messages that will be at all delayed will be where there was a communication + problem with the EDDN Gateway, or it otherwise indicated a problem other + than 'your message is bad'.** +* As a result of this EDDN rework this application now sends appropriate + `gameversion` and `gamebuild` strings in EDDN message headers. + The rework was necessary in order to enable this, in case of any queued + or delayed messages which did not contain this information in the legacy + `replay.jsonl` format. +* For EDSM there is a very unlikely set of circumstances that could, in theory + lead to some events not being sent. This is so as to safeguard against + sending a batch with a gameversion/build claimed that does not match for + *all* of the events in that batch. + + It would take a combination of "communications with EDSM are slow", more + events (the ones that would be lost), a game client crash, *and* starting + a new game client before the 'more events' are sent. + +Update 14 and the Galaxy Split +--- +Due to the galaxy split [announced by Frontier](https://www.elitedangerous.com/news/elite-dangerous-update-14-and-beyond-live-and-legacy-modes) +there are some changes to the major third-party websites and tools. + +* Inara [has chosen](https://inara.cz/elite/board-thread/7049/463292/#463292) + to only accept Live galaxy data on its API. + + This application will not even process Journal data for Inara after + 2022-11-29T09:00:00+00:00 *unless the `gameversion` indicates a Live client*. + This explicitly checks that the game's version is semantically equal to or + greater than '4.0.0'. + + If a Live client is *not* detected, then there is an INFO level logging + message "Inara only accepts Live galaxy data", which is also set as the main + UI status line. This message will repeat, at most, every 5 minutes. + + If you continue to play in the Legacy galaxy only then you probably want to + just disable the Inara plugin with the checkbox on Settings > Inara. +* All batches of events sent to EDSM will be tagged with a `gameversion`, in + a similar manner to the EDDN header. + + Ref: [EDSM api-journal-v1](https://www.edsm.net/en/api-journal-v1) +* All EDDN messages will now have appropriate `gameversion` and `gamebuild` + fields in the `header` as per + [EDDN/docs/Developers.md](https://github.com/EDCD/EDDN/blob/live/docs/Developers.md#gameversions-and-gamebuild). + + As a result of this you can expect third-party sites to choose to filter data + based on that. + + Look for announcements by individual sites/tools as to what they have chosen + to do. + +Known Bugs +--- +In testing if it had been broken at all due to 5.5.0 -> 5.6.0 changes it has +come to light that `EDMC.EXE -n`, to send data to EDDN, was already broken in +5.5.0. + +In addition, there is now some extra 'INFO' logging output which will be +produced by any invocation of `EDMC.EXE`. This might break third-party use of +it, e.g. [Trade Computer Extension Mk.II](https://forums.frontier.co.uk/threads/trade-computer-extension-mk-ii.223056/). +This will be fixed as soon as the dust settles from Update 14, with emphasis +being on ensuring the GUI `EDMarketConnector.exe` functions properly. + +Notes for EDDN Listeners +--- +* Where EDMC sourced data from the Journal files it will set `gameversion` + and `gamebuild` as per their values in `Fileheader` or `LoadGame`, whichever + was more recent (there are some events that occur between these). +* *If any message was already delayed such that it did not + have the EDDN header recorded, then the `gameversion` and `gamebuild` will + be empty strings*. In order to indicate this the `softwareName` will have + ` (legacy replay)` appended to it, e.g. `E:D Market Connector Connector + [Windows] (legacy replay)`. In general this indicates that the message was + queued up using a version of EDMC prior to this one. If you're only + interested in Live galaxy data then you might want to ignore such messages. +* Where EDMC sourced data from a CAPI endpoint, the resulting EDDN message + will have a `gameversion` of `CAPI-` set, e.g. `CAPI-market`. + **At this time it is not 100% certain which galaxy this data will be for, so + all listeners are advised to ignore/queue such data until this is clarified**. + + `gamebuild` will be an empty string for all CAPI-sourced data. + +Plugin Developers +--- +* There is a new flag in `state` passed to plugins, `IsDocked`. See PLUGINS.md + for details. + +--- + Release 5.5.0 === diff --git a/Contributing.md b/Contributing.md index 4d7fc9fc..29135804 100644 --- a/Contributing.md +++ b/Contributing.md @@ -244,6 +244,46 @@ handy if you want to step through the testing code to be sure of anything. Otherwise, see the [pytest documentation](https://docs.pytest.org/en/stable/contents.html). +### Test Coverage +As we work towards actually having tests for as much of the code as possible +it is useful to monitor the current test coverage. + +Running `pytest` will also produce the overall coverage report, see the +configured options in `pyproject.toml`. + +One issue you might run into is where there is code that only runs on one +platform. By default `pytest-cov`/`coverage` will count this code as not +tested when run on a different platform. We utilise the +`coverage-conditional-plugin` module so that `#pragma` comments can be used +to give hints to coverage about this. + +The pragmas are defined in the +`tool.coverage.coverage_conditional_plugin.rules` section of `pyproject.toml`, +e.g. + +```toml +[tool.coverage.coverage_conditional_plugin.rules] +sys-platform-win32 = "sys_platform != 'win32'" +... +``` +And are used as in: +```python +import sys + +if sys.platform == 'win32': # pragma: sys-platform-win32 + ... +else: # pragma: sys-platform-not-win32 + ... +``` +Note the inverted sense of the pragma definitions, as the comments cause +`coverage` to *not* consider that code block on this platform. + +As of 2022-10-02 and `coverage-conditional-plugin==0.7.0` there is no way to +signal that an entire file should be excluded from coverage reporting on the +current platform. See +[this GitHub issue comment](https://github.com/wemake-services/coverage-conditional-plugin/issues/2#issuecomment-1263918296) +. + --- ## Imports used only in core plugins diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 283c3bf5..041f7331 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -922,7 +922,7 @@ class AppWindow(object): return False # Ignore possibly missing shipyard info - elif (config.get_int('output') & config.OUT_MKT_EDDN) \ + elif (config.get_int('output') & config.OUT_EDDN_SEND_STATION_DATA) \ and not (data['lastStarport'].get('commodities') or data['lastStarport'].get('modules')): if not self.status['text']: # LANG: Status - Either no market or no modules data for station from Frontier CAPI diff --git a/L10n/en.template b/L10n/en.template index f6ebd8e1..8bbfad96 100644 --- a/L10n/en.template +++ b/L10n/en.template @@ -46,6 +46,9 @@ /* inara.py: Text for INARA API keys link ( goes to https://inara.cz/settings-api ); In files: inara.py:225; load.py:225; inara.py:234; */ "Inara credentials" = "Inara credentials"; +/* inara.py: The Inara API only accepts Live galaxy data, not Legacy galaxy data; In files: inara.py:383; inara.py:386; */ +"Inara only accepts Live galaxy data" = "Inara only accepts Live galaxy data"; + /* inara.py: INARA API returned some kind of error (error message will be contained in {MSG}); In files: inara.py:1316; inara.py:1328; load.py:1319; load.py:1331; inara.py:1587; inara.py:1600; */ "Error: Inara {MSG}" = "Error: Inara {MSG}"; @@ -744,4 +747,3 @@ /* stats.py: Status dialog title; In files: stats.py:422; */ "Ships" = "Ships"; - diff --git a/L10n/ja.strings b/L10n/ja.strings index 8656cfbd..531e9c6d 100644 --- a/L10n/ja.strings +++ b/L10n/ja.strings @@ -1,3 +1,6 @@ +/* inara.py: The Inara API only accepts Live galaxy data, not Legacy galaxy data; In files: inara.py:383; inara.py:386; */ +"Inara only accepts Live galaxy data" = "Inaraは現行の銀河データのみ受け付けます"; + /* Language name */ "!Language" = "日本語"; diff --git a/L10n/pt-BR.strings b/L10n/pt-BR.strings index 07c0e669..93e34f0c 100644 --- a/L10n/pt-BR.strings +++ b/L10n/pt-BR.strings @@ -1,3 +1,6 @@ +/* inara.py: The Inara API only accepts Live galaxy data, not Legacy galaxy data; In files: inara.py:383; inara.py:386; */ +"Inara only accepts Live galaxy data" = "Inara apenas aceita dados da versão Live."; + /* Language name */ "!Language" = "Português (Brasil)"; diff --git a/L10n/pt-PT.strings b/L10n/pt-PT.strings index 0ee7810f..20821465 100644 --- a/L10n/pt-PT.strings +++ b/L10n/pt-PT.strings @@ -1,3 +1,6 @@ +/* inara.py: The Inara API only accepts Live galaxy data, not Legacy galaxy data; In files: inara.py:383; inara.py:386; */ +"Inara only accepts Live galaxy data" = "A API Inara só aceita dados da versão Live"; + /* Language name */ "!Language" = "Português (Portugal)"; diff --git a/L10n/ru.strings b/L10n/ru.strings index 979f0367..582fdd21 100644 --- a/L10n/ru.strings +++ b/L10n/ru.strings @@ -1,3 +1,6 @@ +/* inara.py: The Inara API only accepts Live galaxy data, not Legacy galaxy data; In files: inara.py:383; inara.py:386; */ +"Inara only accepts Live galaxy data" = "Inara принимает только данные Live-версии"; + /* Language name */ "!Language" = "Русский"; diff --git a/L10n/sr-Latn-BA.strings b/L10n/sr-Latn-BA.strings index 9f94d6c3..bdc47cd8 100644 --- a/L10n/sr-Latn-BA.strings +++ b/L10n/sr-Latn-BA.strings @@ -1,3 +1,6 @@ +/* inara.py: The Inara API only accepts Live galaxy data, not Legacy galaxy data; In files: inara.py:383; inara.py:386; */ +"Inara only accepts Live galaxy data" = "Inara prihvata samo Live galaxy podatke"; + /* Language name */ "!Language" = "Srpski (Latinica, Bosna i Hercegovina)"; diff --git a/L10n/sr-Latn.strings b/L10n/sr-Latn.strings index 189f5192..ed3cee47 100644 --- a/L10n/sr-Latn.strings +++ b/L10n/sr-Latn.strings @@ -1,3 +1,6 @@ +/* inara.py: The Inara API only accepts Live galaxy data, not Legacy galaxy data; In files: inara.py:383; inara.py:386; */ +"Inara only accepts Live galaxy data" = "Inara prihvata samo \"žive\" podatke o galaksiji"; + /* Language name */ "!Language" = "Srpski (Latinica)"; diff --git a/PLUGINS.md b/PLUGINS.md index 1649b47e..e8d47a3e 100644 --- a/PLUGINS.md +++ b/PLUGINS.md @@ -617,6 +617,7 @@ Content of `state` (updated to the current journal entry): | `Modules` | `dict` | Currently fitted modules | | `NavRoute` | `dict` | Last plotted multi-hop route | | `ModuleInfo` | `dict` | Last loaded ModulesInfo.json data | +| `IsDocked` | `bool` | Whether the Cmdr is currently docked *in their own ship*. | | `OnFoot` | `bool` | Whether the Cmdr is on foot | | `Component` | `dict` | 'Component' MicroResources in Odyssey, `int` count each. | | `Item` | `dict` | 'Item' MicroResources in Odyssey, `int` count each. | @@ -710,6 +711,17 @@ NB: It *is* possible, if a player is quick enough, to plot and clear a route before we load it, in which case we'd be retaining the *previous* plotted route. +New in version 5.6.0: + +`IsDocked` boolean added to `state`. This is set True for a `Location` event +having `"Docked":true"`, or the `Docked` event. It is set back to False (its +default value) for an `Undocked` event. Being on-foot in a station at login +time does *not* count as docked for this. + +In general on-foot, including being in a taxi, might not set this 100% +correctly. Its main use in core code is to detect being docked so as to send +any stored EDDN messages due to "Delay sending until docked" option. + ___ ##### Synthetic Events diff --git a/config/__init__.py b/config/__init__.py index 73d55975..100ec7e7 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -52,7 +52,7 @@ appcmdname = 'EDMC' # # Major.Minor.Patch(-prerelease)(+buildmetadata) # NB: Do *not* import this, use the functions appversion() and appversion_nobuild() -_static_appversion = '5.5.0' +_static_appversion = '5.6.0' _cached_version: Optional[semantic_version.Version] = None copyright = '© 2015-2019 Jonathan Harris, 2020-2022 EDCD' @@ -162,7 +162,7 @@ def appversion_nobuild() -> semantic_version.Version: class AbstractConfig(abc.ABC): """Abstract root class of all platform specific Config implementations.""" - OUT_MKT_EDDN = 1 + OUT_EDDN_SEND_STATION_DATA = 1 # OUT_MKT_BPC = 2 # No longer supported OUT_MKT_TD = 4 OUT_MKT_CSV = 8 @@ -171,12 +171,12 @@ class AbstractConfig(abc.ABC): # OUT_SYS_FILE = 32 # No longer supported # OUT_STAT = 64 # No longer available # OUT_SHIP_CORIOLIS = 128 # Replaced by OUT_SHIP - OUT_STATION_ANY = OUT_MKT_EDDN | OUT_MKT_TD | OUT_MKT_CSV # OUT_SYS_EDSM = 256 # Now a plugin # OUT_SYS_AUTO = 512 # Now always automatic OUT_MKT_MANUAL = 1024 - OUT_SYS_EDDN = 2048 - OUT_SYS_DELAY = 4096 + OUT_EDDN_SEND_NON_STATION = 2048 + OUT_EDDN_DELAY = 4096 + OUT_STATION_ANY = OUT_EDDN_SEND_STATION_DATA | OUT_MKT_TD | OUT_MKT_CSV app_dir_path: pathlib.Path plugin_dir_path: pathlib.Path @@ -454,19 +454,19 @@ def get_config(*args, **kwargs) -> AbstractConfig: :param kwargs: Args to be passed through to implementation. :return: Instance of the implementation. """ - if sys.platform == "darwin": + if sys.platform == "darwin": # pragma: sys-platform-darwin from .darwin import MacConfig return MacConfig(*args, **kwargs) - elif sys.platform == "win32": + elif sys.platform == "win32": # pragma: sys-platform-win32 from .windows import WinConfig return WinConfig(*args, **kwargs) - elif sys.platform == "linux": + elif sys.platform == "linux": # pragma: sys-platform-linux from .linux import LinuxConfig return LinuxConfig(*args, **kwargs) - else: + else: # pragma: sys-platform-not-known raise ValueError(f'Unknown platform: {sys.platform=}') diff --git a/config/darwin.py b/config/darwin.py index eb2b887f..895218a8 100644 --- a/config/darwin.py +++ b/config/darwin.py @@ -1,3 +1,4 @@ +"""Darwin/macOS implementation of AbstractConfig.""" import pathlib import sys from typing import Any, Dict, List, Union diff --git a/journal_lock.py b/journal_lock.py index 91a7895f..ef5cf983 100644 --- a/journal_lock.py +++ b/journal_lock.py @@ -94,7 +94,7 @@ class JournalLock: :return: LockResult - See the class Enum definition """ - if sys.platform == 'win32': + if sys.platform == 'win32': # pragma: sys-platform-win32 logger.trace_if('journal-lock', 'win32, using msvcrt') # win32 doesn't have fcntl, so we have to use msvcrt import msvcrt @@ -107,7 +107,7 @@ class JournalLock: f", assuming another process running: {e!r}") return JournalLockResult.ALREADY_LOCKED - else: # pytest coverage only sees this on !win32 + else: # pragma: sys-platform-not-win32 logger.trace_if('journal-lock', 'NOT win32, using fcntl') try: import fcntl @@ -143,7 +143,7 @@ class JournalLock: return True # We weren't locked, and still aren't unlocked = False - if sys.platform == 'win32': + if sys.platform == 'win32': # pragma: sys-platform-win32 logger.trace_if('journal-lock', 'win32, using msvcrt') # win32 doesn't have fcntl, so we have to use msvcrt import msvcrt @@ -160,7 +160,7 @@ class JournalLock: else: unlocked = True - else: # pytest coverage only sees this on !win32 + else: # pragma: sys-platform-not-win32 logger.trace_if('journal-lock', 'NOT win32, using fcntl') try: import fcntl diff --git a/monitor.py b/monitor.py index 859b2a50..d8c5707e 100644 --- a/monitor.py +++ b/monitor.py @@ -21,6 +21,8 @@ from typing import Tuple if TYPE_CHECKING: import tkinter +import semantic_version + import util_ships from config import config from edmc_data import edmc_suit_shortnames, edmc_suit_symbol_localised @@ -111,6 +113,7 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below # Context for journal handling self.version: Optional[str] = None + self.version_semantic: Optional[semantic_version.Version] = None self.is_beta = False self.mode: Optional[str] = None self.group: Optional[str] = None @@ -131,6 +134,11 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below self._fcmaterials_retries_remaining = 0 self._last_fcmaterials_journal_timestamp: Optional[float] = None + # For determining Live versus Legacy galaxy. + # The assumption is gameversion will parse via `coerce()` and always + # be >= for Live, and < for Legacy. + self.live_galaxy_base_version = semantic_version.Version('4.0.0') + self.__init_state() def __init_state(self) -> None: @@ -166,6 +174,7 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below 'Modules': None, 'CargoJSON': None, # The raw data from the last time cargo.json was read 'Route': None, # Last plotted route from Route.json file + 'IsDocked': False, # Whether we think cmdr is docked 'OnFoot': False, # Whether we think you're on-foot 'Component': defaultdict(int), # Odyssey Components in Ship Locker 'Item': defaultdict(int), # Odyssey Items in Ship Locker @@ -293,6 +302,7 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below self.currentdir = None self.version = None + self.version_semantic = None self.mode = None self.group = None self.cmdr = None @@ -306,6 +316,7 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below self.systemaddress = None self.is_beta = False self.state['OnFoot'] = False + self.state['IsDocked'] = False self.state['Body'] = None self.state['BodyType'] = None @@ -725,6 +736,7 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below self.station_marketid = None self.stationtype = None self.stationservices = None + self.state['IsDocked'] = False elif event_type == 'embark': # This event is logged when a player (on foot) gets into a ship or SRV @@ -791,6 +803,7 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below self.state['Dropship'] = False elif event_type == 'docked': + self.state['IsDocked'] = True self.station = entry.get('StationName') # May be None self.station_marketid = entry.get('MarketID') # May be None self.stationtype = entry.get('StationType') # May be None @@ -813,6 +826,8 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below if event_type == 'location': logger.trace_if('journal.locations', '"Location" event') + if entry.get('Docked'): + self.state['IsDocked'] = True elif event_type == 'fsdjump': self.planet = None @@ -1677,6 +1692,20 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below self.state['GameVersion'] = entry['gameversion'] self.state['GameBuild'] = entry['build'] self.version = self.state['GameVersion'] + + try: + self.version_semantic = semantic_version.Version.coerce(self.state['GameVersion']) + + except Exception: + # Catching all Exceptions as this is *one* call, and we won't + # get caught out by any semantic_version changes. + self.version_semantic = None + logger.error(f"Couldn't coerce {self.state['GameVersion']=}") + pass + + else: + logger.info(f"Parsed {self.state['GameVersion']=} into {self.version_semantic=}") + self.is_beta = any(v in self.version.lower() for v in ('alpha', 'beta')) # type: ignore except KeyError: if not suppress: @@ -2348,6 +2377,25 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below self._last_fcmaterials_journal_timestamp = None return file + def is_live_galaxy(self) -> bool: + """ + Indicate if current tracking indicates Live galaxy. + + We assume: + 1) `gameversion` remains something that semantic_verison.Version.coerce() can parse. + 2) Any Live galaxy client reports a version >= the defined base version. + 3) Any Legacy client will always report a version < that base version. + :return: True for Live, False for Legacy or unknown. + """ + # If we don't yet know the version we can't tell, so assume the worst + if self.version_semantic is None: + return False + + if self.version_semantic >= self.live_galaxy_base_version: + return True + + return False + # singleton monitor = EDLogs() diff --git a/plugins/eddn.py b/plugins/eddn.py index cd1d53ff..5af9b6e6 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -22,20 +22,22 @@ # # ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# # ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# +import http import itertools import json +import os import pathlib import re +import sqlite3 import sys import tkinter as tk from collections import OrderedDict -from os import SEEK_SET -from os.path import join from platform import system from textwrap import dedent +from threading import Lock from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Mapping, MutableMapping, Optional from typing import OrderedDict as OrderedDictT -from typing import TextIO, Tuple, Union +from typing import Tuple, Union import requests @@ -52,10 +54,6 @@ from prefs import prefsVersion from ttkHyperlinkLabel import HyperlinkLabel from util import text -if sys.platform != 'win32': - from fcntl import LOCK_EX, LOCK_NB, lockf - - if TYPE_CHECKING: def _(x: str) -> str: return x @@ -67,8 +65,16 @@ class This: """Holds module globals.""" def __init__(self): + # Game version and build + self.game_version = "" + self.game_build = "" + # Commander Name + self.cmdr_name = "" + # Track if we're on foot self.on_foot = False + # Track if we're docked + self.docked = False # Horizons ? self.horizons = False @@ -118,7 +124,6 @@ class This: this = This() - # This SKU is tagged on any module or ship that you must have Horizons for. HORIZONS_SKU = 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' # ELITE_HORIZONS_V_COBRA_MK_IV_1000` is for the Cobra Mk IV, but @@ -130,165 +135,395 @@ HORIZONS_SKU = 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' # one. -# TODO: a good few of these methods are static or could be classmethods. they should be created as such. +class EDDNSender: + """Handle sending of EDDN messages to the Gateway.""" -class EDDN: - """EDDN Data export.""" - - DEFAULT_URL = 'https://eddn.edcd.io:4430/upload/' - if 'eddn' in debug_senders: - DEFAULT_URL = f'http://{edmc_data.DEBUG_WEBSERVER_HOST}:{edmc_data.DEBUG_WEBSERVER_PORT}/eddn' - - REPLAYPERIOD = 400 # Roughly two messages per second, accounting for send delays [ms] - REPLAYFLUSH = 20 # Update log on disk roughly every 10 seconds + SQLITE_DB_FILENAME_V1 = 'eddn_queue-v1.db' + # EDDN schema types that pertain to station data + STATION_SCHEMAS = ('commodity', 'fcmaterials_capi', 'fcmaterials_journal', 'outfitting', 'shipyard') TIMEOUT = 10 # requests timeout - MODULE_RE = re.compile(r'^Hpt_|^Int_|Armour_', re.IGNORECASE) - CANONICALISE_RE = re.compile(r'\$(.+)_name;') UNKNOWN_SCHEMA_RE = re.compile( r"^FAIL: \[JsonValidationException\('Schema " r"https://eddn.edcd.io/schemas/(?P.+)/(?P[0-9]+) is unknown, " r"unable to validate.',\)\]$" ) - CAPI_LOCALISATION_RE = re.compile(r'^loc[A-Z].+') - def __init__(self, parent: tk.Tk): - self.parent: tk.Tk = parent + def __init__(self, eddn: 'EDDN', eddn_endpoint: str) -> None: + """ + Prepare the system for processing messages. + + - Ensure the sqlite3 database for EDDN replays exists and has schema. + - Convert any legacy file into the database. + - (Future) Handle any database migrations. + + :param eddn: Reference to the `EDDN` instance this is for. + :param eddn_endpoint: Where messages should be sent. + """ + self.eddn = eddn + self.eddn_endpoint = eddn_endpoint self.session = requests.Session() self.session.headers['User-Agent'] = user_agent - self.replayfile: Optional[TextIO] = None # For delayed messages - self.replaylog: List[str] = [] - self.fss_signals: List[Mapping[str, Any]] = [] - if config.eddn_url is not None: - self.eddn_url = config.eddn_url + self.db_conn = self.sqlite_queue_v1() + self.db = self.db_conn.cursor() - else: - self.eddn_url = self.DEFAULT_URL + ####################################################################### + # Queue database migration + ####################################################################### + self.convert_legacy_file() + ####################################################################### - def load_journal_replay(self) -> bool: + self.queue_processing = Lock() + # Initiate retry/send-now timer + logger.trace_if( + "plugin.eddn.send", + f"First queue run scheduled for {self.eddn.REPLAY_STARTUP_DELAY}ms from now" + ) + self.eddn.parent.after(self.eddn.REPLAY_STARTUP_DELAY, self.queue_check_and_send, True) + + def sqlite_queue_v1(self) -> sqlite3.Connection: """ - Load cached journal entries from disk. + Initialise a v1 EDDN queue database. - :return: a bool indicating success + :return: sqlite3 connection """ - # Try to obtain exclusive access to the journal cache - filename = join(config.app_dir, 'replay.jsonl') + db_conn = sqlite3.connect(config.app_dir_path / self.SQLITE_DB_FILENAME_V1) + db = db_conn.cursor() + try: - try: - # Try to open existing file - self.replayfile = open(filename, 'r+', buffering=1) + db.execute( + """ + CREATE TABLE messages + ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created TEXT NOT NULL, + cmdr TEXT NOT NULL, + edmc_version TEXT, + game_version TEXT, + game_build TEXT, + message TEXT NOT NULL + ) + """ + ) - except FileNotFoundError: - self.replayfile = open(filename, 'w+', buffering=1) # Create file + db.execute( + """ + CREATE INDEX messages_created ON messages + ( + created + ) + """ + ) - if sys.platform != 'win32': # open for writing is automatically exclusive on Windows - lockf(self.replayfile, LOCK_EX | LOCK_NB) + db.execute( + """ + CREATE INDEX messages_cmdr ON messages + ( + cmdr + ) + """ + ) - except OSError: - logger.exception('Failed opening "replay.jsonl"') - if self.replayfile: - self.replayfile.close() - - self.replayfile = None - return False + except sqlite3.OperationalError as e: + if str(e) != "table messages already exists": + # Cleanup, as schema creation failed + db.close() + db_conn.close() + raise e else: - self.replaylog = [line.strip() for line in self.replayfile] - return True + logger.info("New `eddn_queue-v1.db` created") - def flush(self): - """Flush the replay file, clearing any data currently there that is not in the replaylog list.""" - if self.replayfile is None: - logger.error('replayfile is None!') - return + # We return only the connection, so tidy up + db.close() - self.replayfile.seek(0, SEEK_SET) - self.replayfile.truncate() - for line in self.replaylog: - self.replayfile.write(f'{line}\n') + return db_conn - self.replayfile.flush() + def convert_legacy_file(self): + """Convert a legacy file's contents into the sqlite3 db.""" + filename = config.app_dir_path / 'replay.jsonl' + try: + with open(filename, 'r+', buffering=1) as replay_file: + logger.info("Converting legacy `replay.jsonl` to `eddn_queue-v1.db`") + for line in replay_file: + cmdr, msg = json.loads(line) + self.add_message(cmdr, msg) - def close(self): - """Close down the EDDN class instance.""" - logger.debug('Closing replayfile...') - if self.replayfile: - self.replayfile.close() + except FileNotFoundError: + pass - self.replayfile = None - logger.debug('Done.') + finally: + # Best effort at removing the file/contents + # NB: The legacy code assumed it could write to the file. + logger.info("Conversion` to `eddn_queue-v1.db` complete, removing `replay.jsonl`") + replay_file = open(filename, 'w') # Will truncate + replay_file.close() + os.unlink(filename) + + def close(self) -> None: + """Clean up any resources.""" + logger.debug('Closing db cursor.') + if self.db: + self.db.close() + + logger.debug('Closing db connection.') + if self.db_conn: + self.db_conn.close() logger.debug('Closing EDDN requests.Session.') self.session.close() - def send(self, cmdr: str, msg: Mapping[str, Any]) -> None: + def add_message(self, cmdr: str, msg: MutableMapping[str, Any]) -> int: """ - Send sends an update to EDDN. + Add an EDDN message to the database. - :param cmdr: the CMDR to use as the uploader ID. - :param msg: the payload to send. + `msg` absolutely needs to be the **FULL** EDDN message, including all + of `header`, `$schemaRef` and `message`. Code handling this not being + the case is only for loading the legacy `replay.json` file messages. + + NB: Although `cmdr` *should* be the same as `msg->header->uploaderID` + we choose not to assume that. + + :param cmdr: Name of the Commander that created this message. + :param msg: The full, transmission-ready, EDDN message. + :return: ID of the successfully inserted row. """ - should_return, new_data = killswitch.check_killswitch('plugins.eddn.send', msg) + logger.trace_if("plugin.eddn.send", f"Message for {msg['$schemaRef']=}") + # Cater for legacy replay.json messages + if 'header' not in msg: + msg['header'] = { + # We have to lie and say it's *this* version, but denote that + # it might not actually be this version. + 'softwareName': f'{applongname} [{system() if sys.platform != "darwin" else "Mac OS"}]' + ' (legacy replay)', + 'softwareVersion': str(appversion_nobuild()), + 'uploaderID': cmdr, + 'gameversion': '', # Can't add what we don't know + 'gamebuild': '', # Can't add what we don't know + } + + created = msg['message']['timestamp'] + edmc_version = msg['header']['softwareVersion'] + game_version = msg['header'].get('gameversion', '') + game_build = msg['header'].get('gamebuild', '') + uploader = msg['header']['uploaderID'] + + try: + self.db.execute( + """ + INSERT INTO messages ( + created, cmdr, edmc_version, game_version, game_build, message + ) + VALUES ( + ?, ?, ?, ?, ?, ? + ) + """, + (created, uploader, edmc_version, game_version, game_build, json.dumps(msg)) + ) + self.db_conn.commit() + + except Exception: + logger.exception('INSERT error') + # Can't possibly be a valid row id + return -1 + + logger.trace_if("plugin.eddn.send", f"Message for {msg['$schemaRef']=} recorded, id={self.db.lastrowid}") + return self.db.lastrowid or -1 + + def delete_message(self, row_id: int) -> None: + """ + Delete a queued message by row id. + + :param row_id: id of message to be deleted. + """ + logger.trace_if("plugin.eddn.send", f"Deleting message with {row_id=}") + self.db.execute( + """ + DELETE FROM messages WHERE id = :row_id + """, + {'row_id': row_id} + ) + self.db_conn.commit() + + def send_message_by_id(self, id: int): + """ + Transmit the message identified by the given ID. + + :param id: + :return: + """ + logger.trace_if("plugin.eddn.send", f"Sending message with {id=}") + self.db.execute( + """ + SELECT * FROM messages WHERE id = :row_id + """, + {'row_id': id} + ) + row = dict(zip([c[0] for c in self.db.description], self.db.fetchone())) + + try: + if self.send_message(row['message']): + self.delete_message(id) + return True + + except requests.exceptions.HTTPError as e: + logger.warning(f"HTTPError: {str(e)}") + + return False + + def send_message(self, msg: str) -> bool: + """ + Transmit a fully-formed EDDN message to the Gateway. + + If this is called then the attempt *will* be made. This is not where + options to not send to EDDN, or to delay the sending until docked, + are checked. + + It *is* however the one 'sending' place that the EDDN killswitches are checked. + + Should catch and handle all failure conditions. A `True` return might + mean that the message was successfully sent, *or* that this message + should not be retried after a failure, i.e. too large. + + :param msg: Fully formed, string, message. + :return: `True` for "now remove this message from the queue" + """ + logger.trace_if("plugin.eddn.send", "Sending message") + should_return, new_data = killswitch.check_killswitch('plugins.eddn.send', json.loads(msg)) if should_return: logger.warning('eddn.send has been disabled via killswitch. Returning.') - return + return False - msg = new_data - - uploader_id = cmdr - - to_send: OrderedDictT[str, OrderedDict[str, Any]] = OrderedDict([ - ('$schemaRef', msg['$schemaRef']), - ('header', OrderedDict([ - ('softwareName', f'{applongname} [{system() if sys.platform != "darwin" else "Mac OS"}]'), - ('softwareVersion', str(appversion_nobuild())), - ('uploaderID', uploader_id), - ])), - ('message', msg['message']), - ]) - - # About the smallest request is going to be (newlines added for brevity): - # {"$schemaRef":"https://eddn.edcd.io/schemas/commodity/3","header":{"softwareName":"E:D Market - # Connector Windows","softwareVersion":"5.3.0-beta4extra","uploaderID":"abcdefghijklm"},"messag - # e":{"systemName":"delphi","stationName":"The Oracle","marketId":128782803,"timestamp":"2022-0 - # 1-26T12:00:00Z","commodities":[]}} - # - # Which comes to 315 bytes (including \n) and compresses to 244 bytes. So lets just compress everything - - encoded, compressed = text.gzip(json.dumps(to_send, separators=(',', ':')), max_size=0) + status: tk.Widget = self.eddn.parent.children['status'] + # Even the smallest possible message compresses somewhat, so always compress + encoded, compressed = text.gzip(json.dumps(new_data, separators=(',', ':')), max_size=0) headers: None | dict[str, str] = None if compressed: headers = {'Content-Encoding': 'gzip'} - r = self.session.post(self.eddn_url, data=encoded, timeout=self.TIMEOUT, headers=headers) - if r.status_code != requests.codes.ok: + try: + r = self.session.post(self.eddn_endpoint, data=encoded, timeout=self.TIMEOUT, headers=headers) + if r.status_code == requests.codes.ok: + return True - # Check if EDDN is still objecting to an empty commodities list - if ( - r.status_code == 400 - and msg['$schemaRef'] == 'https://eddn.edcd.io/schemas/commodity/3' - and msg['message']['commodities'] == [] - and r.text == "FAIL: []" - ): - logger.trace_if('plugin.eddn', "EDDN is still objecting to empty commodities data") - return # We want to silence warnings otherwise - - if r.status_code == 413: + if r.status_code == http.HTTPStatus.REQUEST_ENTITY_TOO_LARGE: extra_data = { - 'schema_ref': msg.get('$schemaRef', 'Unset $schemaRef!'), + 'schema_ref': new_data.get('$schemaRef', 'Unset $schemaRef!'), 'sent_data_len': str(len(encoded)), } if '/journal/' in extra_data['schema_ref']: - extra_data['event'] = msg.get('message', {}).get('event', 'No Event Set') + extra_data['event'] = new_data.get('message', {}).get('event', 'No Event Set') - self._log_response(r, header_msg='Got a 413 while POSTing data', **extra_data) - return # drop the error + self._log_response(r, header_msg='Got "Payload Too Large" while POSTing data', **extra_data) + return True - if not self.UNKNOWN_SCHEMA_RE.match(r.text): - self._log_response(r, header_msg='Status from POST wasn\'t 200 (OK)') + self._log_response(r, header_msg="Status from POST wasn't 200 (OK)") + r.raise_for_status() - r.raise_for_status() + except requests.exceptions.HTTPError as e: + if unknown_schema := self.UNKNOWN_SCHEMA_RE.match(e.response.text): + logger.debug(f"EDDN doesn't (yet?) know about schema: {unknown_schema['schema_name']}" + f"/{unknown_schema['schema_version']}") + # This dropping is to cater for the time period when EDDN doesn't *yet* support a new schema. + return True + + elif e.response.status_code == http.HTTPStatus.BAD_REQUEST: + # EDDN straight up says no, so drop the message + logger.debug(f"EDDN responded '400 Bad Request' to the message, dropping:\n{msg!r}") + return True + + else: + # This should catch anything else, e.g. timeouts, gateway errors + status['text'] = self.http_error_to_log(e) + + except requests.exceptions.RequestException as e: + logger.debug('Failed sending', exc_info=e) + # LANG: Error while trying to send data to EDDN + status['text'] = _("Error: Can't connect to EDDN") + + except Exception as e: + logger.debug('Failed sending', exc_info=e) + status['text'] = str(e) + + return False + + def queue_check_and_send(self, reschedule: bool = False) -> None: # noqa: CCR001 + """ + Check if we should be sending queued messages, and send if we should. + + :param reschedule: Boolean indicating if we should call `after()` again. + """ + logger.trace_if("plugin.eddn.send", "Called") + # Mutex in case we're already processing + if not self.queue_processing.acquire(blocking=False): + logger.trace_if("plugin.eddn.send", "Couldn't obtain mutex") + if reschedule: + logger.trace_if("plugin.eddn.send", f"Next run scheduled for {self.eddn.REPLAY_PERIOD}ms from now") + self.eddn.parent.after(self.eddn.REPLAY_PERIOD, self.queue_check_and_send, reschedule) + + else: + logger.trace_if("plugin.eddn.send", "NO next run scheduled (there should be another one already set)") + + return + + logger.trace_if("plugin.eddn.send", "Obtained mutex") + # Used to indicate if we've rescheduled at the faster rate already. + have_rescheduled = False + # We send either if docked or 'Delay sending until docked' not set + if this.docked or not (config.get_int('output') & config.OUT_EDDN_DELAY): + logger.trace_if("plugin.eddn.send", "Should send") + # We need our own cursor here, in case the semantics of + # tk `after()` could allow this to run in the middle of other + # database usage. + db_cursor = self.db_conn.cursor() + + # Options: + # 1. Process every queued message, regardless. + # 2. Bail if we get any sort of connection error from EDDN. + + # Every queued message that is for *this* commander. We do **NOT** + # check if it's station/not-station, as the control of if a message + # was even created, versus the Settings > EDDN options, is applied + # *then*, not at time of sending. + try: + db_cursor.execute( + """ + SELECT id FROM messages + ORDER BY created ASC + LIMIT 1 + """ + ) + + except Exception: + logger.exception("DB error querying queued messages") + + else: + row = db_cursor.fetchone() + if row: + row = dict(zip([c[0] for c in db_cursor.description], row)) + if self.send_message_by_id(row['id']): + # If `True` was returned then we're done with this message. + # `False` means "failed to send, but not because the message + # is bad", i.e. an EDDN Gateway problem. Thus, in that case + # we do *NOT* schedule attempting the next message. + # Always re-schedule as this is only a "Don't hammer EDDN" delay + logger.trace_if("plugin.eddn.send", f"Next run scheduled for {self.eddn.REPLAY_DELAY}ms from " + "now") + self.eddn.parent.after(self.eddn.REPLAY_DELAY, self.queue_check_and_send, reschedule) + have_rescheduled = True + + db_cursor.close() + + else: + logger.trace_if("plugin.eddn.send", "Should NOT send") + + self.queue_processing.release() + logger.trace_if("plugin.eddn.send", "Mutex released") + if reschedule and not have_rescheduled: + # Set us up to run again per the configured period + logger.trace_if("plugin.eddn.send", f"Next run scheduled for {self.eddn.REPLAY_PERIOD}ms from now") + self.eddn.parent.after(self.eddn.REPLAY_PERIOD, self.queue_check_and_send, reschedule) def _log_response( self, @@ -315,91 +550,6 @@ class EDDN: Content :\t{response.text} ''')+additional_data) - def sendreplay(self) -> None: # noqa: CCR001 - """Send cached Journal lines to EDDN.""" - if not self.replayfile: - return # Probably closing app - - status: tk.Widget = self.parent.children['status'] - - if not self.replaylog: - status['text'] = '' - return - - localized: str = _('Sending data to EDDN...') # LANG: Status text shown while attempting to send data - if len(self.replaylog) == 1: - status['text'] = localized - - else: - status['text'] = f'{localized.replace("...", "")} [{len(self.replaylog)}]' - - self.parent.update_idletasks() - - # Paranoia check in case this function gets chain-called. - if not self.replaylog: - # import traceback - # logger.error( - # f'self.replaylog (type: {type(self.replaylog)}) is falsey after update_idletasks(). Traceback:\n' - # f'{"".join(traceback.format_list(traceback.extract_stack()))}') - return - - try: - cmdr, msg = json.loads(self.replaylog[0], object_pairs_hook=OrderedDict) - - except json.JSONDecodeError as e: - # Couldn't decode - shouldn't happen! - logger.debug(f'\n{self.replaylog[0]}\n', exc_info=e) - # Discard and continue - self.replaylog.pop(0) - - else: - # TODO: Check message against *current* relevant schema so we don't try - # to send an old message that's now invalid. - - # Rewrite old schema name - if msg['$schemaRef'].startswith('http://schemas.elite-markets.net/eddn/'): - msg['$schemaRef'] = str(msg['$schemaRef']).replace( - 'http://schemas.elite-markets.net/eddn/', - 'https://eddn.edcd.io/schemas/' - ) - - try: - self.send(cmdr, msg) - self.replaylog.pop(0) - if not len(self.replaylog) % self.REPLAYFLUSH: - self.flush() - - except requests.exceptions.HTTPError as e: - if unknown_schema := self.UNKNOWN_SCHEMA_RE.match(e.response.text): - logger.debug(f"EDDN doesn't (yet?) know about schema: {unknown_schema['schema_name']}" - f"/{unknown_schema['schema_version']}") - # NB: This dropping is to cater for the time when EDDN - # doesn't *yet* support a new schema. - self.replaylog.pop(0) # Drop the message - self.flush() # Truncates the file, then writes the extant data - - elif e.response.status_code == 400: - # EDDN straight up says no, so drop the message - logger.debug(f"EDDN responded '400' to the message, dropping:\n{msg!r}") - self.replaylog.pop(0) # Drop the message - self.flush() # Truncates the file, then writes the extant data - - else: - status['text'] = self.http_error_to_log(e) - - except requests.exceptions.RequestException as e: - logger.debug('Failed sending', exc_info=e) - # LANG: Error while trying to send data to EDDN - status['text'] = _("Error: Can't connect to EDDN") - return # stop sending - - except Exception as e: - logger.debug('Failed sending', exc_info=e) - status['text'] = str(e) - return # stop sending - - self.parent.after(self.REPLAYPERIOD, self.sendreplay) - @staticmethod def http_error_to_log(exception: requests.exceptions.HTTPError) -> str: """Convert an exception from raise_for_status to a log message and displayed error.""" @@ -421,6 +571,45 @@ class EDDN: # LANG: EDDN returned some sort of HTTP error, one we didn't expect. {STATUS} contains a number return _('EDDN Error: Returned {STATUS} status code').format(STATUS=status_code) + +# TODO: a good few of these methods are static or could be classmethods. they should be created as such. +class EDDN: + """EDDN Data export.""" + + DEFAULT_URL = 'https://eddn.edcd.io:4430/upload/' + if 'eddn' in debug_senders: + DEFAULT_URL = f'http://{edmc_data.DEBUG_WEBSERVER_HOST}:{edmc_data.DEBUG_WEBSERVER_PORT}/eddn' + + # FIXME: Change back to `300_000` + REPLAY_STARTUP_DELAY = 10_000 # Delay during startup before checking queue [milliseconds] + REPLAY_PERIOD = 300_000 # How often to try (re-)sending the queue, [milliseconds] + REPLAY_DELAY = 400 # Roughly two messages per second, accounting for send delays [milliseconds] + REPLAYFLUSH = 20 # Update log on disk roughly every 10 seconds + MODULE_RE = re.compile(r'^Hpt_|^Int_|Armour_', re.IGNORECASE) + CANONICALISE_RE = re.compile(r'\$(.+)_name;') + CAPI_LOCALISATION_RE = re.compile(r'^loc[A-Z].+') + + def __init__(self, parent: tk.Tk): + self.parent: tk.Tk = parent + + if config.eddn_url is not None: + self.eddn_url = config.eddn_url + + else: + self.eddn_url = self.DEFAULT_URL + + self.sender = EDDNSender(self, self.eddn_url) + + self.fss_signals: List[Mapping[str, Any]] = [] + + def close(self): + """Close down the EDDN class instance.""" + logger.debug('Closing Sender...') + if self.sender: + self.sender.close() + + logger.debug('Done.') + def export_commodities(self, data: Mapping[str, Any], is_beta: bool) -> None: # noqa: CCR001 """ Update EDDN with the commodities on the current (lastStarport) station. @@ -486,9 +675,10 @@ class EDDN: if 'prohibited' in data['lastStarport']: message['prohibited'] = sorted(x for x in (data['lastStarport']['prohibited'] or {}).values()) - self.send(data['commander']['name'], { + self.send_message(data['commander']['name'], { '$schemaRef': f'https://eddn.edcd.io/schemas/commodity/3{"/test" if is_beta else ""}', 'message': message, + 'header': self.standard_header(game_version='CAPI-market', game_build=''), }) this.commodities = commodities @@ -571,7 +761,7 @@ class EDDN: # Don't send empty modules list - schema won't allow it if outfitting and this.outfitting != (horizons, outfitting): - self.send(data['commander']['name'], { + self.send_message(data['commander']['name'], { '$schemaRef': f'https://eddn.edcd.io/schemas/outfitting/2{"/test" if is_beta else ""}', 'message': OrderedDict([ ('timestamp', data['timestamp']), @@ -582,6 +772,7 @@ class EDDN: ('modules', outfitting), ('odyssey', this.odyssey), ]), + 'header': self.standard_header(game_version='CAPI-shipyard', game_build=''), }) this.outfitting = (horizons, outfitting) @@ -615,7 +806,7 @@ class EDDN: ) # Don't send empty ships list - shipyard data is only guaranteed present if user has visited the shipyard. if shipyard and this.shipyard != (horizons, shipyard): - self.send(data['commander']['name'], { + self.send_message(data['commander']['name'], { '$schemaRef': f'https://eddn.edcd.io/schemas/shipyard/2{"/test" if is_beta else ""}', 'message': OrderedDict([ ('timestamp', data['timestamp']), @@ -626,6 +817,7 @@ class EDDN: ('ships', shipyard), ('odyssey', this.odyssey), ]), + 'header': self.standard_header(game_version='CAPI-shipyard', game_build=''), }) this.shipyard = (horizons, shipyard) @@ -663,7 +855,7 @@ class EDDN: # none and that really does need to be recorded over EDDN so that, e.g. # EDDB can update in a timely manner. if this.commodities != commodities: - self.send(cmdr, { + self.send_message(cmdr, { '$schemaRef': f'https://eddn.edcd.io/schemas/commodity/3{"/test" if is_beta else ""}', 'message': OrderedDict([ ('timestamp', entry['timestamp']), @@ -702,7 +894,7 @@ class EDDN: ) # Don't send empty modules list - schema won't allow it if outfitting and this.outfitting != (horizons, outfitting): - self.send(cmdr, { + self.send_message(cmdr, { '$schemaRef': f'https://eddn.edcd.io/schemas/outfitting/2{"/test" if is_beta else ""}', 'message': OrderedDict([ ('timestamp', entry['timestamp']), @@ -736,7 +928,7 @@ class EDDN: shipyard = sorted(ship['ShipType'] for ship in ships) # Don't send empty ships list - shipyard data is only guaranteed present if user has visited the shipyard. if shipyard and this.shipyard != (horizons, shipyard): - self.send(cmdr, { + self.send_message(cmdr, { '$schemaRef': f'https://eddn.edcd.io/schemas/shipyard/2{"/test" if is_beta else ""}', 'message': OrderedDict([ ('timestamp', entry['timestamp']), @@ -751,35 +943,72 @@ class EDDN: # this.shipyard = (horizons, shipyard) - def export_journal_entry(self, cmdr: str, entry: Mapping[str, Any], msg: Mapping[str, Any]) -> None: + def send_message(self, cmdr: str, msg: MutableMapping[str, Any]) -> None: """ - Update EDDN with an event from the journal. - - Additionally if other lines have been saved for retry, it may send - those as well. + Send an EDDN message. :param cmdr: Commander name as passed in through `journal_entry()`. - :param entry: The full journal event dictionary (due to checks in this function). :param msg: The EDDN message body to be sent. """ - if self.replayfile or self.load_journal_replay(): - # Store the entry - self.replaylog.append(json.dumps([cmdr, msg])) - self.replayfile.write(f'{self.replaylog[-1]}\n') # type: ignore + # Check if the user configured messages to be sent. + # + # 1. If this is a 'station' data message then check config.EDDN_SEND_STATION_DATA + # 2. Else check against config.EDDN_SEND_NON_STATION *and* config.OUT_EDDN_DELAY + if any(f'{s}' in msg['$schemaRef'] for s in EDDNSender.STATION_SCHEMAS): + # 'Station data' + if config.get_int('output') & config.OUT_EDDN_SEND_STATION_DATA: + # And user has 'station data' configured to be sent + logger.trace_if("plugin.eddn.send", "Recording/sending 'station' message") + if 'header' not in msg: + msg['header'] = self.standard_header() - if ( - entry['event'] == 'Docked' or (entry['event'] == 'Location' and entry['Docked']) or not - (config.get_int('output') & config.OUT_SYS_DELAY) - ): - self.parent.after(self.REPLAYPERIOD, self.sendreplay) # Try to send this and previous entries + msg_id = self.sender.add_message(cmdr, msg) + # 'Station data' is never delayed on construction of message + self.sender.send_message_by_id(msg_id) + + elif config.get_int('output') & config.OUT_EDDN_SEND_NON_STATION: + # Any data that isn't 'station' is configured to be sent + logger.trace_if("plugin.eddn.send", "Recording 'non-station' message") + if 'header' not in msg: + msg['header'] = self.standard_header() + + msg_id = self.sender.add_message(cmdr, msg) + if this.docked or not (config.get_int('output') & config.OUT_EDDN_DELAY): + # No delay in sending configured, so attempt immediately + logger.trace_if("plugin.eddn.send", "Sending 'non-station' message") + self.sender.send_message_by_id(msg_id) + + def standard_header( + self, game_version: Optional[str] = None, game_build: Optional[str] = None + ) -> MutableMapping[str, Any]: + """ + Return the standard header for an EDDN message, given tracked state. + + NB: This should *only* be called for newly constructed messages, not + for either a legacy message or an already queued one! + + :return: The standard header + """ + # We want to pass `''` sometimes, so can't just use a Truthiness test + if game_version is not None: + gv = game_version else: - # Can't access replay file! Send immediately. - # LANG: Status text shown while attempting to send data - self.parent.children['status']['text'] = _('Sending data to EDDN...') - self.parent.update_idletasks() - self.send(cmdr, msg) - self.parent.children['status']['text'] = '' + gv = this.game_version + + if game_build is not None: + gb = game_build + + else: + gb = this.game_build + + return { + 'softwareName': f'{applongname} [{system() if sys.platform != "darwin" else "Mac OS"}]', + 'softwareVersion': str(appversion_nobuild()), + 'uploaderID': this.cmdr_name, + 'gameversion': gv, + 'gamebuild': gb, + } def export_journal_generic(self, cmdr: str, is_beta: bool, entry: Mapping[str, Any]) -> None: """ @@ -793,7 +1022,7 @@ class EDDN: '$schemaRef': f'https://eddn.edcd.io/schemas/journal/1{"/test" if is_beta else ""}', 'message': entry } - this.eddn.export_journal_entry(cmdr, entry, msg) + this.eddn.send_message(cmdr, msg) def entry_augment_system_data( self, @@ -881,7 +1110,7 @@ class EDDN: 'message': entry } - this.eddn.export_journal_entry(cmdr, entry, msg) + this.eddn.send_message(cmdr, msg) return None def export_journal_navbeaconscan( @@ -923,7 +1152,7 @@ class EDDN: 'message': entry } - this.eddn.export_journal_entry(cmdr, entry, msg) + this.eddn.send_message(cmdr, msg) return None def export_journal_codexentry( # noqa: CCR001 @@ -1023,7 +1252,7 @@ class EDDN: 'message': entry } - this.eddn.export_journal_entry(cmdr, entry, msg) + this.eddn.send_message(cmdr, msg) return None def export_journal_scanbarycentre( @@ -1077,7 +1306,7 @@ class EDDN: 'message': entry } - this.eddn.export_journal_entry(cmdr, entry, msg) + this.eddn.send_message(cmdr, msg) return None def export_journal_navroute( @@ -1150,7 +1379,7 @@ class EDDN: 'message': entry } - this.eddn.export_journal_entry(cmdr, entry, msg) + this.eddn.send_message(cmdr, msg) return None def export_journal_fcmaterials( @@ -1196,6 +1425,9 @@ class EDDN: # } # ] # } + # Abort if we're not configured to send 'station' data. + if not config.get_int('output') & config.OUT_EDDN_SEND_STATION_DATA: + return None # Sanity check if 'Items' not in entry: @@ -1231,7 +1463,7 @@ class EDDN: 'message': entry } - this.eddn.export_journal_entry(cmdr, entry, msg) + this.eddn.send_message(cmdr, msg) return None def export_capi_fcmaterials( @@ -1286,10 +1518,11 @@ class EDDN: msg = { '$schemaRef': f'https://eddn.edcd.io/schemas/fcmaterials_capi/1{"/test" if is_beta else ""}', - 'message': entry + 'message': entry, + 'header': self.standard_header(game_version='CAPI-market', game_build=''), } - this.eddn.export_journal_entry(data['commander']['name'], entry, msg) + this.eddn.send_message(data['commander']['name'], msg) return None def export_journal_approachsettlement( @@ -1364,7 +1597,7 @@ class EDDN: 'message': entry } - this.eddn.export_journal_entry(cmdr, entry, msg) + this.eddn.send_message(cmdr, msg) return None def export_journal_fssallbodiesfound( @@ -1414,7 +1647,7 @@ class EDDN: 'message': entry } - this.eddn.export_journal_entry(cmdr, entry, msg) + this.eddn.send_message(cmdr, msg) return None def export_journal_fssbodysignals( @@ -1470,7 +1703,7 @@ class EDDN: 'message': entry } - this.eddn.export_journal_entry(cmdr, entry, msg) + this.eddn.send_message(cmdr, msg) return None def enqueue_journal_fsssignaldiscovered(self, entry: MutableMapping[str, Any]) -> None: @@ -1577,8 +1810,7 @@ class EDDN: logger.trace_if("plugin.eddn.fsssignaldiscovered", f"FSSSignalDiscovered batch is {json.dumps(msg)}") - # Fake an 'entry' as it's only there for some "should we send replay?" checks in the called function. - this.eddn.export_journal_entry(cmdr, {'event': 'send_fsssignaldiscovered'}, msg) + this.eddn.send_message(cmdr, msg) self.fss_signals = [] return None @@ -1611,20 +1843,13 @@ def plugin_app(parent: tk.Tk) -> Optional[tk.Frame]: Set up any plugin-specific UI. In this case we need the tkinter parent in order to later call - `update_idletasks()` on it. - - TODO: Re-work the whole replaylog and general sending to EDDN so this isn't - necessary. + `update_idletasks()` on it, or schedule things with `after()`. :param parent: tkinter parent frame. :return: Optional tk.Frame, if the tracking UI is active. """ this.parent = parent this.eddn = EDDN(parent) - # Try to obtain exclusive lock on journal cache, even if we don't need it yet - if not this.eddn.load_journal_replay(): - # Shouldn't happen - don't bother localizing - this.parent.children['status']['text'] = 'Error: Is another copy of this app already running?' if config.eddn_tracking_ui: this.ui = tk.Frame(parent) @@ -1686,7 +1911,7 @@ def plugin_prefs(parent, cmdr: str, is_beta: bool) -> Frame: BUTTONX = 12 # noqa: N806 # indent Checkbuttons and Radiobuttons if prefsVersion.shouldSetDefaults('0.0.0.0', not bool(config.get_int('output'))): - output: int = (config.OUT_MKT_EDDN | config.OUT_SYS_EDDN) # default settings + output: int = (config.OUT_EDDN_SEND_STATION_DATA | config.OUT_EDDN_SEND_NON_STATION) # default settings else: output = config.get_int('output') @@ -1697,11 +1922,11 @@ def plugin_prefs(parent, cmdr: str, is_beta: bool) -> Frame: eddnframe, text='Elite Dangerous Data Network', background=nb.Label().cget('background'), - url='https://github.com/EDSM-NET/EDDN/wiki', + url='https://github.com/EDCD/EDDN#eddn---elite-dangerous-data-network', underline=True ).grid(padx=PADX, sticky=tk.W) # Don't translate - this.eddn_station = tk.IntVar(value=(output & config.OUT_MKT_EDDN) and 1) + this.eddn_station = tk.IntVar(value=(output & config.OUT_EDDN_SEND_STATION_DATA) and 1) this.eddn_station_button = nb.Checkbutton( eddnframe, # LANG: Enable EDDN support for station data checkbox label @@ -1711,7 +1936,7 @@ def plugin_prefs(parent, cmdr: str, is_beta: bool) -> Frame: ) # Output setting this.eddn_station_button.grid(padx=BUTTONX, pady=(5, 0), sticky=tk.W) - this.eddn_system = tk.IntVar(value=(output & config.OUT_SYS_EDDN) and 1) + this.eddn_system = tk.IntVar(value=(output & config.OUT_EDDN_SEND_NON_STATION) and 1) # Output setting new in E:D 2.2 this.eddn_system_button = nb.Checkbutton( eddnframe, @@ -1722,7 +1947,7 @@ def plugin_prefs(parent, cmdr: str, is_beta: bool) -> Frame: ) this.eddn_system_button.grid(padx=BUTTONX, pady=(5, 0), sticky=tk.W) - this.eddn_delay = tk.IntVar(value=(output & config.OUT_SYS_DELAY) and 1) + this.eddn_delay = tk.IntVar(value=(output & config.OUT_EDDN_DELAY) and 1) # Output setting under 'Send system and scan data to the Elite Dangerous Data Network' new in E:D 2.2 this.eddn_delay_button = nb.Checkbutton( eddnframe, @@ -1741,9 +1966,12 @@ def prefsvarchanged(event=None) -> None: :param event: tkinter event ? """ + # These two lines are legacy and probably not even needed this.eddn_station_button['state'] = tk.NORMAL this.eddn_system_button['state'] = tk.NORMAL - this.eddn_delay_button['state'] = this.eddn.replayfile and this.eddn_system.get() and tk.NORMAL or tk.DISABLED + # This line will grey out the 'Delay sending ...' option if the 'Send + # system and scan data' option is off. + this.eddn_delay_button['state'] = this.eddn_system.get() and tk.NORMAL or tk.DISABLED def prefs_changed(cmdr: str, is_beta: bool) -> None: @@ -1757,9 +1985,9 @@ def prefs_changed(cmdr: str, is_beta: bool) -> None: 'output', (config.get_int('output') & (config.OUT_MKT_TD | config.OUT_MKT_CSV | config.OUT_SHIP | config.OUT_MKT_MANUAL)) + - (this.eddn_station.get() and config.OUT_MKT_EDDN) + - (this.eddn_system.get() and config.OUT_SYS_EDDN) + - (this.eddn_delay.get() and config.OUT_SYS_DELAY) + (this.eddn_station.get() and config.OUT_EDDN_SEND_STATION_DATA) + + (this.eddn_system.get() and config.OUT_EDDN_SEND_NON_STATION) + + (this.eddn_delay.get() and config.OUT_EDDN_DELAY) ) @@ -1829,7 +2057,7 @@ def journal_entry( # noqa: C901, CCR001 """ Process a new Journal entry. - :param cmdr: `str` - Name of currennt Cmdr. + :param cmdr: `str` - Name of current Cmdr. :param is_beta: `bool` - True if this is a beta version of the Game. :param system: `str` - Name of system Cmdr is in. :param station: `str` - Name of station Cmdr is docked at, if applicable. @@ -1849,7 +2077,11 @@ def journal_entry( # noqa: C901, CCR001 entry = new_data event_name = entry['event'].lower() + this.cmdr_name = cmdr + this.game_version = state['GameVersion'] + this.game_build = state['GameBuild'] this.on_foot = state['OnFoot'] + this.docked = state['IsDocked'] # Note if we're under Horizons and/or Odyssey # The only event these are already in is `LoadGame` which isn't sent to EDDN. @@ -1909,6 +2141,9 @@ def journal_entry( # noqa: C901, CCR001 # Yes, explicitly state `None` here, so it's crystal clear. this.systemaddress = entry.get('SystemAddress', None) # type: ignore + if event_name == 'docked': + this.eddn.parent.after(this.eddn.REPLAY_DELAY, this.eddn.sender.queue_check_and_send, False) + elif event_name == 'approachbody': this.body_name = entry['Body'] this.body_id = entry.get('BodyID') @@ -1928,7 +2163,7 @@ def journal_entry( # noqa: C901, CCR001 this.status_body_name = None # Events with their own EDDN schema - if config.get_int('output') & config.OUT_SYS_EDDN and not state['Captain']: + if config.get_int('output') & config.OUT_EDDN_SEND_NON_STATION and not state['Captain']: if event_name == 'fssdiscoveryscan': return this.eddn.export_journal_fssdiscoveryscan(cmdr, system, state['StarPos'], is_beta, entry) @@ -1985,7 +2220,7 @@ def journal_entry( # noqa: C901, CCR001 ) # Send journal schema events to EDDN, but not when on a crew - if (config.get_int('output') & config.OUT_SYS_EDDN and not state['Captain'] and + if (config.get_int('output') & config.OUT_EDDN_SEND_NON_STATION and not state['Captain'] and (event_name in ('location', 'fsdjump', 'docked', 'scan', 'saasignalsfound', 'carrierjump')) and ('StarPos' in entry or this.coordinates)): @@ -2066,15 +2301,15 @@ def journal_entry( # noqa: C901, CCR001 this.eddn.export_journal_generic(cmdr, is_beta, filter_localised(entry)) except requests.exceptions.RequestException as e: - logger.debug('Failed in export_journal_entry', exc_info=e) + logger.debug('Failed in send_message', exc_info=e) return _("Error: Can't connect to EDDN") # LANG: Error while trying to send data to EDDN except Exception as e: - logger.debug('Failed in export_journal_entry', exc_info=e) + logger.debug('Failed in export_journal_generic', exc_info=e) return str(e) - elif (config.get_int('output') & config.OUT_MKT_EDDN and not state['Captain'] and - event_name in ('market', 'outfitting', 'shipyard')): + elif (config.get_int('output') & config.OUT_EDDN_SEND_STATION_DATA and not state['Captain'] and + event_name in ('market', 'outfitting', 'shipyard')): # Market.json, Outfitting.json or Shipyard.json to process try: @@ -2116,7 +2351,7 @@ def journal_entry( # noqa: C901, CCR001 return None -def cmdr_data(data: CAPIData, is_beta: bool) -> Optional[str]: # noqa: CCR001 +def cmdr_data(data: CAPIData, is_beta: bool) -> Optional[str]: """ Process new CAPI data. @@ -2125,7 +2360,7 @@ def cmdr_data(data: CAPIData, is_beta: bool) -> Optional[str]: # noqa: CCR001 :return: str - Error message, or `None` if no errors. """ if (data['commander'].get('docked') or (this.on_foot and monitor.station) - and config.get_int('output') & config.OUT_MKT_EDDN): + and config.get_int('output') & config.OUT_EDDN_SEND_STATION_DATA): try: if this.marketId != data['lastStarport']['id']: this.commodities = this.outfitting = this.shipyard = None diff --git a/plugins/edsm.py b/plugins/edsm.py index 8bd5fd30..8ddb7661 100644 --- a/plugins/edsm.py +++ b/plugins/edsm.py @@ -71,6 +71,9 @@ class This: def __init__(self): self.shutting_down = False # Plugin is shutting down. + self.game_version = "" + self.game_build = "" + self.session: requests.Session = requests.Session() self.session.headers['User-Agent'] = user_agent self.queue: Queue = Queue() # Items to be sent to EDSM by worker thread @@ -432,6 +435,9 @@ def journal_entry( # noqa: C901, CCR001 if should_return: return + this.game_version = state['GameVersion'] + this.game_build = state['GameBuild'] + entry = new_entry this.on_foot = state['OnFoot'] @@ -546,7 +552,7 @@ entry: {entry!r}''' materials.update(transient) logger.trace_if(CMDR_EVENTS, f'"LoadGame" event, queueing Materials: {cmdr=}') - this.queue.put((cmdr, materials)) + this.queue.put((cmdr, this.game_version, this.game_build, materials)) if entry['event'] in ('CarrierJump', 'FSDJump', 'Location', 'Docked'): logger.trace_if( @@ -555,7 +561,7 @@ Queueing: {entry!r}''' ) logger.trace_if(CMDR_EVENTS, f'"{entry["event"]=}" event, queueing: {cmdr=}') - this.queue.put((cmdr, entry)) + this.queue.put((cmdr, this.game_version, this.game_build, entry)) # Update system data @@ -638,6 +644,8 @@ def worker() -> None: # noqa: CCR001 C901 # Cant be broken up currently pending: List[Mapping[str, Any]] = [] # Unsent events closing = False cmdr: str = "" + last_game_version = "" + last_game_build = "" entry: Mapping[str, Any] = {} while not this.discarded_events: @@ -657,10 +665,10 @@ def worker() -> None: # noqa: CCR001 C901 # Cant be broken up currently logger.debug(f'{this.shutting_down=}, so setting closing = True') closing = True - item: Optional[Tuple[str, Mapping[str, Any]]] = this.queue.get() + item: Optional[Tuple[str, str, str, Mapping[str, Any]]] = this.queue.get() if item: - (cmdr, entry) = item - logger.trace_if(CMDR_EVENTS, f'De-queued ({cmdr=}, {entry["event"]=})') + (cmdr, game_version, game_build, entry) = item + logger.trace_if(CMDR_EVENTS, f'De-queued ({cmdr=}, {game_version=}, {game_build=}, {entry["event"]=})') else: logger.debug('Empty queue message, setting closing = True') @@ -686,6 +694,20 @@ def worker() -> None: # noqa: CCR001 C901 # Cant be broken up currently logger.trace_if( CMDR_EVENTS, f'({cmdr=}, {entry["event"]=}): not in discarded_events, appending to pending') + # Discard the pending list if it's a new Journal file OR + # if the gameversion has changed. We claim a single + # gameversion for an entire batch of events so can't mix + # them. + # The specific gameversion check caters for scenarios where + # we took some time in the last POST, had new events queued + # in the meantime *and* the game client crashed *and* was + # changed to a different gameversion. + if ( + entry['event'].lower() == 'fileheader' + or last_game_version != game_version or last_game_build != game_build + ): + pending = [] + pending.append(entry) # drop events if required by killswitch @@ -726,6 +748,8 @@ def worker() -> None: # noqa: CCR001 C901 # Cant be broken up currently 'apiKey': apikey, 'fromSoftware': applongname, 'fromSoftwareVersion': str(appversion()), + 'fromGameVersion': game_version, + 'fromGameBuild': game_build, 'message': json.dumps(pending, ensure_ascii=False).encode('utf-8'), } @@ -807,7 +831,7 @@ def worker() -> None: # noqa: CCR001 C901 # Cant be broken up currently plug.show_error(_("Error: Can't connect to EDSM")) if entry['event'].lower() in ('shutdown', 'commander', 'fileheader'): - # Game shutdown or new login so we MUST not hang on to pending + # Game shutdown or new login, so we MUST not hang on to pending pending = [] logger.trace_if(CMDR_EVENTS, f'Blanked pending because of event: {entry["event"]}') @@ -815,6 +839,9 @@ def worker() -> None: # noqa: CCR001 C901 # Cant be broken up currently logger.debug('closing, so returning.') return + last_game_version = game_version + last_game_build = game_build + logger.debug('Done.') diff --git a/plugins/inara.py b/plugins/inara.py index 5c09ec27..1a7cf1f6 100644 --- a/plugins/inara.py +++ b/plugins/inara.py @@ -28,6 +28,7 @@ import time import tkinter as tk from collections import OrderedDict, defaultdict, deque from dataclasses import dataclass +from datetime import datetime, timedelta, timezone from operator import itemgetter from threading import Lock, Thread from typing import TYPE_CHECKING, Any, Callable, Deque, Dict, List, Mapping, NamedTuple, Optional @@ -44,6 +45,7 @@ import timeout_session from companion import CAPIData from config import applongname, appversion, config, debug_senders from EDMCLogging import get_main_logger +from monitor import monitor from ttkHyperlinkLabel import HyperlinkLabel logger = get_main_logger() @@ -88,6 +90,11 @@ class This: def __init__(self): self.session = timeout_session.new_session() self.thread: Thread + self.parent: tk.Tk + + # Handle only sending Live galaxy data + self.legacy_galaxy_last_notified: Optional[datetime] = None + self.lastlocation = None # eventData from the last Commander's Flight Log event self.lastship = None # eventData from the last addCommanderShip or setCommanderShip event @@ -210,6 +217,7 @@ def plugin_start3(plugin_dir: str) -> str: def plugin_app(parent: tk.Tk) -> None: """Plugin UI setup Hook.""" + this.parent = parent this.system_link = parent.children['system'] # system label in main window this.station_link = parent.children['station'] # station label in main window this.system_link.bind_all('<>', update_location) @@ -361,6 +369,24 @@ def journal_entry( # noqa: C901, CCR001 :return: str - empty if no error, else error string. """ + if not monitor.is_live_galaxy(): + # This only applies after Update 14, which as of 2022-11-27 is scheduled + # for 2022-11-29, with the game servers presumably being down around + # 09:00 + if datetime.now(timezone.utc) >= datetime.fromisoformat("2022-11-27T09:00:00+00:00"): + # Update 14 ETA has passed, so perform the check + if ( + this.legacy_galaxy_last_notified is None + or (datetime.now(timezone.utc) - this.legacy_galaxy_last_notified) > timedelta(seconds=300) + ): + # LANG: The Inara API only accepts Live galaxy data, not Legacy galaxy data + logger.info(_("Inara only accepts Live galaxy data")) + # this.parent.children['status']['text'] = + this.legacy_galaxy_last_notified = datetime.now(timezone.utc) + return _("Inara only accepts Live galaxy data") + + return '' + should_return, new_entry = killswitch.check_killswitch('plugins.inara.journal', entry, logger) if should_return: plug.show_error(_('Inara disabled. See Log.')) # LANG: INARA support disabled via killswitch diff --git a/prefs.py b/prefs.py index 03b8dcd8..376b6c67 100644 --- a/prefs.py +++ b/prefs.py @@ -1221,7 +1221,9 @@ class PreferencesDialog(tk.Toplevel): (self.out_csv.get() and config.OUT_MKT_CSV) + (config.OUT_MKT_MANUAL if not self.out_auto.get() else 0) + (self.out_ship.get() and config.OUT_SHIP) + - (config.get_int('output') & (config.OUT_MKT_EDDN | config.OUT_SYS_EDDN | config.OUT_SYS_DELAY)) + (config.get_int('output') & ( + config.OUT_EDDN_SEND_STATION_DATA | config.OUT_EDDN_SEND_NON_STATION | config.OUT_EDDN_DELAY + )) ) config.set( diff --git a/pyproject.toml b/pyproject.toml index a7f41ad4..91c6e7e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,9 +7,25 @@ line_length = 119 [tool.pytest.ini_options] testpaths = ["tests"] # Search for tests in tests/ +addopts = "--cov . --cov plugins --cov-report=term-missing --no-cov-on-fail" +# --cov-fail-under 80" [tool.coverage.run] -omit = ["venv/*"] # when running pytest --cov, dont report coverage in venv directories +omit = [ "tests/*", "venv/*", "dist.win32/*" ] +plugins = [ "coverage_conditional_plugin" ] + +[tool.coverage.coverage_conditional_plugin.rules] +# NB: The name versus content of all of these are inverted because of the way +# they're used. When a pragma cites one it causes that code block to +# **NOT** be considered for code coverage. +# See Contributing.md#test-coverage for more details. +sys-platform-win32 = "sys_platform != 'win32'" +sys-platform-not-win32 = "sys_platform == 'win32'" +sys-platform-darwin = "sys_platform != 'darwin'" +sys-platform-not-darwin = "sys_platform == 'darwin'" +sys-platform-linux = "sys_platform != 'linux'" +sys-platform-not-linux = "sys_platform == 'linux'" +sys-platform-not-known = "sys_platform in ('darwin', 'linux', 'win32')" [tool.pyright] # pythonPlatform = 'Darwin' diff --git a/requirements-dev.txt b/requirements-dev.txt index 7fb4f90f..8da7c400 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,28 +5,28 @@ wheel # We can't rely on just picking this up from either the base (not venv), # or venv-init-time version. Specify here so that dependabot will prod us # about new versions. -setuptools==65.3.0 +setuptools==65.6.0 # Static analysis tools flake8==5.0.4 flake8-annotations-coverage==0.0.6 flake8-cognitive-complexity==0.1.0 -flake8-comprehensions==3.10.0 +flake8-comprehensions==3.10.1 flake8-docstrings==1.6.0 isort==5.10.1 -flake8-isort==4.2.0 +flake8-isort==5.0.3 flake8-json==21.7.0 flake8-noqa==1.2.9 flake8-polyfill==1.0.2 flake8-use-fstring==1.4 -mypy==0.971 +mypy==0.991 pep8-naming==0.13.2 -safety==2.2.0 -types-requests==2.28.11 +safety==2.3.1 +types-requests==2.28.11.5 # Code formatting tools -autopep8==1.7.0 +autopep8==2.0.0 # HTML changelogs grip==4.6.1 @@ -37,14 +37,15 @@ lxml==4.9.1 # We only need py2exe on windows. # Pre-release version addressing semantic_version 2.9.0+ issues: # -py2exe==0.12.0.1; sys_platform == 'win32' +py2exe==0.13.0.0; sys_platform == 'win32' # Testing -pytest==7.1.3 -pytest-cov==3.0.0 # Pytest code coverage support -coverage[toml]==6.4.4 # pytest-cov dep. This is here to ensure that it includes TOML support for pyproject.toml configs +pytest==7.2.0 +pytest-cov==4.0.0 # Pytest code coverage support +coverage[toml]==6.5.0 # pytest-cov dep. This is here to ensure that it includes TOML support for pyproject.toml configs +coverage-conditional-plugin==0.7.0 # For manipulating folder permissions and the like. -pywin32==304; sys_platform == 'win32' +pywin32==305; sys_platform == 'win32' # All of the normal requirements diff --git a/requirements.txt b/requirements.txt index 04102949..d014f67c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -certifi==2022.9.14 +certifi==2022.9.24 requests==2.28.1 watchdog==2.1.9 # Commented out because this doesn't package well with py2exe diff --git a/tests/journal_lock.py/test_journal_lock.py b/tests/journal_lock.py/test_journal_lock.py index 016a8f1b..72252e9f 100644 --- a/tests/journal_lock.py/test_journal_lock.py +++ b/tests/journal_lock.py/test_journal_lock.py @@ -8,7 +8,6 @@ import pytest # Import as other names else they get picked up when used as fixtures from _pytest import monkeypatch as _pytest_monkeypatch from _pytest import tmpdir as _pytest_tmpdir -from py._path.local import LocalPath as py_path_local_LocalPath from config import config from journal_lock import JournalLock, JournalLockResult @@ -120,7 +119,7 @@ class TestJournalLock: def mock_journaldir( self, monkeypatch: _pytest_monkeypatch, tmp_path_factory: _pytest_tmpdir.TempPathFactory - ) -> py_path_local_LocalPath: + ) -> _pytest_tmpdir.TempPathFactory: """Fixture for mocking config.get_str('journaldir').""" def get_str(key: str, *, default: str = None) -> str: """Mock config.*Config get_str to provide fake journaldir.""" @@ -139,7 +138,7 @@ class TestJournalLock: self, monkeypatch: _pytest_monkeypatch, tmp_path_factory: _pytest_tmpdir.TempPathFactory - ) -> py_path_local_LocalPath: + ) -> _pytest_tmpdir.TempPathFactory: """Fixture for mocking config.get_str('journaldir').""" def get_str(key: str, *, default: str = None) -> str: """Mock config.*Config get_str to provide fake journaldir.""" @@ -155,7 +154,7 @@ class TestJournalLock: ########################################################################### # Tests against JournalLock.__init__() - def test_journal_lock_init(self, mock_journaldir: py_path_local_LocalPath): + def test_journal_lock_init(self, mock_journaldir: _pytest_tmpdir.TempPathFactory): """Test JournalLock instantiation.""" print(f'{type(mock_journaldir)=}') tmpdir = str(mock_journaldir.getbasetemp()) @@ -177,7 +176,7 @@ class TestJournalLock: jlock.set_path_from_journaldir() assert jlock.journal_dir_path is None - def test_path_from_journaldir_with_tmpdir(self, mock_journaldir: py_path_local_LocalPath): + def test_path_from_journaldir_with_tmpdir(self, mock_journaldir: _pytest_tmpdir.TempPathFactory): """Test JournalLock.set_path_from_journaldir() with tmpdir.""" tmpdir = mock_journaldir @@ -201,7 +200,7 @@ class TestJournalLock: locked = jlock.obtain_lock() assert locked == JournalLockResult.JOURNALDIR_IS_NONE - def test_obtain_lock_with_tmpdir(self, mock_journaldir: py_path_local_LocalPath): + def test_obtain_lock_with_tmpdir(self, mock_journaldir: _pytest_tmpdir.TempPathFactory): """Test JournalLock.obtain_lock() with tmpdir.""" jlock = JournalLock() @@ -214,7 +213,7 @@ class TestJournalLock: assert jlock.release_lock() os.unlink(str(jlock.journal_dir_lockfile_name)) - def test_obtain_lock_with_tmpdir_ro(self, mock_journaldir: py_path_local_LocalPath): + def test_obtain_lock_with_tmpdir_ro(self, mock_journaldir: _pytest_tmpdir.TempPathFactory): """Test JournalLock.obtain_lock() with read-only tmpdir.""" tmpdir = str(mock_journaldir.getbasetemp()) print(f'{tmpdir=}') @@ -281,7 +280,7 @@ class TestJournalLock: assert locked == JournalLockResult.JOURNALDIR_READONLY - def test_obtain_lock_already_locked(self, mock_journaldir: py_path_local_LocalPath): + def test_obtain_lock_already_locked(self, mock_journaldir: _pytest_tmpdir.TempPathFactory): """Test JournalLock.obtain_lock() with tmpdir.""" continue_q: mp.Queue = mp.Queue() exit_q: mp.Queue = mp.Queue() @@ -313,7 +312,7 @@ class TestJournalLock: ########################################################################### # Tests against JournalLock.release_lock() - def test_release_lock(self, mock_journaldir: py_path_local_LocalPath): + def test_release_lock(self, mock_journaldir: _pytest_tmpdir.TempPathFactory): """Test JournalLock.release_lock().""" # First actually obtain the lock, and check it worked jlock = JournalLock() @@ -331,12 +330,12 @@ class TestJournalLock: # Cleanup, to avoid side-effect on other tests os.unlink(str(jlock.journal_dir_lockfile_name)) - def test_release_lock_not_locked(self, mock_journaldir: py_path_local_LocalPath): + def test_release_lock_not_locked(self, mock_journaldir: _pytest_tmpdir.TempPathFactory): """Test JournalLock.release_lock() when not locked.""" jlock = JournalLock() assert jlock.release_lock() - def test_release_lock_lie_locked(self, mock_journaldir: py_path_local_LocalPath): + def test_release_lock_lie_locked(self, mock_journaldir: _pytest_tmpdir.TempPathFactory): """Test JournalLock.release_lock() when not locked, but lie we are.""" jlock = JournalLock() jlock.locked = True @@ -346,7 +345,7 @@ class TestJournalLock: # Tests against JournalLock.update_lock() def test_update_lock( self, - mock_journaldir_changing: py_path_local_LocalPath): + mock_journaldir_changing: _pytest_tmpdir.TempPathFactory): """ Test JournalLock.update_lock(). @@ -374,7 +373,7 @@ class TestJournalLock: # And the old_journaldir's lockfile too os.unlink(str(old_journaldir_lockfile_name)) - def test_update_lock_same(self, mock_journaldir: py_path_local_LocalPath): + def test_update_lock_same(self, mock_journaldir: _pytest_tmpdir.TempPathFactory): """ Test JournalLock.update_lock().