From 9faae8b9bca0d66f9f3bb2639cf3b84d40d9591a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 28 Sep 2022 17:08:41 +0100 Subject: [PATCH 01/46] EDDN: Open & create sqlite3 db for replay * sqlite3 open, and creation of table. * Change `load_journal_replay()` to `load_journal_replay_file()` and change the semantics to just return the `list[str]` loaded from it. It also now catches no exceptions. * Remove the "lock the journal cache" on init as it's not necessary. There's still a lot more changes to come on this. --- plugins/eddn.py | 98 ++++++++++++++++++++++++++++++------------------- 1 file changed, 60 insertions(+), 38 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index cd1d53ff..50cb402c 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -26,16 +26,16 @@ import itertools import json import pathlib import re +import sqlite3 import sys import tkinter as tk from collections import OrderedDict from os import SEEK_SET -from os.path import join from platform import system from textwrap import dedent from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Mapping, MutableMapping, Optional from typing import OrderedDict as OrderedDictT -from typing import TextIO, Tuple, Union +from typing import Tuple, Union import requests @@ -52,10 +52,6 @@ from prefs import prefsVersion from ttkHyperlinkLabel import HyperlinkLabel from util import text -if sys.platform != 'win32': - from fcntl import LOCK_EX, LOCK_NB, lockf - - if TYPE_CHECKING: def _(x: str) -> str: return x @@ -155,8 +151,25 @@ class EDDN: self.parent: tk.Tk = parent self.session = requests.Session() self.session.headers['User-Agent'] = user_agent - self.replayfile: Optional[TextIO] = None # For delayed messages - self.replaylog: List[str] = [] + + ####################################################################### + # EDDN delayed sending/retry + ####################################################################### + self.replaydb = self.journal_replay_sqlite_init() + # Kept only for converting legacy file to sqlite3 + try: + replaylog = self.load_journal_replay_file() + + except FileNotFoundError: + pass + + finally: + # TODO: Convert `replaylog` into the database. + # Remove the file. + ... + + ####################################################################### + self.fss_signals: List[Mapping[str, Any]] = [] if config.eddn_url is not None: @@ -165,36 +178,49 @@ class EDDN: else: self.eddn_url = self.DEFAULT_URL - def load_journal_replay(self) -> bool: + def journal_replay_sqlite_init(self) -> sqlite3.Cursor: + """ + Ensure the sqlite3 database for EDDN replays exists and has schema. + + :return: sqlite3 cursor for the database. + """ + self.replaydb_conn = sqlite3.connect(config.app_dir_path / 'eddn_replay.db') + replaydb = self.replaydb_conn.cursor() + try: + replaydb.execute( + """ + CREATE TABLE messages + ( + id INT PRIMARY KEY NOT NULL, + created TEXT NOT NULL, + cmdr TEXT NOT NULL, + edmc_version TEXT, + game_version TEXT, + game_build TEXT, + message TEXT NOT NULL + ) + """ + ) + + except sqlite3.OperationalError as e: + if str(e) != "table messages already exists": + raise e + + return replaydb + + def load_journal_replay_file(self) -> list[str]: """ Load cached journal entries from disk. - :return: a bool indicating success + Simply let any exceptions propagate up if there's an error. + + :return: Contents of the file as a list. """ # Try to obtain exclusive access to the journal cache - filename = join(config.app_dir, 'replay.jsonl') - try: - try: - # Try to open existing file - self.replayfile = open(filename, 'r+', buffering=1) - - except FileNotFoundError: - self.replayfile = open(filename, 'w+', buffering=1) # Create file - - if sys.platform != 'win32': # open for writing is automatically exclusive on Windows - lockf(self.replayfile, LOCK_EX | LOCK_NB) - - except OSError: - logger.exception('Failed opening "replay.jsonl"') - if self.replayfile: - self.replayfile.close() - - self.replayfile = None - return False - - else: - self.replaylog = [line.strip() for line in self.replayfile] - return True + filename = config.app_dir_path / 'replay.jsonl' + # Try to open existing file + with open(filename, 'r+', buffering=1) as replay_file: + return [line.strip() for line in replay_file] def flush(self): """Flush the replay file, clearing any data currently there that is not in the replaylog list.""" @@ -762,7 +788,7 @@ class EDDN: :param entry: The full journal event dictionary (due to checks in this function). :param msg: The EDDN message body to be sent. """ - if self.replayfile or self.load_journal_replay(): + if self.replayfile or self.load_journal_replay_file(): # Store the entry self.replaylog.append(json.dumps([cmdr, msg])) self.replayfile.write(f'{self.replaylog[-1]}\n') # type: ignore @@ -1621,10 +1647,6 @@ def plugin_app(parent: tk.Tk) -> Optional[tk.Frame]: """ this.parent = parent this.eddn = EDDN(parent) - # Try to obtain exclusive lock on journal cache, even if we don't need it yet - if not this.eddn.load_journal_replay(): - # Shouldn't happen - don't bother localizing - this.parent.children['status']['text'] = 'Error: Is another copy of this app already running?' if config.eddn_tracking_ui: this.ui = tk.Frame(parent) From 072eadd89373de6117a8b348b3d26f04edec37f0 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 28 Sep 2022 17:17:18 +0100 Subject: [PATCH 02/46] EDDN: messages.id AUTOINCREMENT, and index created & cmdr We'll definitely want to query against `cmdr`, and possibly `created`. We shouldn't need to against other fields, they'll just be checked during processing of an already selected message. --- plugins/eddn.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 50cb402c..eee9e8d1 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -191,7 +191,7 @@ class EDDN: """ CREATE TABLE messages ( - id INT PRIMARY KEY NOT NULL, + id INTEGER PRIMARY KEY AUTOINCREMENT, created TEXT NOT NULL, cmdr TEXT NOT NULL, edmc_version TEXT, @@ -202,6 +202,24 @@ class EDDN: """ ) + replaydb.execute( + """ + CREATE INDEX messages_created ON messages + ( + created + ) + """ + ) + + replaydb.execute( + """ + CREATE INDEX messages_cmdr ON messages + ( + cmdr + ) + """ + ) + except sqlite3.OperationalError as e: if str(e) != "table messages already exists": raise e From 03e432034f3a1eb6f56250ee825f213169fc3070 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 28 Sep 2022 17:39:36 +0100 Subject: [PATCH 03/46] EDDN: Moving replay functionality into its own class --- plugins/eddn.py | 161 +++++++++++++++++++++++++----------------------- 1 file changed, 83 insertions(+), 78 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index eee9e8d1..b511be16 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -114,7 +114,6 @@ class This: this = This() - # This SKU is tagged on any module or ship that you must have Horizons for. HORIZONS_SKU = 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' # ELITE_HORIZONS_V_COBRA_MK_IV_1000` is for the Cobra Mk IV, but @@ -126,8 +125,84 @@ HORIZONS_SKU = 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' # one. -# TODO: a good few of these methods are static or could be classmethods. they should be created as such. +class EDDNReplay: + """Store and retry sending of EDDN messages.""" + SQLITE_DB_FILENAME = 'eddn_replay.db' + + def __init__(self) -> None: + """ + Prepare the system for processing messages. + + - Ensure the sqlite3 database for EDDN replays exists and has schema. + - Convert any legacy file into the database. + """ + self.db_conn = sqlite3.connect(config.app_dir_path / self.SQLITE_DB_FILENAME) + self.db = self.db_conn.cursor() + + try: + self.db.execute( + """ + CREATE TABLE messages + ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created TEXT NOT NULL, + cmdr TEXT NOT NULL, + edmc_version TEXT, + game_version TEXT, + game_build TEXT, + message TEXT NOT NULL + ) + """ + ) + + self.db.execute( + """ + CREATE INDEX messages_created ON messages + ( + created + ) + """ + ) + + self.db.execute( + """ + CREATE INDEX messages_cmdr ON messages + ( + cmdr + ) + """ + ) + + except sqlite3.OperationalError as e: + if str(e) != "table messages already exists": + raise e + + self.convert_legacy_file() + + def convert_legacy_file(self): + """Convert a legacy file's contents into the sqlite3 db.""" + try: + for m in self.load_legacy_file(): + ... + + except FileNotFoundError: + pass + + def load_legacy_file(self) -> list[str]: + """ + Load cached journal entries from disk. + + :return: Contents of the file as a list. + """ + # Try to obtain exclusive access to the journal cache + filename = config.app_dir_path / 'replay.jsonl' + # Try to open existing file + with open(filename, 'r+', buffering=1) as replay_file: + return [line.strip() for line in replay_file] + + +# TODO: a good few of these methods are static or could be classmethods. they should be created as such. class EDDN: """EDDN Data export.""" @@ -155,19 +230,7 @@ class EDDN: ####################################################################### # EDDN delayed sending/retry ####################################################################### - self.replaydb = self.journal_replay_sqlite_init() - # Kept only for converting legacy file to sqlite3 - try: - replaylog = self.load_journal_replay_file() - - except FileNotFoundError: - pass - - finally: - # TODO: Convert `replaylog` into the database. - # Remove the file. - ... - + self.replay = EDDNReplay() ####################################################################### self.fss_signals: List[Mapping[str, Any]] = [] @@ -178,68 +241,6 @@ class EDDN: else: self.eddn_url = self.DEFAULT_URL - def journal_replay_sqlite_init(self) -> sqlite3.Cursor: - """ - Ensure the sqlite3 database for EDDN replays exists and has schema. - - :return: sqlite3 cursor for the database. - """ - self.replaydb_conn = sqlite3.connect(config.app_dir_path / 'eddn_replay.db') - replaydb = self.replaydb_conn.cursor() - try: - replaydb.execute( - """ - CREATE TABLE messages - ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - created TEXT NOT NULL, - cmdr TEXT NOT NULL, - edmc_version TEXT, - game_version TEXT, - game_build TEXT, - message TEXT NOT NULL - ) - """ - ) - - replaydb.execute( - """ - CREATE INDEX messages_created ON messages - ( - created - ) - """ - ) - - replaydb.execute( - """ - CREATE INDEX messages_cmdr ON messages - ( - cmdr - ) - """ - ) - - except sqlite3.OperationalError as e: - if str(e) != "table messages already exists": - raise e - - return replaydb - - def load_journal_replay_file(self) -> list[str]: - """ - Load cached journal entries from disk. - - Simply let any exceptions propagate up if there's an error. - - :return: Contents of the file as a list. - """ - # Try to obtain exclusive access to the journal cache - filename = config.app_dir_path / 'replay.jsonl' - # Try to open existing file - with open(filename, 'r+', buffering=1) as replay_file: - return [line.strip() for line in replay_file] - def flush(self): """Flush the replay file, clearing any data currently there that is not in the replaylog list.""" if self.replayfile is None: @@ -361,6 +362,10 @@ class EDDN: def sendreplay(self) -> None: # noqa: CCR001 """Send cached Journal lines to EDDN.""" + # TODO: Convert to using the sqlite3 db + # **IF** this is moved to a thread worker then we need to ensure + # that we're operating sqlite3 in a thread-safe manner, + # Ref: if not self.replayfile: return # Probably closing app @@ -806,7 +811,7 @@ class EDDN: :param entry: The full journal event dictionary (due to checks in this function). :param msg: The EDDN message body to be sent. """ - if self.replayfile or self.load_journal_replay_file(): + if self.replayfile or self.journal_replay_load_file(): # Store the entry self.replaylog.append(json.dumps([cmdr, msg])) self.replayfile.write(f'{self.replaylog[-1]}\n') # type: ignore From 424d5f023c6b6c6fd404bf006e2a491961d3f58f Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 28 Sep 2022 18:11:12 +0100 Subject: [PATCH 04/46] EDDNReplay.add_message() is now functional And that includes the code to handle legacy `replay.json` messages. --- plugins/eddn.py | 68 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 54 insertions(+), 14 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index b511be16..9f0d7e61 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -180,27 +180,67 @@ class EDDNReplay: self.convert_legacy_file() + def add_message(self, cmdr, msg): + """ + Add an EDDN message to the database. + + `msg` absolutely needs to be the **FULL** EDDN message, including all + of `header`, `$schemaRef` and `message`. Code handling this not being + the case is only for loading the legacy `replay.json` file messages. + + :param cmdr: Name of the Commander that created this message. + :param msg: The full, transmission-ready, EDDN message. + """ + # Cater for legacy replay.json messages + if 'header' not in msg: + msg['header'] = { + # We have to lie and say it's *this* version, but denote that + # it might not actually be this version. + 'softwareName': f'{applongname} [{system() if sys.platform != "darwin" else "Mac OS"}]' + ' (legacy replay)', + 'softwareVersion': str(appversion_nobuild()), + 'uploaderID': cmdr, + 'gameversion': '', # Can't add what we don't know + 'gamebuild': '', # Can't add what we don't know + } + + created = msg['message']['timestamp'] + edmc_version = msg['header']['softwareVersion'] + game_version = msg['header']['gameversion'] + game_build = msg['header']['gamebuild'] + uploader = msg['header']['uploaderID'] + + try: + self.db.execute( + """ + INSERT INTO messages ( + created, cmdr, edmc_version, game_version, game_build, message + ) + VALUES ( + ?, ?, ?, ?, ?, ? + ) + """, + (created, uploader, edmc_version, game_version, game_build, json.dumps(msg)) + ) + self.db_conn.commit() + + except Exception: + logger.exception('EDDNReplay INSERT error') + def convert_legacy_file(self): """Convert a legacy file's contents into the sqlite3 db.""" try: - for m in self.load_legacy_file(): - ... + filename = config.app_dir_path / 'replay.jsonl' + with open(filename, 'r+', buffering=1) as replay_file: + for line in replay_file: + j = json.loads(line) + cmdr, msg = j + self.add_message(cmdr, msg) + break except FileNotFoundError: pass - def load_legacy_file(self) -> list[str]: - """ - Load cached journal entries from disk. - - :return: Contents of the file as a list. - """ - # Try to obtain exclusive access to the journal cache - filename = config.app_dir_path / 'replay.jsonl' - # Try to open existing file - with open(filename, 'r+', buffering=1) as replay_file: - return [line.strip() for line in replay_file] - # TODO: a good few of these methods are static or could be classmethods. they should be created as such. class EDDN: From 9a660b3b49de0a06458d8ced46d28bf30b94dcd3 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 29 Sep 2022 16:06:57 +0100 Subject: [PATCH 05/46] EDDN: New class as EDDNSender now, and version the queue database file * It makes more sense for this new class to be concerned with all the 'send it' functionality, not just 'replay', so rename it. * Athough we're trying to get the schema right *first* time, let's plan ahead and version the filename in case of needing to migrations in the future. --- plugins/eddn.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 9f0d7e61..9a0a3324 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -125,10 +125,10 @@ HORIZONS_SKU = 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' # one. -class EDDNReplay: +class EDDNSender: """Store and retry sending of EDDN messages.""" - SQLITE_DB_FILENAME = 'eddn_replay.db' + SQLITE_DB_FILENAME = 'eddn_queue-v1.db' def __init__(self) -> None: """ @@ -188,6 +188,8 @@ class EDDNReplay: of `header`, `$schemaRef` and `message`. Code handling this not being the case is only for loading the legacy `replay.json` file messages. + TODO: Return the unique row id of the added message. + :param cmdr: Name of the Commander that created this message. :param msg: The full, transmission-ready, EDDN message. """ @@ -267,11 +269,7 @@ class EDDN: self.session = requests.Session() self.session.headers['User-Agent'] = user_agent - ####################################################################### - # EDDN delayed sending/retry - ####################################################################### - self.replay = EDDNReplay() - ####################################################################### + self.sender = EDDNSender() self.fss_signals: List[Mapping[str, Any]] = [] From 80129361fe0c569d7093f0c7fd3137fe36c271fe Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 29 Sep 2022 16:30:09 +0100 Subject: [PATCH 06/46] EDDN: Refactor queue db open/creation to own function --- plugins/eddn.py | 45 +++++++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 9a0a3324..dfda017e 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -126,22 +126,37 @@ HORIZONS_SKU = 'ELITE_HORIZONS_V_PLANETARY_LANDINGS' class EDDNSender: - """Store and retry sending of EDDN messages.""" + """Handle sending of EDDN messages to the Gateway.""" - SQLITE_DB_FILENAME = 'eddn_queue-v1.db' + SQLITE_DB_FILENAME_V1 = 'eddn_queue-v1.db' - def __init__(self) -> None: + def __init__(self, eddn_endpoint) -> None: """ Prepare the system for processing messages. - Ensure the sqlite3 database for EDDN replays exists and has schema. - Convert any legacy file into the database. """ - self.db_conn = sqlite3.connect(config.app_dir_path / self.SQLITE_DB_FILENAME) + self.db_conn = self.sqlite_queue_v1() self.db = self.db_conn.cursor() + ####################################################################### + # Queue database migration + ####################################################################### + self.convert_legacy_file() + ####################################################################### + + def sqlite_queue_v1(self): + """ + Initialise a v1 EDDN queue database. + + :return: sqlite3 connection + """ + db_conn = sqlite3.connect(config.app_dir_path / self.SQLITE_DB_FILENAME_V1) + db = db_conn.cursor() + try: - self.db.execute( + db.execute( """ CREATE TABLE messages ( @@ -156,7 +171,7 @@ class EDDNSender: """ ) - self.db.execute( + db.execute( """ CREATE INDEX messages_created ON messages ( @@ -165,7 +180,7 @@ class EDDNSender: """ ) - self.db.execute( + db.execute( """ CREATE INDEX messages_cmdr ON messages ( @@ -176,9 +191,15 @@ class EDDNSender: except sqlite3.OperationalError as e: if str(e) != "table messages already exists": + # Cleanup, as schema creation failed + db.close() + db_conn.close() raise e - self.convert_legacy_file() + # We return only the connection, so tidy up + db.close() + + return db_conn def add_message(self, cmdr, msg): """ @@ -269,16 +290,16 @@ class EDDN: self.session = requests.Session() self.session.headers['User-Agent'] = user_agent - self.sender = EDDNSender() - - self.fss_signals: List[Mapping[str, Any]] = [] - if config.eddn_url is not None: self.eddn_url = config.eddn_url else: self.eddn_url = self.DEFAULT_URL + self.sender = EDDNSender(self.eddn_url) + + self.fss_signals: List[Mapping[str, Any]] = [] + def flush(self): """Flush the replay file, clearing any data currently there that is not in the replaylog list.""" if self.replayfile is None: From 089c33002c560cd16543679c9f304c07e5b3ac8b Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 29 Sep 2022 16:34:43 +0100 Subject: [PATCH 07/46] EDDNSender->add_message() returns ID of INSERTed row --- plugins/eddn.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index dfda017e..b71233c6 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -201,7 +201,7 @@ class EDDNSender: return db_conn - def add_message(self, cmdr, msg): + def add_message(self, cmdr, msg) -> int: """ Add an EDDN message to the database. @@ -209,10 +209,12 @@ class EDDNSender: of `header`, `$schemaRef` and `message`. Code handling this not being the case is only for loading the legacy `replay.json` file messages. - TODO: Return the unique row id of the added message. + NB: Although `cmdr` *should* be the same as `msg->header->uploaderID` + we choose not to assume that. :param cmdr: Name of the Commander that created this message. :param msg: The full, transmission-ready, EDDN message. + :return: ID of the successfully inserted row. """ # Cater for legacy replay.json messages if 'header' not in msg: @@ -250,6 +252,8 @@ class EDDNSender: except Exception: logger.exception('EDDNReplay INSERT error') + return self.db.lastrowid + def convert_legacy_file(self): """Convert a legacy file's contents into the sqlite3 db.""" try: From 86ff787aed0723c39278e277ee6fd449ab7c273d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 29 Sep 2022 16:39:30 +0100 Subject: [PATCH 08/46] EDDNSender: Convert all of a legacy file * I had a `break` in there to only convert the first message. * Also collapsed the assignment to `cmdr, msg` to not go via `j`. --- plugins/eddn.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index b71233c6..9a08bdb2 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -260,10 +260,8 @@ class EDDNSender: filename = config.app_dir_path / 'replay.jsonl' with open(filename, 'r+', buffering=1) as replay_file: for line in replay_file: - j = json.loads(line) - cmdr, msg = j + cmdr, msg = json.loads(line) self.add_message(cmdr, msg) - break except FileNotFoundError: pass From 09f646a249c7f2449df93f10af1d95d416f2b3ca Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 29 Sep 2022 16:59:34 +0100 Subject: [PATCH 09/46] EDDNSender: Add delete_message() method This was tested by temporary code in `convert_legacy_file()` to delete the last added row once all done. --- plugins/eddn.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/plugins/eddn.py b/plugins/eddn.py index 9a08bdb2..2266d4ad 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -254,6 +254,20 @@ class EDDNSender: return self.db.lastrowid + def delete_message(self, row_id: int) -> None: + """ + Delete a queued message by row id. + + :param row_id: + """ + self.db.execute( + """ + DELETE FROM messages WHERE id = :row_id + """, + {'row_id': row_id} + ) + self.db_conn.commit() + def convert_legacy_file(self): """Convert a legacy file's contents into the sqlite3 db.""" try: From c1793ad8399f45b5e99ce7303e23e51ae1e7b310 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 29 Sep 2022 17:01:50 +0100 Subject: [PATCH 10/46] EDDN: Remove EDDN->flush() --- plugins/eddn.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 2266d4ad..0dc8b101 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -316,19 +316,6 @@ class EDDN: self.fss_signals: List[Mapping[str, Any]] = [] - def flush(self): - """Flush the replay file, clearing any data currently there that is not in the replaylog list.""" - if self.replayfile is None: - logger.error('replayfile is None!') - return - - self.replayfile.seek(0, SEEK_SET) - self.replayfile.truncate() - for line in self.replaylog: - self.replayfile.write(f'{line}\n') - - self.replayfile.flush() - def close(self): """Close down the EDDN class instance.""" logger.debug('Closing replayfile...') From 51fb90b999ea0a2f7bdaa08c14ce27841470c898 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 29 Sep 2022 17:04:34 +0100 Subject: [PATCH 11/46] EDDN: Change EDDN.close() to call into EDDNSender.close() --- plugins/eddn.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 0dc8b101..ac71bf8a 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -201,6 +201,14 @@ class EDDNSender: return db_conn + def close(self) -> None: + """Clean up any resources.""" + if self.db: + self.db.close() + + if self.db_conn: + self.db_conn.close() + def add_message(self, cmdr, msg) -> int: """ Add an EDDN message to the database. @@ -318,11 +326,10 @@ class EDDN: def close(self): """Close down the EDDN class instance.""" - logger.debug('Closing replayfile...') - if self.replayfile: - self.replayfile.close() + logger.debug('Closing Sender...') + if self.sender: + self.sender.close() - self.replayfile = None logger.debug('Done.') logger.debug('Closing EDDN requests.Session.') From f3017d40ec1c400fdcf62f0a11189a8f25899a43 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 29 Sep 2022 17:11:49 +0100 Subject: [PATCH 12/46] EDDNSender: Fill out type hints --- plugins/eddn.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index ac71bf8a..a5755b48 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -30,7 +30,6 @@ import sqlite3 import sys import tkinter as tk from collections import OrderedDict -from os import SEEK_SET from platform import system from textwrap import dedent from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Mapping, MutableMapping, Optional @@ -130,13 +129,17 @@ class EDDNSender: SQLITE_DB_FILENAME_V1 = 'eddn_queue-v1.db' - def __init__(self, eddn_endpoint) -> None: + def __init__(self, eddn_endpoint: str) -> None: """ Prepare the system for processing messages. - Ensure the sqlite3 database for EDDN replays exists and has schema. - Convert any legacy file into the database. + + :param eddn_endpoint: Where messages should be sent. """ + self.eddn_endpoint = eddn_endpoint + self.db_conn = self.sqlite_queue_v1() self.db = self.db_conn.cursor() @@ -146,7 +149,7 @@ class EDDNSender: self.convert_legacy_file() ####################################################################### - def sqlite_queue_v1(self): + def sqlite_queue_v1(self) -> sqlite3.Connection: """ Initialise a v1 EDDN queue database. @@ -209,7 +212,7 @@ class EDDNSender: if self.db_conn: self.db_conn.close() - def add_message(self, cmdr, msg) -> int: + def add_message(self, cmdr: str, msg: dict) -> int: """ Add an EDDN message to the database. @@ -266,7 +269,7 @@ class EDDNSender: """ Delete a queued message by row id. - :param row_id: + :param row_id: id of message to be deleted. """ self.db.execute( """ From 0e20f4bc00ed7f71629b0b47c35f1732fc64a96a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 29 Sep 2022 17:18:39 +0100 Subject: [PATCH 13/46] EDDNSender: Remove legacy file after migration --- plugins/eddn.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index a5755b48..f09a4544 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -24,6 +24,7 @@ # ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# import itertools import json +import os import pathlib import re import sqlite3 @@ -281,8 +282,8 @@ class EDDNSender: def convert_legacy_file(self): """Convert a legacy file's contents into the sqlite3 db.""" + filename = config.app_dir_path / 'replay.jsonl' try: - filename = config.app_dir_path / 'replay.jsonl' with open(filename, 'r+', buffering=1) as replay_file: for line in replay_file: cmdr, msg = json.loads(line) @@ -291,6 +292,13 @@ class EDDNSender: except FileNotFoundError: pass + finally: + # Best effort at removing the file/contents + # NB: The legacy code assumed it could write to the file. + replay_file = open(filename, 'w') # Will truncate + replay_file.close() + os.unlink(filename) + # TODO: a good few of these methods are static or could be classmethods. they should be created as such. class EDDN: From 2b957d140cc66aba6a2487a7bb7e28cf58a5923d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 30 Sep 2022 13:35:50 +0100 Subject: [PATCH 14/46] EDDNSender: `convert_legacy_file()` belongs with "open the database" --- plugins/eddn.py | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index f09a4544..505039c4 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -136,6 +136,7 @@ class EDDNSender: - Ensure the sqlite3 database for EDDN replays exists and has schema. - Convert any legacy file into the database. + - (Future) Handle any database migrations. :param eddn_endpoint: Where messages should be sent. """ @@ -205,6 +206,25 @@ class EDDNSender: return db_conn + def convert_legacy_file(self): + """Convert a legacy file's contents into the sqlite3 db.""" + filename = config.app_dir_path / 'replay.jsonl' + try: + with open(filename, 'r+', buffering=1) as replay_file: + for line in replay_file: + cmdr, msg = json.loads(line) + self.add_message(cmdr, msg) + + except FileNotFoundError: + pass + + finally: + # Best effort at removing the file/contents + # NB: The legacy code assumed it could write to the file. + replay_file = open(filename, 'w') # Will truncate + replay_file.close() + os.unlink(filename) + def close(self) -> None: """Clean up any resources.""" if self.db: @@ -280,25 +300,6 @@ class EDDNSender: ) self.db_conn.commit() - def convert_legacy_file(self): - """Convert a legacy file's contents into the sqlite3 db.""" - filename = config.app_dir_path / 'replay.jsonl' - try: - with open(filename, 'r+', buffering=1) as replay_file: - for line in replay_file: - cmdr, msg = json.loads(line) - self.add_message(cmdr, msg) - - except FileNotFoundError: - pass - - finally: - # Best effort at removing the file/contents - # NB: The legacy code assumed it could write to the file. - replay_file = open(filename, 'w') # Will truncate - replay_file.close() - os.unlink(filename) - # TODO: a good few of these methods are static or could be classmethods. they should be created as such. class EDDN: From f66a98464ee03a9f8a6b3c2e26562ca9beb8b463 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 30 Sep 2022 16:10:25 +0100 Subject: [PATCH 15/46] EDDNSender: Closer to actually sending messages now --- plugins/eddn.py | 357 +++++++++++++++++++++++++----------------------- 1 file changed, 183 insertions(+), 174 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 505039c4..22152f83 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -22,6 +22,7 @@ # # ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# # ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# ! $# +import http import itertools import json import os @@ -129,8 +130,14 @@ class EDDNSender: """Handle sending of EDDN messages to the Gateway.""" SQLITE_DB_FILENAME_V1 = 'eddn_queue-v1.db' + TIMEOUT = 10 # requests timeout + UNKNOWN_SCHEMA_RE = re.compile( + r"^FAIL: \[JsonValidationException\('Schema " + r"https://eddn.edcd.io/schemas/(?P.+)/(?P[0-9]+) is unknown, " + r"unable to validate.',\)\]$" + ) - def __init__(self, eddn_endpoint: str) -> None: + def __init__(self, eddn: 'EDDN', eddn_endpoint: str) -> None: """ Prepare the system for processing messages. @@ -138,9 +145,13 @@ class EDDNSender: - Convert any legacy file into the database. - (Future) Handle any database migrations. + :param eddn: Reference to the `EDDN` instance this is for. :param eddn_endpoint: Where messages should be sent. """ + self.eddn = eddn self.eddn_endpoint = eddn_endpoint + self.session = requests.Session() + self.session.headers['User-Agent'] = user_agent self.db_conn = self.sqlite_queue_v1() self.db = self.db_conn.cursor() @@ -224,7 +235,7 @@ class EDDNSender: replay_file = open(filename, 'w') # Will truncate replay_file.close() os.unlink(filename) - + def close(self) -> None: """Clean up any resources.""" if self.db: @@ -263,8 +274,8 @@ class EDDNSender: created = msg['message']['timestamp'] edmc_version = msg['header']['softwareVersion'] - game_version = msg['header']['gameversion'] - game_build = msg['header']['gamebuild'] + game_version = msg['header'].get('gameversion', '') + game_build = msg['header'].get('gamebuild', '') uploader = msg['header']['uploaderID'] try: @@ -300,121 +311,103 @@ class EDDNSender: ) self.db_conn.commit() - -# TODO: a good few of these methods are static or could be classmethods. they should be created as such. -class EDDN: - """EDDN Data export.""" - - DEFAULT_URL = 'https://eddn.edcd.io:4430/upload/' - if 'eddn' in debug_senders: - DEFAULT_URL = f'http://{edmc_data.DEBUG_WEBSERVER_HOST}:{edmc_data.DEBUG_WEBSERVER_PORT}/eddn' - - REPLAYPERIOD = 400 # Roughly two messages per second, accounting for send delays [ms] - REPLAYFLUSH = 20 # Update log on disk roughly every 10 seconds - TIMEOUT = 10 # requests timeout - MODULE_RE = re.compile(r'^Hpt_|^Int_|Armour_', re.IGNORECASE) - CANONICALISE_RE = re.compile(r'\$(.+)_name;') - UNKNOWN_SCHEMA_RE = re.compile( - r"^FAIL: \[JsonValidationException\('Schema " - r"https://eddn.edcd.io/schemas/(?P.+)/(?P[0-9]+) is unknown, " - r"unable to validate.',\)\]$" - ) - CAPI_LOCALISATION_RE = re.compile(r'^loc[A-Z].+') - - def __init__(self, parent: tk.Tk): - self.parent: tk.Tk = parent - self.session = requests.Session() - self.session.headers['User-Agent'] = user_agent - - if config.eddn_url is not None: - self.eddn_url = config.eddn_url - - else: - self.eddn_url = self.DEFAULT_URL - - self.sender = EDDNSender(self.eddn_url) - - self.fss_signals: List[Mapping[str, Any]] = [] - - def close(self): - """Close down the EDDN class instance.""" - logger.debug('Closing Sender...') - if self.sender: - self.sender.close() - - logger.debug('Done.') - - logger.debug('Closing EDDN requests.Session.') - self.session.close() - - def send(self, cmdr: str, msg: Mapping[str, Any]) -> None: + def send_message_by_id(self, id: int): """ - Send sends an update to EDDN. + Transmit the message identified by the given ID. - :param cmdr: the CMDR to use as the uploader ID. - :param msg: the payload to send. + :param id: + :return: """ - should_return, new_data = killswitch.check_killswitch('plugins.eddn.send', msg) + self.db.execute( + """ + SELECT * FROM messages WHERE id = :row_id + """, + {'row_id': id} + ) + row = dict(zip([c[0] for c in self.db.description], self.db.fetchone())) + + try: + self.send_message(row['message']) + + except requests.exceptions.HTTPError as e: + logger.warning(f"HTTPError: {str(e)}") + + finally: + # Remove from queue + ... + + + def send_message(self, msg: str) -> bool: + """ + Transmit a fully-formed EDDN message to the Gateway. + + Should catch and handle all failure conditions. A `True` return might + mean that the message was successfully sent, *or* that this message + should not be retried after a failure, i.e. too large. + + :param msg: Fully formed, string, message. + :return: `True` for "now remove this message from the queue" + """ + should_return, new_data = killswitch.check_killswitch('plugins.eddn.send', json.loads(msg)) if should_return: logger.warning('eddn.send has been disabled via killswitch. Returning.') return - msg = new_data + msg = json.dumps(new_data) - uploader_id = cmdr - - to_send: OrderedDictT[str, OrderedDict[str, Any]] = OrderedDict([ - ('$schemaRef', msg['$schemaRef']), - ('header', OrderedDict([ - ('softwareName', f'{applongname} [{system() if sys.platform != "darwin" else "Mac OS"}]'), - ('softwareVersion', str(appversion_nobuild())), - ('uploaderID', uploader_id), - ])), - ('message', msg['message']), - ]) - - # About the smallest request is going to be (newlines added for brevity): - # {"$schemaRef":"https://eddn.edcd.io/schemas/commodity/3","header":{"softwareName":"E:D Market - # Connector Windows","softwareVersion":"5.3.0-beta4extra","uploaderID":"abcdefghijklm"},"messag - # e":{"systemName":"delphi","stationName":"The Oracle","marketId":128782803,"timestamp":"2022-0 - # 1-26T12:00:00Z","commodities":[]}} - # - # Which comes to 315 bytes (including \n) and compresses to 244 bytes. So lets just compress everything - - encoded, compressed = text.gzip(json.dumps(to_send, separators=(',', ':')), max_size=0) + status: tk.Widget = self.eddn.parent.children['status'] + # Even the smallest possible message compresses somewhat, so always compress + encoded, compressed = text.gzip(json.dumps(msg, separators=(',', ':')), max_size=0) headers: None | dict[str, str] = None if compressed: headers = {'Content-Encoding': 'gzip'} - r = self.session.post(self.eddn_url, data=encoded, timeout=self.TIMEOUT, headers=headers) - if r.status_code != requests.codes.ok: + try: + r = self.session.post(self.eddn_endpoint, data=encoded, timeout=self.TIMEOUT, headers=headers) + if r.status_code == requests.codes.ok: + return True - # Check if EDDN is still objecting to an empty commodities list - if ( - r.status_code == 400 - and msg['$schemaRef'] == 'https://eddn.edcd.io/schemas/commodity/3' - and msg['message']['commodities'] == [] - and r.text == "FAIL: []" - ): - logger.trace_if('plugin.eddn', "EDDN is still objecting to empty commodities data") - return # We want to silence warnings otherwise - - if r.status_code == 413: + if r.status_code == http.HTTPStatus.REQUEST_ENTITY_TOO_LARGE: extra_data = { - 'schema_ref': msg.get('$schemaRef', 'Unset $schemaRef!'), + 'schema_ref': new_data.get('$schemaRef', 'Unset $schemaRef!'), 'sent_data_len': str(len(encoded)), } if '/journal/' in extra_data['schema_ref']: - extra_data['event'] = msg.get('message', {}).get('event', 'No Event Set') + extra_data['event'] = new_data.get('message', {}).get('event', 'No Event Set') - self._log_response(r, header_msg='Got a 413 while POSTing data', **extra_data) - return # drop the error + self._log_response(r, header_msg='Got "Payload Too Large" while POSTing data', **extra_data) + return True - if not self.UNKNOWN_SCHEMA_RE.match(r.text): - self._log_response(r, header_msg='Status from POST wasn\'t 200 (OK)') + self._log_response(r, header_msg="Status from POST wasn't 200 (OK)") + r.raise_for_status() - r.raise_for_status() + except requests.exceptions.HTTPError as e: + if unknown_schema := self.UNKNOWN_SCHEMA_RE.match(e.response.text): + logger.debug(f"EDDN doesn't (yet?) know about schema: {unknown_schema['schema_name']}" + f"/{unknown_schema['schema_version']}") + # This dropping is to cater for the time period when EDDN doesn't *yet* support a new schema. + return True + + elif e.response.status_code == http.HTTPStatus.BAD_REQUEST: + # EDDN straight up says no, so drop the message + logger.debug(f"EDDN responded '400 Bad Request' to the message, dropping:\n{msg!r}") + return True + + else: + # This should catch anything else, e.g. timeouts, gateway errors + status['text'] = self.http_error_to_log(e) + + except requests.exceptions.RequestException as e: + logger.debug('Failed sending', exc_info=e) + # LANG: Error while trying to send data to EDDN + status['text'] = _("Error: Can't connect to EDDN") + + except Exception as e: + logger.debug('Failed sending', exc_info=e) + status['text'] = str(e) + + return False def _log_response( self, @@ -441,6 +434,92 @@ class EDDN: Content :\t{response.text} ''')+additional_data) + @staticmethod + def http_error_to_log(exception: requests.exceptions.HTTPError) -> str: + """Convert an exception from raise_for_status to a log message and displayed error.""" + status_code = exception.errno + + if status_code == 429: # HTTP UPGRADE REQUIRED + logger.warning('EDMC is sending schemas that are too old') + # LANG: EDDN has banned this version of our client + return _('EDDN Error: EDMC is too old for EDDN. Please update.') + + elif status_code == 400: + # we a validation check or something else. + logger.warning(f'EDDN Error: {status_code} -- {exception.response}') + # LANG: EDDN returned an error that indicates something about what we sent it was wrong + return _('EDDN Error: Validation Failed (EDMC Too Old?). See Log') + + else: + logger.warning(f'Unknown status code from EDDN: {status_code} -- {exception.response}') + # LANG: EDDN returned some sort of HTTP error, one we didn't expect. {STATUS} contains a number + return _('EDDN Error: Returned {STATUS} status code').format(STATUS=status_code) + + +# TODO: a good few of these methods are static or could be classmethods. they should be created as such. +class EDDN: + """EDDN Data export.""" + + DEFAULT_URL = 'https://eddn.edcd.io:4430/upload/' + if 'eddn' in debug_senders: + DEFAULT_URL = f'http://{edmc_data.DEBUG_WEBSERVER_HOST}:{edmc_data.DEBUG_WEBSERVER_PORT}/eddn' + + REPLAYPERIOD = 400 # Roughly two messages per second, accounting for send delays [ms] + REPLAYFLUSH = 20 # Update log on disk roughly every 10 seconds + MODULE_RE = re.compile(r'^Hpt_|^Int_|Armour_', re.IGNORECASE) + CANONICALISE_RE = re.compile(r'\$(.+)_name;') + CAPI_LOCALISATION_RE = re.compile(r'^loc[A-Z].+') + + def __init__(self, parent: tk.Tk): + self.parent: tk.Tk = parent + + if config.eddn_url is not None: + self.eddn_url = config.eddn_url + + else: + self.eddn_url = self.DEFAULT_URL + + self.sender = EDDNSender(self, self.eddn_url) + + self.fss_signals: List[Mapping[str, Any]] = [] + + def close(self): + """Close down the EDDN class instance.""" + logger.debug('Closing Sender...') + if self.sender: + self.sender.close() + + logger.debug('Done.') + + logger.debug('Closing EDDN requests.Session.') + self.session.close() + + def send(self, cmdr: str, msg: Mapping[str, Any]) -> None: + """ + Enqueue a message for transmission. + + :param cmdr: the CMDR to use as the uploader ID. + :param msg: the payload to send. + """ + to_send: OrderedDictT[str, OrderedDict[str, Any]] = OrderedDict([ + ('$schemaRef', msg['$schemaRef']), + ('header', OrderedDict([ + ('softwareName', f'{applongname} [{system() if sys.platform != "darwin" else "Mac OS"}]'), + ('softwareVersion', str(appversion_nobuild())), + ('uploaderID', cmdr), + # TODO: Add `gameversion` and `gamebuild` if that change is live + # on EDDN. + ])), + ('message', msg['message']), + ]) + + # Ensure it's en-queued + msg_id = self.sender.add_message(cmdr, to_send) + # Now try to transmit it immediately + if self.sender.send_message_by_id(msg_id): + # De-queue + self.sender.delete_message(msg_id) + def sendreplay(self) -> None: # noqa: CCR001 """Send cached Journal lines to EDDN.""" # TODO: Convert to using the sqlite3 db @@ -493,63 +572,13 @@ class EDDN: 'https://eddn.edcd.io/schemas/' ) - try: - self.send(cmdr, msg) - self.replaylog.pop(0) - if not len(self.replaylog) % self.REPLAYFLUSH: - self.flush() - - except requests.exceptions.HTTPError as e: - if unknown_schema := self.UNKNOWN_SCHEMA_RE.match(e.response.text): - logger.debug(f"EDDN doesn't (yet?) know about schema: {unknown_schema['schema_name']}" - f"/{unknown_schema['schema_version']}") - # NB: This dropping is to cater for the time when EDDN - # doesn't *yet* support a new schema. - self.replaylog.pop(0) # Drop the message - self.flush() # Truncates the file, then writes the extant data - - elif e.response.status_code == 400: - # EDDN straight up says no, so drop the message - logger.debug(f"EDDN responded '400' to the message, dropping:\n{msg!r}") - self.replaylog.pop(0) # Drop the message - self.flush() # Truncates the file, then writes the extant data - - else: - status['text'] = self.http_error_to_log(e) - - except requests.exceptions.RequestException as e: - logger.debug('Failed sending', exc_info=e) - # LANG: Error while trying to send data to EDDN - status['text'] = _("Error: Can't connect to EDDN") - return # stop sending - - except Exception as e: - logger.debug('Failed sending', exc_info=e) - status['text'] = str(e) - return # stop sending + self.send(cmdr, msg) + self.replaylog.pop(0) + if not len(self.replaylog) % self.REPLAYFLUSH: + self.flush() self.parent.after(self.REPLAYPERIOD, self.sendreplay) - @staticmethod - def http_error_to_log(exception: requests.exceptions.HTTPError) -> str: - """Convert an exception from raise_for_status to a log message and displayed error.""" - status_code = exception.errno - - if status_code == 429: # HTTP UPGRADE REQUIRED - logger.warning('EDMC is sending schemas that are too old') - # LANG: EDDN has banned this version of our client - return _('EDDN Error: EDMC is too old for EDDN. Please update.') - - elif status_code == 400: - # we a validation check or something else. - logger.warning(f'EDDN Error: {status_code} -- {exception.response}') - # LANG: EDDN returned an error that indicates something about what we sent it was wrong - return _('EDDN Error: Validation Failed (EDMC Too Old?). See Log') - - else: - logger.warning(f'Unknown status code from EDDN: {status_code} -- {exception.response}') - # LANG: EDDN returned some sort of HTTP error, one we didn't expect. {STATUS} contains a number - return _('EDDN Error: Returned {STATUS} status code').format(STATUS=status_code) def export_commodities(self, data: Mapping[str, Any], is_beta: bool) -> None: # noqa: CCR001 """ @@ -883,33 +912,13 @@ class EDDN: def export_journal_entry(self, cmdr: str, entry: Mapping[str, Any], msg: Mapping[str, Any]) -> None: """ - Update EDDN with an event from the journal. - - Additionally if other lines have been saved for retry, it may send - those as well. + Send a Journal-sourced EDDN message. :param cmdr: Commander name as passed in through `journal_entry()`. :param entry: The full journal event dictionary (due to checks in this function). :param msg: The EDDN message body to be sent. """ - if self.replayfile or self.journal_replay_load_file(): - # Store the entry - self.replaylog.append(json.dumps([cmdr, msg])) - self.replayfile.write(f'{self.replaylog[-1]}\n') # type: ignore - - if ( - entry['event'] == 'Docked' or (entry['event'] == 'Location' and entry['Docked']) or not - (config.get_int('output') & config.OUT_SYS_DELAY) - ): - self.parent.after(self.REPLAYPERIOD, self.sendreplay) # Try to send this and previous entries - - else: - # Can't access replay file! Send immediately. - # LANG: Status text shown while attempting to send data - self.parent.children['status']['text'] = _('Sending data to EDDN...') - self.parent.update_idletasks() - self.send(cmdr, msg) - self.parent.children['status']['text'] = '' + self.send(cmdr, msg) def export_journal_generic(self, cmdr: str, is_beta: bool, entry: Mapping[str, Any]) -> None: """ From 598e54eaa4e869cd07608e7ebc5646e8355f98dd Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 30 Sep 2022 17:02:43 +0100 Subject: [PATCH 16/46] EDDNSender: Now properly sends messages to Gateway Including removing from the queue if it succeeded, or didn't and should be dropped. --- plugins/eddn.py | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 22152f83..6f867fcd 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -293,9 +293,11 @@ class EDDNSender: self.db_conn.commit() except Exception: - logger.exception('EDDNReplay INSERT error') + logger.exception('INSERT error') + # Can't possibly be a valid row id + return -1 - return self.db.lastrowid + return self.db.lastrowid or -1 def delete_message(self, row_id: int) -> None: """ @@ -327,17 +329,16 @@ class EDDNSender: row = dict(zip([c[0] for c in self.db.description], self.db.fetchone())) try: - self.send_message(row['message']) + if self.send_message(row['message']): + self.delete_message(id) + return True except requests.exceptions.HTTPError as e: logger.warning(f"HTTPError: {str(e)}") - finally: - # Remove from queue - ... + return False - - def send_message(self, msg: str) -> bool: + def send_message(self, msg: str) -> bool: # noqa: CCR001 """ Transmit a fully-formed EDDN message to the Gateway. @@ -351,13 +352,11 @@ class EDDNSender: should_return, new_data = killswitch.check_killswitch('plugins.eddn.send', json.loads(msg)) if should_return: logger.warning('eddn.send has been disabled via killswitch. Returning.') - return - - msg = json.dumps(new_data) + return False status: tk.Widget = self.eddn.parent.children['status'] # Even the smallest possible message compresses somewhat, so always compress - encoded, compressed = text.gzip(json.dumps(msg, separators=(',', ':')), max_size=0) + encoded, compressed = text.gzip(json.dumps(new_data, separators=(',', ':')), max_size=0) headers: None | dict[str, str] = None if compressed: headers = {'Content-Encoding': 'gzip'} @@ -514,13 +513,12 @@ class EDDN: ]) # Ensure it's en-queued - msg_id = self.sender.add_message(cmdr, to_send) - # Now try to transmit it immediately - if self.sender.send_message_by_id(msg_id): - # De-queue - self.sender.delete_message(msg_id) + if (msg_id := self.sender.add_message(cmdr, to_send)) == -1: + return - def sendreplay(self) -> None: # noqa: CCR001 + self.sender.send_message_by_id(msg_id) + + def sendreplay(self) -> None: """Send cached Journal lines to EDDN.""" # TODO: Convert to using the sqlite3 db # **IF** this is moved to a thread worker then we need to ensure @@ -579,7 +577,6 @@ class EDDN: self.parent.after(self.REPLAYPERIOD, self.sendreplay) - def export_commodities(self, data: Mapping[str, Any], is_beta: bool) -> None: # noqa: CCR001 """ Update EDDN with the commodities on the current (lastStarport) station. From 3a57b53bbde003fb5a734daaa2d19b75ae5898e9 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 5 Oct 2022 15:11:37 +0100 Subject: [PATCH 17/46] config/EDDN: Rename OUT_SYS_DELAY to OUT_EDDN_DO_NOT_DELAY The sense of this `output` flag has been inverted (always?) for a long time. 1. I have the option "Delay sending until docked" showing as *off* in the UI. 2. My config.output value is `100000000001`. 3. The value of this flag is `4096`, which means 12th bit (starting from 1, not zero). 4. So I have the bit set, but the option visibly off. So, rename this both to be more pertinent to its use *and* to be correct as to what `True` for it means. --- config/__init__.py | 2 +- plugins/eddn.py | 10 ++++++++-- prefs.py | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/config/__init__.py b/config/__init__.py index 56679019..b7437b2d 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -176,7 +176,7 @@ class AbstractConfig(abc.ABC): # OUT_SYS_AUTO = 512 # Now always automatic OUT_MKT_MANUAL = 1024 OUT_SYS_EDDN = 2048 - OUT_SYS_DELAY = 4096 + OUT_EDDN_DO_NOT_DELAY = 4096 app_dir_path: pathlib.Path plugin_dir_path: pathlib.Path diff --git a/plugins/eddn.py b/plugins/eddn.py index 6f867fcd..f5469c54 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -349,6 +349,9 @@ class EDDNSender: :param msg: Fully formed, string, message. :return: `True` for "now remove this message from the queue" """ + + # TODO: Check if user options require us to send at this time. + should_return, new_data = killswitch.check_killswitch('plugins.eddn.send', json.loads(msg)) if should_return: logger.warning('eddn.send has been disabled via killswitch. Returning.') @@ -500,6 +503,9 @@ class EDDN: :param cmdr: the CMDR to use as the uploader ID. :param msg: the payload to send. """ + + # TODO: Check if the global 'Send to EDDN' option is off + to_send: OrderedDictT[str, OrderedDict[str, Any]] = OrderedDict([ ('$schemaRef', msg['$schemaRef']), ('header', OrderedDict([ @@ -1854,7 +1860,7 @@ def plugin_prefs(parent, cmdr: str, is_beta: bool) -> Frame: ) this.eddn_system_button.grid(padx=BUTTONX, pady=(5, 0), sticky=tk.W) - this.eddn_delay = tk.IntVar(value=(output & config.OUT_SYS_DELAY) and 1) + this.eddn_delay = tk.IntVar(value=(output & config.OUT_EDDN_DO_NOT_DELAY) and 1) # Output setting under 'Send system and scan data to the Elite Dangerous Data Network' new in E:D 2.2 this.eddn_delay_button = nb.Checkbutton( eddnframe, @@ -1891,7 +1897,7 @@ def prefs_changed(cmdr: str, is_beta: bool) -> None: & (config.OUT_MKT_TD | config.OUT_MKT_CSV | config.OUT_SHIP | config.OUT_MKT_MANUAL)) + (this.eddn_station.get() and config.OUT_MKT_EDDN) + (this.eddn_system.get() and config.OUT_SYS_EDDN) + - (this.eddn_delay.get() and config.OUT_SYS_DELAY) + (this.eddn_delay.get() and config.OUT_EDDN_DO_NOT_DELAY) ) diff --git a/prefs.py b/prefs.py index 03b8dcd8..acc7f12c 100644 --- a/prefs.py +++ b/prefs.py @@ -1221,7 +1221,7 @@ class PreferencesDialog(tk.Toplevel): (self.out_csv.get() and config.OUT_MKT_CSV) + (config.OUT_MKT_MANUAL if not self.out_auto.get() else 0) + (self.out_ship.get() and config.OUT_SHIP) + - (config.get_int('output') & (config.OUT_MKT_EDDN | config.OUT_SYS_EDDN | config.OUT_SYS_DELAY)) + (config.get_int('output') & (config.OUT_MKT_EDDN | config.OUT_SYS_EDDN | config.OUT_EDDN_DO_NOT_DELAY)) ) config.set( From 9f02f18408ba72d88826838da73b6fd7ce112e5b Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 5 Oct 2022 15:26:43 +0100 Subject: [PATCH 18/46] config/EDDN: Rename OUT_SYS_EDDN to OUT_EDDN_SEND_NON_STATION * This was perhaps originally meant for what the UI option says, i.e. "send system and scan data", but is actually being used for anything that is **NOT** 'station data' (even though *that* option has 'MKT' it includes outfitting and shipyard as well). So, just name this more sanely such that code using it is more obvious as to the actual intent. --- config/__init__.py | 2 +- plugins/eddn.py | 10 +++++----- prefs.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/config/__init__.py b/config/__init__.py index b7437b2d..d57412c4 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -175,7 +175,7 @@ class AbstractConfig(abc.ABC): # OUT_SYS_EDSM = 256 # Now a plugin # OUT_SYS_AUTO = 512 # Now always automatic OUT_MKT_MANUAL = 1024 - OUT_SYS_EDDN = 2048 + OUT_EDDN_SEND_NON_STATION = 2048 OUT_EDDN_DO_NOT_DELAY = 4096 app_dir_path: pathlib.Path diff --git a/plugins/eddn.py b/plugins/eddn.py index f5469c54..ab7e7ef9 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -1824,7 +1824,7 @@ def plugin_prefs(parent, cmdr: str, is_beta: bool) -> Frame: BUTTONX = 12 # noqa: N806 # indent Checkbuttons and Radiobuttons if prefsVersion.shouldSetDefaults('0.0.0.0', not bool(config.get_int('output'))): - output: int = (config.OUT_MKT_EDDN | config.OUT_SYS_EDDN) # default settings + output: int = (config.OUT_MKT_EDDN | config.OUT_EDDN_SEND_NON_STATION) # default settings else: output = config.get_int('output') @@ -1849,7 +1849,7 @@ def plugin_prefs(parent, cmdr: str, is_beta: bool) -> Frame: ) # Output setting this.eddn_station_button.grid(padx=BUTTONX, pady=(5, 0), sticky=tk.W) - this.eddn_system = tk.IntVar(value=(output & config.OUT_SYS_EDDN) and 1) + this.eddn_system = tk.IntVar(value=(output & config.OUT_EDDN_SEND_NON_STATION) and 1) # Output setting new in E:D 2.2 this.eddn_system_button = nb.Checkbutton( eddnframe, @@ -1896,7 +1896,7 @@ def prefs_changed(cmdr: str, is_beta: bool) -> None: (config.get_int('output') & (config.OUT_MKT_TD | config.OUT_MKT_CSV | config.OUT_SHIP | config.OUT_MKT_MANUAL)) + (this.eddn_station.get() and config.OUT_MKT_EDDN) + - (this.eddn_system.get() and config.OUT_SYS_EDDN) + + (this.eddn_system.get() and config.OUT_EDDN_SEND_NON_STATION) + (this.eddn_delay.get() and config.OUT_EDDN_DO_NOT_DELAY) ) @@ -2066,7 +2066,7 @@ def journal_entry( # noqa: C901, CCR001 this.status_body_name = None # Events with their own EDDN schema - if config.get_int('output') & config.OUT_SYS_EDDN and not state['Captain']: + if config.get_int('output') & config.OUT_EDDN_SEND_NON_STATION and not state['Captain']: if event_name == 'fssdiscoveryscan': return this.eddn.export_journal_fssdiscoveryscan(cmdr, system, state['StarPos'], is_beta, entry) @@ -2123,7 +2123,7 @@ def journal_entry( # noqa: C901, CCR001 ) # Send journal schema events to EDDN, but not when on a crew - if (config.get_int('output') & config.OUT_SYS_EDDN and not state['Captain'] and + if (config.get_int('output') & config.OUT_EDDN_SEND_NON_STATION and not state['Captain'] and (event_name in ('location', 'fsdjump', 'docked', 'scan', 'saasignalsfound', 'carrierjump')) and ('StarPos' in entry or this.coordinates)): diff --git a/prefs.py b/prefs.py index acc7f12c..d2fb69b0 100644 --- a/prefs.py +++ b/prefs.py @@ -1221,7 +1221,7 @@ class PreferencesDialog(tk.Toplevel): (self.out_csv.get() and config.OUT_MKT_CSV) + (config.OUT_MKT_MANUAL if not self.out_auto.get() else 0) + (self.out_ship.get() and config.OUT_SHIP) + - (config.get_int('output') & (config.OUT_MKT_EDDN | config.OUT_SYS_EDDN | config.OUT_EDDN_DO_NOT_DELAY)) + (config.get_int('output') & (config.OUT_MKT_EDDN | config.OUT_EDDN_SEND_NON_STATION | config.OUT_EDDN_DO_NOT_DELAY)) ) config.set( From 0d35f8874a3b7b746a2f1e502b2834981e7ef0ee Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 5 Oct 2022 15:29:11 +0100 Subject: [PATCH 19/46] config/EDDN: Rename OUT_MKT_EDDN to OUT_EDDN_SEND_STATION_DATA This flag controls whether commodity, outfitting or shipyard schema messages are sent. Thus 'MKT' ('market') is misleading. Rename it so the intent when used is clear. --- EDMarketConnector.py | 2 +- config/__init__.py | 4 ++-- plugins/eddn.py | 13 +++++++------ prefs.py | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/EDMarketConnector.py b/EDMarketConnector.py index 283c3bf5..041f7331 100755 --- a/EDMarketConnector.py +++ b/EDMarketConnector.py @@ -922,7 +922,7 @@ class AppWindow(object): return False # Ignore possibly missing shipyard info - elif (config.get_int('output') & config.OUT_MKT_EDDN) \ + elif (config.get_int('output') & config.OUT_EDDN_SEND_STATION_DATA) \ and not (data['lastStarport'].get('commodities') or data['lastStarport'].get('modules')): if not self.status['text']: # LANG: Status - Either no market or no modules data for station from Frontier CAPI diff --git a/config/__init__.py b/config/__init__.py index d57412c4..98056504 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -162,7 +162,7 @@ def appversion_nobuild() -> semantic_version.Version: class AbstractConfig(abc.ABC): """Abstract root class of all platform specific Config implementations.""" - OUT_MKT_EDDN = 1 + OUT_EDDN_SEND_STATION_DATA = 1 # OUT_MKT_BPC = 2 # No longer supported OUT_MKT_TD = 4 OUT_MKT_CSV = 8 @@ -171,7 +171,7 @@ class AbstractConfig(abc.ABC): # OUT_SYS_FILE = 32 # No longer supported # OUT_STAT = 64 # No longer available # OUT_SHIP_CORIOLIS = 128 # Replaced by OUT_SHIP - OUT_STATION_ANY = OUT_MKT_EDDN | OUT_MKT_TD | OUT_MKT_CSV + OUT_STATION_ANY = OUT_EDDN_SEND_STATION_DATA | OUT_MKT_TD | OUT_MKT_CSV # OUT_SYS_EDSM = 256 # Now a plugin # OUT_SYS_AUTO = 512 # Now always automatic OUT_MKT_MANUAL = 1024 diff --git a/plugins/eddn.py b/plugins/eddn.py index ab7e7ef9..b0f28a27 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -1824,7 +1824,7 @@ def plugin_prefs(parent, cmdr: str, is_beta: bool) -> Frame: BUTTONX = 12 # noqa: N806 # indent Checkbuttons and Radiobuttons if prefsVersion.shouldSetDefaults('0.0.0.0', not bool(config.get_int('output'))): - output: int = (config.OUT_MKT_EDDN | config.OUT_EDDN_SEND_NON_STATION) # default settings + output: int = (config.OUT_EDDN_SEND_STATION_DATA | config.OUT_EDDN_SEND_NON_STATION) # default settings else: output = config.get_int('output') @@ -1839,7 +1839,7 @@ def plugin_prefs(parent, cmdr: str, is_beta: bool) -> Frame: underline=True ).grid(padx=PADX, sticky=tk.W) # Don't translate - this.eddn_station = tk.IntVar(value=(output & config.OUT_MKT_EDDN) and 1) + this.eddn_station = tk.IntVar(value=(output & config.OUT_EDDN_SEND_STATION_DATA) and 1) this.eddn_station_button = nb.Checkbutton( eddnframe, # LANG: Enable EDDN support for station data checkbox label @@ -1895,7 +1895,7 @@ def prefs_changed(cmdr: str, is_beta: bool) -> None: 'output', (config.get_int('output') & (config.OUT_MKT_TD | config.OUT_MKT_CSV | config.OUT_SHIP | config.OUT_MKT_MANUAL)) + - (this.eddn_station.get() and config.OUT_MKT_EDDN) + + (this.eddn_station.get() and config.OUT_EDDN_SEND_STATION_DATA) + (this.eddn_system.get() and config.OUT_EDDN_SEND_NON_STATION) + (this.eddn_delay.get() and config.OUT_EDDN_DO_NOT_DELAY) ) @@ -2208,11 +2208,12 @@ def journal_entry( # noqa: C901, CCR001 return _("Error: Can't connect to EDDN") # LANG: Error while trying to send data to EDDN except Exception as e: + return logger.debug('Failed in export_journal_entry', exc_info=e) return str(e) - elif (config.get_int('output') & config.OUT_MKT_EDDN and not state['Captain'] and - event_name in ('market', 'outfitting', 'shipyard')): + elif (config.get_int('output') & config.OUT_EDDN_SEND_STATION_DATA and not state['Captain'] and + event_name in ('market', 'outfitting', 'shipyard')): # Market.json, Outfitting.json or Shipyard.json to process try: @@ -2263,7 +2264,7 @@ def cmdr_data(data: CAPIData, is_beta: bool) -> Optional[str]: # noqa: CCR001 :return: str - Error message, or `None` if no errors. """ if (data['commander'].get('docked') or (this.on_foot and monitor.station) - and config.get_int('output') & config.OUT_MKT_EDDN): + and config.get_int('output') & config.OUT_EDDN_SEND_STATION_DATA): try: if this.marketId != data['lastStarport']['id']: this.commodities = this.outfitting = this.shipyard = None diff --git a/prefs.py b/prefs.py index d2fb69b0..dd52c78b 100644 --- a/prefs.py +++ b/prefs.py @@ -1221,7 +1221,7 @@ class PreferencesDialog(tk.Toplevel): (self.out_csv.get() and config.OUT_MKT_CSV) + (config.OUT_MKT_MANUAL if not self.out_auto.get() else 0) + (self.out_ship.get() and config.OUT_SHIP) + - (config.get_int('output') & (config.OUT_MKT_EDDN | config.OUT_EDDN_SEND_NON_STATION | config.OUT_EDDN_DO_NOT_DELAY)) + (config.get_int('output') & (config.OUT_EDDN_SEND_STATION_DATA | config.OUT_EDDN_SEND_NON_STATION | config.OUT_EDDN_DO_NOT_DELAY)) ) config.set( From 7d0ae88757fa0fe1ffb86502fd3492cb49e63bf3 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 5 Oct 2022 15:46:12 +0100 Subject: [PATCH 20/46] EDDN: EDDNSender.send_message() doesn't care about do/don't send options By this point other code will have made that decision. --- plugins/eddn.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index b0f28a27..14c42dbb 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -342,6 +342,10 @@ class EDDNSender: """ Transmit a fully-formed EDDN message to the Gateway. + If this is called then the attempt *will* be made. This is not where + options to not send to EDDN, or to delay the sending until docked, + are checked. + Should catch and handle all failure conditions. A `True` return might mean that the message was successfully sent, *or* that this message should not be retried after a failure, i.e. too large. @@ -349,9 +353,6 @@ class EDDNSender: :param msg: Fully formed, string, message. :return: `True` for "now remove this message from the queue" """ - - # TODO: Check if user options require us to send at this time. - should_return, new_data = killswitch.check_killswitch('plugins.eddn.send', json.loads(msg)) if should_return: logger.warning('eddn.send has been disabled via killswitch. Returning.') From 5eb4296ec6a7c1cab2210f430234145cf2ffd7e1 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 5 Oct 2022 16:12:28 +0100 Subject: [PATCH 21/46] EDDN: First steps to ensure new code respects user configuration It's easier to check "should we send this message at all?" earlier. Currently all of the following ('station data') do so: * CAPI commodity, outfitting (also fcmaterials) and shipyard. * Journal commodity, fcmaterials, outfitting, and shipyard. --- plugins/eddn.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 14c42dbb..1f9ee487 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -130,6 +130,8 @@ class EDDNSender: """Handle sending of EDDN messages to the Gateway.""" SQLITE_DB_FILENAME_V1 = 'eddn_queue-v1.db' + # EDDN schema types that pertain to station data + STATION_SCHEMAS = ('commodity', 'fcmaterials_capi', 'fcmaterials_journal', 'outfitting', 'shipyard') TIMEOUT = 10 # requests timeout UNKNOWN_SCHEMA_RE = re.compile( r"^FAIL: \[JsonValidationException\('Schema " @@ -504,8 +506,17 @@ class EDDN: :param cmdr: the CMDR to use as the uploader ID. :param msg: the payload to send. """ - - # TODO: Check if the global 'Send to EDDN' option is off + # TODO: Check if we should actually send this message: + # 1. Is sending of this 'class' of message configured on ? + # 2. Are we *not* docked and delayed sending is configured on ? + # NB: This is a placeholder whilst all the "start of processing data" + # code points are confirmed to have their own check. + if ( + any(f'/{s}/' in msg['$schemaRef'] for s in EDDNSender.STATION_SCHEMAS) + and not config.get_int('output') & config.OUT_EDDN_SEND_STATION_DATA + ): + # Sending of station data configured off + return to_send: OrderedDictT[str, OrderedDict[str, Any]] = OrderedDict([ ('$schemaRef', msg['$schemaRef']), @@ -523,7 +534,7 @@ class EDDN: if (msg_id := self.sender.add_message(cmdr, to_send)) == -1: return - self.sender.send_message_by_id(msg_id) + self.sender.send_message_by_id(msg_id) def sendreplay(self) -> None: """Send cached Journal lines to EDDN.""" @@ -1340,6 +1351,10 @@ class EDDN: # ] # } + # TODO: Check we're configured to send station data + if not config.get_int('output') & config.OUT_EDDN_SEND_STATION_DATA: + return None + # Sanity check if 'Items' not in entry: logger.warning(f"FCMaterials didn't contain an Items array!\n{entry!r}") @@ -2209,7 +2224,6 @@ def journal_entry( # noqa: C901, CCR001 return _("Error: Can't connect to EDDN") # LANG: Error while trying to send data to EDDN except Exception as e: - return logger.debug('Failed in export_journal_entry', exc_info=e) return str(e) From 03b36cbe39a86d7108d34ec1c252d21734a46e0a Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 5 Oct 2022 16:15:08 +0100 Subject: [PATCH 22/46] EDDN: Improve export_journal_fcmaterials "don't send" comment --- plugins/eddn.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 1f9ee487..d9cb6809 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -1350,8 +1350,7 @@ class EDDN: # } # ] # } - - # TODO: Check we're configured to send station data + # Abort if we're not configured to send 'station' data. if not config.get_int('output') & config.OUT_EDDN_SEND_STATION_DATA: return None From db7bb735d2423162b007730096fd9a8f4666e821 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 5 Oct 2022 16:42:27 +0100 Subject: [PATCH 23/46] EDDN: Put "should we send (now?)" checks into `EDDN.export_journal_entry()` In some cases the check might already have been done, but if not then this is the last easy place to perform it. NB: Unlike the old code this does *not* attempt to check "are we docked *now* ?" for triggering sending of previously queue messages. That's going to need a thread worker. --- plugins/eddn.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index d9cb6809..8558815f 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -246,7 +246,7 @@ class EDDNSender: if self.db_conn: self.db_conn.close() - def add_message(self, cmdr: str, msg: dict) -> int: + def add_message(self, cmdr: str, msg: MutableMapping[str, Any]) -> int: """ Add an EDDN message to the database. @@ -925,7 +925,7 @@ class EDDN: # this.shipyard = (horizons, shipyard) - def export_journal_entry(self, cmdr: str, entry: Mapping[str, Any], msg: Mapping[str, Any]) -> None: + def export_journal_entry(self, cmdr: str, entry: Mapping[str, Any], msg: MutableMapping[str, Any]) -> None: """ Send a Journal-sourced EDDN message. @@ -933,7 +933,24 @@ class EDDN: :param entry: The full journal event dictionary (due to checks in this function). :param msg: The EDDN message body to be sent. """ - self.send(cmdr, msg) + # Check if the user configured messages to be sent. + # + # 1. If this is a 'station' data message then check config.EDDN_SEND_STATION_DATA + # 2. Else check against config.EDDN_SEND_NON_STATION *and* config.OUT_EDDN_DO_NOT_DELAY + if any(f'{s}' in msg['$schemaRef'] for s in EDDNSender.STATION_SCHEMAS): + # 'Station data' + if config.get_int('output') & config.OUT_EDDN_SEND_STATION_DATA: + # And user has 'station data' configured to be sent + msg_id = self.sender.add_message(cmdr, msg) + # 'Station data' is never delayed on construction of message + self.sender.send_message_by_id(msg_id) + + elif config.get_int('output') & config.OUT_EDDN_SEND_NON_STATION: + # Any data that isn't 'station' is configured to be sent + msg_id = self.sender.add_message(cmdr, msg) + if not config.get_int('output') & config.OUT_SYS_DELAY: + # No delay in sending configured, so attempt immediately + self.sender.send_message_by_id(msg_id) def export_journal_generic(self, cmdr: str, is_beta: bool, entry: Mapping[str, Any]) -> None: """ From fe24cf7e9596277d10bb68d930251ffa479785c9 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 5 Oct 2022 16:52:30 +0100 Subject: [PATCH 24/46] EDDN: Remove legacy `sendreplay()` * `EDDN.sendreplay()` is no longer used. * In `prefsvarschanged()` there was a reference to `eddn.replayfile`, so as to grey out the "Delay sending..." option if the file wasn't available. So that's moot and also removed, but also... * Comment the purpose of that line in `prefsvarchanged()` because it's not immediately obvious. --- plugins/eddn.py | 64 ++++--------------------------------------------- 1 file changed, 4 insertions(+), 60 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 8558815f..7250a8a7 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -536,65 +536,6 @@ class EDDN: self.sender.send_message_by_id(msg_id) - def sendreplay(self) -> None: - """Send cached Journal lines to EDDN.""" - # TODO: Convert to using the sqlite3 db - # **IF** this is moved to a thread worker then we need to ensure - # that we're operating sqlite3 in a thread-safe manner, - # Ref: - if not self.replayfile: - return # Probably closing app - - status: tk.Widget = self.parent.children['status'] - - if not self.replaylog: - status['text'] = '' - return - - localized: str = _('Sending data to EDDN...') # LANG: Status text shown while attempting to send data - if len(self.replaylog) == 1: - status['text'] = localized - - else: - status['text'] = f'{localized.replace("...", "")} [{len(self.replaylog)}]' - - self.parent.update_idletasks() - - # Paranoia check in case this function gets chain-called. - if not self.replaylog: - # import traceback - # logger.error( - # f'self.replaylog (type: {type(self.replaylog)}) is falsey after update_idletasks(). Traceback:\n' - # f'{"".join(traceback.format_list(traceback.extract_stack()))}') - return - - try: - cmdr, msg = json.loads(self.replaylog[0], object_pairs_hook=OrderedDict) - - except json.JSONDecodeError as e: - # Couldn't decode - shouldn't happen! - logger.debug(f'\n{self.replaylog[0]}\n', exc_info=e) - # Discard and continue - self.replaylog.pop(0) - - else: - # TODO: Check message against *current* relevant schema so we don't try - # to send an old message that's now invalid. - - # Rewrite old schema name - if msg['$schemaRef'].startswith('http://schemas.elite-markets.net/eddn/'): - msg['$schemaRef'] = str(msg['$schemaRef']).replace( - 'http://schemas.elite-markets.net/eddn/', - 'https://eddn.edcd.io/schemas/' - ) - - self.send(cmdr, msg) - self.replaylog.pop(0) - if not len(self.replaylog) % self.REPLAYFLUSH: - self.flush() - - self.parent.after(self.REPLAYPERIOD, self.sendreplay) - def export_commodities(self, data: Mapping[str, Any], is_beta: bool) -> None: # noqa: CCR001 """ Update EDDN with the commodities on the current (lastStarport) station. @@ -1911,9 +1852,12 @@ def prefsvarchanged(event=None) -> None: :param event: tkinter event ? """ + # These two lines are legacy and probably not even needed this.eddn_station_button['state'] = tk.NORMAL this.eddn_system_button['state'] = tk.NORMAL - this.eddn_delay_button['state'] = this.eddn.replayfile and this.eddn_system.get() and tk.NORMAL or tk.DISABLED + # This line will grey out the 'Delay sending ...' option if the 'Send + # system and scan data' option is off. + this.eddn_delay_button['state'] = this.eddn_system.get() and tk.NORMAL or tk.DISABLED def prefs_changed(cmdr: str, is_beta: bool) -> None: From 06edcf3ea96cf18b7314dba8b450da3294b20f13 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 5 Oct 2022 17:21:27 +0100 Subject: [PATCH 25/46] EDDN: Remove `EDDN.send()` in favour of renamed `.export_journal_entry()` * Now that we're not trying to do "did we just/are we know docked?" in this code it turns out that both CAPI and Journal messages can use the same function for this. * And as it's no longer journal-specific `EDDN.export_journal_entry()` has been renamed to `EDDN.send_message()`. This whole branch now needs to actually implement sending queued messages when docked, and periodically in the case of initial failures. --- plugins/eddn.py | 83 +++++++++++++------------------------------------ 1 file changed, 22 insertions(+), 61 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 7250a8a7..f3c3e9d6 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -499,43 +499,6 @@ class EDDN: logger.debug('Closing EDDN requests.Session.') self.session.close() - def send(self, cmdr: str, msg: Mapping[str, Any]) -> None: - """ - Enqueue a message for transmission. - - :param cmdr: the CMDR to use as the uploader ID. - :param msg: the payload to send. - """ - # TODO: Check if we should actually send this message: - # 1. Is sending of this 'class' of message configured on ? - # 2. Are we *not* docked and delayed sending is configured on ? - # NB: This is a placeholder whilst all the "start of processing data" - # code points are confirmed to have their own check. - if ( - any(f'/{s}/' in msg['$schemaRef'] for s in EDDNSender.STATION_SCHEMAS) - and not config.get_int('output') & config.OUT_EDDN_SEND_STATION_DATA - ): - # Sending of station data configured off - return - - to_send: OrderedDictT[str, OrderedDict[str, Any]] = OrderedDict([ - ('$schemaRef', msg['$schemaRef']), - ('header', OrderedDict([ - ('softwareName', f'{applongname} [{system() if sys.platform != "darwin" else "Mac OS"}]'), - ('softwareVersion', str(appversion_nobuild())), - ('uploaderID', cmdr), - # TODO: Add `gameversion` and `gamebuild` if that change is live - # on EDDN. - ])), - ('message', msg['message']), - ]) - - # Ensure it's en-queued - if (msg_id := self.sender.add_message(cmdr, to_send)) == -1: - return - - self.sender.send_message_by_id(msg_id) - def export_commodities(self, data: Mapping[str, Any], is_beta: bool) -> None: # noqa: CCR001 """ Update EDDN with the commodities on the current (lastStarport) station. @@ -601,7 +564,7 @@ class EDDN: if 'prohibited' in data['lastStarport']: message['prohibited'] = sorted(x for x in (data['lastStarport']['prohibited'] or {}).values()) - self.send(data['commander']['name'], { + self.send_message(data['commander']['name'], { '$schemaRef': f'https://eddn.edcd.io/schemas/commodity/3{"/test" if is_beta else ""}', 'message': message, }) @@ -686,7 +649,7 @@ class EDDN: # Don't send empty modules list - schema won't allow it if outfitting and this.outfitting != (horizons, outfitting): - self.send(data['commander']['name'], { + self.send_message(data['commander']['name'], { '$schemaRef': f'https://eddn.edcd.io/schemas/outfitting/2{"/test" if is_beta else ""}', 'message': OrderedDict([ ('timestamp', data['timestamp']), @@ -730,7 +693,7 @@ class EDDN: ) # Don't send empty ships list - shipyard data is only guaranteed present if user has visited the shipyard. if shipyard and this.shipyard != (horizons, shipyard): - self.send(data['commander']['name'], { + self.send_message(data['commander']['name'], { '$schemaRef': f'https://eddn.edcd.io/schemas/shipyard/2{"/test" if is_beta else ""}', 'message': OrderedDict([ ('timestamp', data['timestamp']), @@ -778,7 +741,7 @@ class EDDN: # none and that really does need to be recorded over EDDN so that, e.g. # EDDB can update in a timely manner. if this.commodities != commodities: - self.send(cmdr, { + self.send_message(cmdr, { '$schemaRef': f'https://eddn.edcd.io/schemas/commodity/3{"/test" if is_beta else ""}', 'message': OrderedDict([ ('timestamp', entry['timestamp']), @@ -817,7 +780,7 @@ class EDDN: ) # Don't send empty modules list - schema won't allow it if outfitting and this.outfitting != (horizons, outfitting): - self.send(cmdr, { + self.send_message(cmdr, { '$schemaRef': f'https://eddn.edcd.io/schemas/outfitting/2{"/test" if is_beta else ""}', 'message': OrderedDict([ ('timestamp', entry['timestamp']), @@ -851,7 +814,7 @@ class EDDN: shipyard = sorted(ship['ShipType'] for ship in ships) # Don't send empty ships list - shipyard data is only guaranteed present if user has visited the shipyard. if shipyard and this.shipyard != (horizons, shipyard): - self.send(cmdr, { + self.send_message(cmdr, { '$schemaRef': f'https://eddn.edcd.io/schemas/shipyard/2{"/test" if is_beta else ""}', 'message': OrderedDict([ ('timestamp', entry['timestamp']), @@ -866,12 +829,11 @@ class EDDN: # this.shipyard = (horizons, shipyard) - def export_journal_entry(self, cmdr: str, entry: Mapping[str, Any], msg: MutableMapping[str, Any]) -> None: + def send_message(self, cmdr: str, msg: MutableMapping[str, Any]) -> None: """ - Send a Journal-sourced EDDN message. + Send an EDDN message. :param cmdr: Commander name as passed in through `journal_entry()`. - :param entry: The full journal event dictionary (due to checks in this function). :param msg: The EDDN message body to be sent. """ # Check if the user configured messages to be sent. @@ -905,7 +867,7 @@ class EDDN: '$schemaRef': f'https://eddn.edcd.io/schemas/journal/1{"/test" if is_beta else ""}', 'message': entry } - this.eddn.export_journal_entry(cmdr, entry, msg) + this.eddn.send_message(cmdr, msg) def entry_augment_system_data( self, @@ -993,7 +955,7 @@ class EDDN: 'message': entry } - this.eddn.export_journal_entry(cmdr, entry, msg) + this.eddn.send_message(cmdr, msg) return None def export_journal_navbeaconscan( @@ -1035,7 +997,7 @@ class EDDN: 'message': entry } - this.eddn.export_journal_entry(cmdr, entry, msg) + this.eddn.send_message(cmdr, msg) return None def export_journal_codexentry( # noqa: CCR001 @@ -1135,7 +1097,7 @@ class EDDN: 'message': entry } - this.eddn.export_journal_entry(cmdr, entry, msg) + this.eddn.send_message(cmdr, msg) return None def export_journal_scanbarycentre( @@ -1189,7 +1151,7 @@ class EDDN: 'message': entry } - this.eddn.export_journal_entry(cmdr, entry, msg) + this.eddn.send_message(cmdr, msg) return None def export_journal_navroute( @@ -1262,7 +1224,7 @@ class EDDN: 'message': entry } - this.eddn.export_journal_entry(cmdr, entry, msg) + this.eddn.send_message(cmdr, msg) return None def export_journal_fcmaterials( @@ -1346,7 +1308,7 @@ class EDDN: 'message': entry } - this.eddn.export_journal_entry(cmdr, entry, msg) + this.eddn.send_message(cmdr, msg) return None def export_capi_fcmaterials( @@ -1404,7 +1366,7 @@ class EDDN: 'message': entry } - this.eddn.export_journal_entry(data['commander']['name'], entry, msg) + this.eddn.send_message(data['commander']['name'], msg) return None def export_journal_approachsettlement( @@ -1479,7 +1441,7 @@ class EDDN: 'message': entry } - this.eddn.export_journal_entry(cmdr, entry, msg) + this.eddn.send_message(cmdr, msg) return None def export_journal_fssallbodiesfound( @@ -1529,7 +1491,7 @@ class EDDN: 'message': entry } - this.eddn.export_journal_entry(cmdr, entry, msg) + this.eddn.send_message(cmdr, msg) return None def export_journal_fssbodysignals( @@ -1585,7 +1547,7 @@ class EDDN: 'message': entry } - this.eddn.export_journal_entry(cmdr, entry, msg) + this.eddn.send_message(cmdr, msg) return None def enqueue_journal_fsssignaldiscovered(self, entry: MutableMapping[str, Any]) -> None: @@ -1692,8 +1654,7 @@ class EDDN: logger.trace_if("plugin.eddn.fsssignaldiscovered", f"FSSSignalDiscovered batch is {json.dumps(msg)}") - # Fake an 'entry' as it's only there for some "should we send replay?" checks in the called function. - this.eddn.export_journal_entry(cmdr, {'event': 'send_fsssignaldiscovered'}, msg) + this.eddn.send_message(cmdr, msg) self.fss_signals = [] return None @@ -2180,11 +2141,11 @@ def journal_entry( # noqa: C901, CCR001 this.eddn.export_journal_generic(cmdr, is_beta, filter_localised(entry)) except requests.exceptions.RequestException as e: - logger.debug('Failed in export_journal_entry', exc_info=e) + logger.debug('Failed in send_message', exc_info=e) return _("Error: Can't connect to EDDN") # LANG: Error while trying to send data to EDDN except Exception as e: - logger.debug('Failed in export_journal_entry', exc_info=e) + logger.debug('Failed in export_journal_generic', exc_info=e) return str(e) elif (config.get_int('output') & config.OUT_EDDN_SEND_STATION_DATA and not state['Captain'] and From 871f50288e54a1a78f620734962b539164ef4c8c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 5 Oct 2022 18:01:39 +0100 Subject: [PATCH 26/46] prefs.py: Fix overly long line (config constants renames) --- prefs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/prefs.py b/prefs.py index dd52c78b..7be52526 100644 --- a/prefs.py +++ b/prefs.py @@ -1221,7 +1221,9 @@ class PreferencesDialog(tk.Toplevel): (self.out_csv.get() and config.OUT_MKT_CSV) + (config.OUT_MKT_MANUAL if not self.out_auto.get() else 0) + (self.out_ship.get() and config.OUT_SHIP) + - (config.get_int('output') & (config.OUT_EDDN_SEND_STATION_DATA | config.OUT_EDDN_SEND_NON_STATION | config.OUT_EDDN_DO_NOT_DELAY)) + (config.get_int('output') & ( + config.OUT_EDDN_SEND_STATION_DATA | config.OUT_EDDN_SEND_NON_STATION | config.OUT_EDDN_DO_NOT_DELAY + )) ) config.set( From 876c34ecfdf5ec07495de83e3809b2e31f8b02b7 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 22 Nov 2022 14:09:08 +0000 Subject: [PATCH 27/46] eddn: Remove two 'noqa: CCR001' that are no longer applicable --- plugins/eddn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index f3c3e9d6..ed3f4cef 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -340,7 +340,7 @@ class EDDNSender: return False - def send_message(self, msg: str) -> bool: # noqa: CCR001 + def send_message(self, msg: str) -> bool: """ Transmit a fully-formed EDDN message to the Gateway. @@ -2191,7 +2191,7 @@ def journal_entry( # noqa: C901, CCR001 return None -def cmdr_data(data: CAPIData, is_beta: bool) -> Optional[str]: # noqa: CCR001 +def cmdr_data(data: CAPIData, is_beta: bool) -> Optional[str]: """ Process new CAPI data. From a2d2723f491d94f7b7a4fc419a40759e3e138a6d Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 22 Nov 2022 15:29:05 +0000 Subject: [PATCH 28/46] monitor: Add an 'IsDocked' flag to monitor.state. * This is cleaner than starting to track it in `plugins/eddn.py` specifically. * This is literally only about if we're piloting a ship that is docked, so not even trying to resolve "on-foot, in a station, 'Location' said not docked though". --- monitor.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/monitor.py b/monitor.py index 859b2a50..040afa82 100644 --- a/monitor.py +++ b/monitor.py @@ -166,6 +166,7 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below 'Modules': None, 'CargoJSON': None, # The raw data from the last time cargo.json was read 'Route': None, # Last plotted route from Route.json file + 'IsDocked': False, # Whether we think cmdr is docked 'OnFoot': False, # Whether we think you're on-foot 'Component': defaultdict(int), # Odyssey Components in Ship Locker 'Item': defaultdict(int), # Odyssey Items in Ship Locker @@ -306,6 +307,7 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below self.systemaddress = None self.is_beta = False self.state['OnFoot'] = False + self.state['IsDocked'] = False self.state['Body'] = None self.state['BodyType'] = None @@ -725,6 +727,7 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below self.station_marketid = None self.stationtype = None self.stationservices = None + self.state['IsDocked'] = False elif event_type == 'embark': # This event is logged when a player (on foot) gets into a ship or SRV @@ -791,6 +794,7 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below self.state['Dropship'] = False elif event_type == 'docked': + self.state['IsDocked'] = True self.station = entry.get('StationName') # May be None self.station_marketid = entry.get('MarketID') # May be None self.stationtype = entry.get('StationType') # May be None @@ -813,6 +817,8 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below if event_type == 'location': logger.trace_if('journal.locations', '"Location" event') + if entry.get('Docked'): + self.state['IsDocked'] = True elif event_type == 'fsdjump': self.planet = None From be1ef32238a22d6186669e99d096004d0f337986 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 22 Nov 2022 15:30:30 +0000 Subject: [PATCH 29/46] config: Minor ordering change to ensure `OUT_STATION_ANY` set correctly * This depends on `OUT_EDDN_SEND_STATION_DATA` which is defined below where this originally was. --- config/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/__init__.py b/config/__init__.py index 98056504..1715cd46 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -171,12 +171,12 @@ class AbstractConfig(abc.ABC): # OUT_SYS_FILE = 32 # No longer supported # OUT_STAT = 64 # No longer available # OUT_SHIP_CORIOLIS = 128 # Replaced by OUT_SHIP - OUT_STATION_ANY = OUT_EDDN_SEND_STATION_DATA | OUT_MKT_TD | OUT_MKT_CSV # OUT_SYS_EDSM = 256 # Now a plugin # OUT_SYS_AUTO = 512 # Now always automatic OUT_MKT_MANUAL = 1024 OUT_EDDN_SEND_NON_STATION = 2048 OUT_EDDN_DO_NOT_DELAY = 4096 + OUT_STATION_ANY = OUT_EDDN_SEND_STATION_DATA | OUT_MKT_TD | OUT_MKT_CSV app_dir_path: pathlib.Path plugin_dir_path: pathlib.Path From 59f046ee67e39400f2107cbf737f46f0b5626c01 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 22 Nov 2022 15:51:10 +0000 Subject: [PATCH 30/46] eddn: First cut of periodic retry of sending messages --- plugins/eddn.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index ed3f4cef..95627765 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -34,6 +34,7 @@ import tkinter as tk from collections import OrderedDict from platform import system from textwrap import dedent +from threading import Lock from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Mapping, MutableMapping, Optional from typing import OrderedDict as OrderedDictT from typing import Tuple, Union @@ -66,6 +67,8 @@ class This: def __init__(self): # Track if we're on foot self.on_foot = False + # Track if we're docked + self.docked = False # Horizons ? self.horizons = False @@ -164,6 +167,10 @@ class EDDNSender: self.convert_legacy_file() ####################################################################### + self.queue_processing = Lock() + # Initiate retry/send-now timer + self.eddn.parent.after(self.eddn.REPLAYPERIOD, self.queue_check_and_send) + def sqlite_queue_v1(self) -> sqlite3.Connection: """ Initialise a v1 EDDN queue database. @@ -414,6 +421,48 @@ class EDDNSender: return False + def queue_check_and_send(self) -> None: + """Check if we should be sending queued messages, and send if we should.""" + # Mutex in case we're already processing + if not self.queue_processing.acquire(blocking=False): + return + + # We send either if docked or 'Delay sending until docked' not set + if this.docked or config.get_int('output') & config.OUT_EDDN_DO_NOT_DELAY: + # We need our own cursor here, in case the semantics of + # tk `after()` could allow this to run in the middle of other + # database usage. + db_cursor = self.db_conn.cursor() + + # Options: + # 1. Process every queued message, regardless. + # 2. Bail if we get any sort of connection error from EDDN. + + # Every queued message that is for *this* commander. We do **NOT** + # check if it's station/not-station, as the control of if a message + # was even created, versus the Settings > EDDN options, is applied + # *then*, not at time of sending. + try: + db_cursor.execute( + """ + SELECT message_id FROM messages + ORDER BY created ASC + """ + ) + + except Exception: + logger.exception("DB error querying queued messages") + + else: + row = dict(zip([c[0] for c in db_cursor.description], db_cursor.fetchone())) + self.send_message_by_id(row['message_id']) + + db_cursor.close() + + # Set us up to run again after a delay + self.queue_processing.release() + self.eddn.parent.after(self.eddn.REPLAYPERIOD, self.queue_check_and_send) + def _log_response( self, response: requests.Response, @@ -851,7 +900,7 @@ class EDDN: elif config.get_int('output') & config.OUT_EDDN_SEND_NON_STATION: # Any data that isn't 'station' is configured to be sent msg_id = self.sender.add_message(cmdr, msg) - if not config.get_int('output') & config.OUT_SYS_DELAY: + if config.get_int('output') & config.OUT_EDDN_DO_NOT_DELAY: # No delay in sending configured, so attempt immediately self.sender.send_message_by_id(msg_id) @@ -1925,6 +1974,7 @@ def journal_entry( # noqa: C901, CCR001 event_name = entry['event'].lower() this.on_foot = state['OnFoot'] + this.docked = state['IsDocked'] # Note if we're under Horizons and/or Odyssey # The only event these are already in is `LoadGame` which isn't sent to EDDN. From 6070f82c6b054b0d87cf759360338733dbf65451 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 22 Nov 2022 16:47:17 +0000 Subject: [PATCH 31/46] OUT_EDDN_DO_NOT_DELAY should be OUT_EDDN_DELAY * The old name was OUT_SYS_DELAY. * Yes, this is the inverse of what we want, which is "should we not delay messages", but this is the legacy. --- config/__init__.py | 2 +- prefs.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/__init__.py b/config/__init__.py index 1715cd46..67eae43b 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -175,7 +175,7 @@ class AbstractConfig(abc.ABC): # OUT_SYS_AUTO = 512 # Now always automatic OUT_MKT_MANUAL = 1024 OUT_EDDN_SEND_NON_STATION = 2048 - OUT_EDDN_DO_NOT_DELAY = 4096 + OUT_EDDN_DELAY = 4096 OUT_STATION_ANY = OUT_EDDN_SEND_STATION_DATA | OUT_MKT_TD | OUT_MKT_CSV app_dir_path: pathlib.Path diff --git a/prefs.py b/prefs.py index 7be52526..376b6c67 100644 --- a/prefs.py +++ b/prefs.py @@ -1222,7 +1222,7 @@ class PreferencesDialog(tk.Toplevel): (config.OUT_MKT_MANUAL if not self.out_auto.get() else 0) + (self.out_ship.get() and config.OUT_SHIP) + (config.get_int('output') & ( - config.OUT_EDDN_SEND_STATION_DATA | config.OUT_EDDN_SEND_NON_STATION | config.OUT_EDDN_DO_NOT_DELAY + config.OUT_EDDN_SEND_STATION_DATA | config.OUT_EDDN_SEND_NON_STATION | config.OUT_EDDN_DELAY )) ) From 06fa3629ea806feede46d7d79d9adce652e5b6e8 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 22 Nov 2022 16:49:02 +0000 Subject: [PATCH 32/46] eddn: OUT_EDDN_DELAY (not inverted) & further work on the sending * The eddn parts of the OUT_EDDN_DO_NOT_DELAY -> OUT_EDDN_DELAY change. This includes the 'sense' of it being inverted from what it was. * EDDN.REPLAY_DELAY is now a float, as it's used with `time.sleep()`. *This* is the 400ms value for inter-message cooldown. * EDDN.REPLAY_PERIOD is still an int, used with tk `after()`. This is how often we attempt the queue. * EDDN.session is no longer a thing, move that part of EDDN.close() to EDDNSender.close(). * EDDN queue DB has `id`, not `message_id`. * Now *looping* in the queue sender, not only sending the oldest message. --- plugins/eddn.py | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 95627765..15a0c61c 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -30,6 +30,7 @@ import pathlib import re import sqlite3 import sys +import time import tkinter as tk from collections import OrderedDict from platform import system @@ -169,7 +170,7 @@ class EDDNSender: self.queue_processing = Lock() # Initiate retry/send-now timer - self.eddn.parent.after(self.eddn.REPLAYPERIOD, self.queue_check_and_send) + self.eddn.parent.after(self.eddn.REPLAY_PERIOD, self.queue_check_and_send) def sqlite_queue_v1(self) -> sqlite3.Connection: """ @@ -253,6 +254,9 @@ class EDDNSender: if self.db_conn: self.db_conn.close() + logger.debug('Closing EDDN requests.Session.') + self.session.close() + def add_message(self, cmdr: str, msg: MutableMapping[str, Any]) -> int: """ Add an EDDN message to the database. @@ -423,12 +427,16 @@ class EDDNSender: def queue_check_and_send(self) -> None: """Check if we should be sending queued messages, and send if we should.""" + # logger.debug("Called") # Mutex in case we're already processing if not self.queue_processing.acquire(blocking=False): + logger.debug("Couldn't obtain mutex") return + # logger.debug("Obtained mutex") # We send either if docked or 'Delay sending until docked' not set - if this.docked or config.get_int('output') & config.OUT_EDDN_DO_NOT_DELAY: + if this.docked or not (config.get_int('output') & config.OUT_EDDN_DELAY): + # logger.debug("Should send") # We need our own cursor here, in case the semantics of # tk `after()` could allow this to run in the middle of other # database usage. @@ -445,7 +453,7 @@ class EDDNSender: try: db_cursor.execute( """ - SELECT message_id FROM messages + SELECT id FROM messages ORDER BY created ASC """ ) @@ -454,14 +462,20 @@ class EDDNSender: logger.exception("DB error querying queued messages") else: - row = dict(zip([c[0] for c in db_cursor.description], db_cursor.fetchone())) - self.send_message_by_id(row['message_id']) + while row := db_cursor.fetchone(): + row = dict(zip([c[0] for c in db_cursor.description], row)) + self.send_message_by_id(row['id']) + time.sleep(self.eddn.REPLAY_DELAY) db_cursor.close() + # else: + # logger.debug("Should NOT send") + # Set us up to run again after a delay self.queue_processing.release() - self.eddn.parent.after(self.eddn.REPLAYPERIOD, self.queue_check_and_send) + # logger.debug("Mutex released") + self.eddn.parent.after(self.eddn.REPLAY_PERIOD, self.queue_check_and_send) def _log_response( self, @@ -518,7 +532,9 @@ class EDDN: if 'eddn' in debug_senders: DEFAULT_URL = f'http://{edmc_data.DEBUG_WEBSERVER_HOST}:{edmc_data.DEBUG_WEBSERVER_PORT}/eddn' - REPLAYPERIOD = 400 # Roughly two messages per second, accounting for send delays [ms] + # FIXME: Change back to `300_000` + REPLAY_PERIOD = 1_000 # How often to try (re-)sending the queue, [milliseconds] + REPLAY_DELAY = 0.400 # Roughly two messages per second, accounting for send delays [seconds] REPLAYFLUSH = 20 # Update log on disk roughly every 10 seconds MODULE_RE = re.compile(r'^Hpt_|^Int_|Armour_', re.IGNORECASE) CANONICALISE_RE = re.compile(r'\$(.+)_name;') @@ -545,9 +561,6 @@ class EDDN: logger.debug('Done.') - logger.debug('Closing EDDN requests.Session.') - self.session.close() - def export_commodities(self, data: Mapping[str, Any], is_beta: bool) -> None: # noqa: CCR001 """ Update EDDN with the commodities on the current (lastStarport) station. @@ -888,7 +901,7 @@ class EDDN: # Check if the user configured messages to be sent. # # 1. If this is a 'station' data message then check config.EDDN_SEND_STATION_DATA - # 2. Else check against config.EDDN_SEND_NON_STATION *and* config.OUT_EDDN_DO_NOT_DELAY + # 2. Else check against config.EDDN_SEND_NON_STATION *and* config.OUT_EDDN_DELAY if any(f'{s}' in msg['$schemaRef'] for s in EDDNSender.STATION_SCHEMAS): # 'Station data' if config.get_int('output') & config.OUT_EDDN_SEND_STATION_DATA: @@ -900,7 +913,7 @@ class EDDN: elif config.get_int('output') & config.OUT_EDDN_SEND_NON_STATION: # Any data that isn't 'station' is configured to be sent msg_id = self.sender.add_message(cmdr, msg) - if config.get_int('output') & config.OUT_EDDN_DO_NOT_DELAY: + if not (config.get_int('output') & config.OUT_EDDN_DELAY): # No delay in sending configured, so attempt immediately self.sender.send_message_by_id(msg_id) @@ -1843,7 +1856,7 @@ def plugin_prefs(parent, cmdr: str, is_beta: bool) -> Frame: ) this.eddn_system_button.grid(padx=BUTTONX, pady=(5, 0), sticky=tk.W) - this.eddn_delay = tk.IntVar(value=(output & config.OUT_EDDN_DO_NOT_DELAY) and 1) + this.eddn_delay = tk.IntVar(value=(output & config.OUT_EDDN_DELAY) and 1) # Output setting under 'Send system and scan data to the Elite Dangerous Data Network' new in E:D 2.2 this.eddn_delay_button = nb.Checkbutton( eddnframe, @@ -1883,7 +1896,7 @@ def prefs_changed(cmdr: str, is_beta: bool) -> None: & (config.OUT_MKT_TD | config.OUT_MKT_CSV | config.OUT_SHIP | config.OUT_MKT_MANUAL)) + (this.eddn_station.get() and config.OUT_EDDN_SEND_STATION_DATA) + (this.eddn_system.get() and config.OUT_EDDN_SEND_NON_STATION) + - (this.eddn_delay.get() and config.OUT_EDDN_DO_NOT_DELAY) + (this.eddn_delay.get() and config.OUT_EDDN_DELAY) ) From 4a0518da9ffc145a9c9d936fe1638c5894b69270 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 22 Nov 2022 17:26:32 +0000 Subject: [PATCH 33/46] eddn: Set header up properly for all journal messages * New function `EDDN.add_header()`. * It utilises new `this` members, set from `journal_entry()`. --- plugins/eddn.py | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 15a0c61c..2dc30c1d 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -66,6 +66,12 @@ class This: """Holds module globals.""" def __init__(self): + # Game version and build + self.game_version = "" + self.game_build = "" + # Commander Name + self.cmdr_name = "" + # Track if we're on foot self.on_foot = False # Track if we're docked @@ -906,17 +912,43 @@ class EDDN: # 'Station data' if config.get_int('output') & config.OUT_EDDN_SEND_STATION_DATA: # And user has 'station data' configured to be sent + msg = self.add_header(msg) msg_id = self.sender.add_message(cmdr, msg) # 'Station data' is never delayed on construction of message self.sender.send_message_by_id(msg_id) elif config.get_int('output') & config.OUT_EDDN_SEND_NON_STATION: # Any data that isn't 'station' is configured to be sent + msg = self.add_header(msg) msg_id = self.sender.add_message(cmdr, msg) if not (config.get_int('output') & config.OUT_EDDN_DELAY): # No delay in sending configured, so attempt immediately self.sender.send_message_by_id(msg_id) + def add_header(self, msg: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + """ + Augment the given EDDN message with our header defaults. + + NB: This should *only* be called for newly constructed messages, not + for either a legacy message or an already queued one! + + :param msg: Message to be augmented + :return: The augmented version + """ + if 'header' in msg: + logger.error("Passed `msg` which already has a header") + return msg + + msg['header'] = { + 'softwareName': f'{applongname} [{system() if sys.platform != "darwin" else "Mac OS"}]', + 'softwareVersion': str(appversion_nobuild()), + 'uploaderID': this.cmdr_name, + 'gameversion': this.game_version, + 'gamebuild': this.game_build, + } + + return msg + def export_journal_generic(self, cmdr: str, is_beta: bool, entry: Mapping[str, Any]) -> None: """ Send an EDDN event on the journal schema. @@ -1966,7 +1998,7 @@ def journal_entry( # noqa: C901, CCR001 """ Process a new Journal entry. - :param cmdr: `str` - Name of currennt Cmdr. + :param cmdr: `str` - Name of current Cmdr. :param is_beta: `bool` - True if this is a beta version of the Game. :param system: `str` - Name of system Cmdr is in. :param station: `str` - Name of station Cmdr is docked at, if applicable. @@ -1986,6 +2018,9 @@ def journal_entry( # noqa: C901, CCR001 entry = new_data event_name = entry['event'].lower() + this.cmdr_name = cmdr + this.game_version = state['GameVersion'] + this.game_build = state['GameBuild'] this.on_foot = state['OnFoot'] this.docked = state['IsDocked'] From b31c8c05362cba2839bc964d386126662b70b2ee Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 22 Nov 2022 17:35:17 +0000 Subject: [PATCH 34/46] eddn: A start on ensuring header (gameversion/build) are always set --- plugins/eddn.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 2dc30c1d..4204b226 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -912,43 +912,38 @@ class EDDN: # 'Station data' if config.get_int('output') & config.OUT_EDDN_SEND_STATION_DATA: # And user has 'station data' configured to be sent - msg = self.add_header(msg) + msg['header'] = self.standard_header() msg_id = self.sender.add_message(cmdr, msg) # 'Station data' is never delayed on construction of message self.sender.send_message_by_id(msg_id) elif config.get_int('output') & config.OUT_EDDN_SEND_NON_STATION: # Any data that isn't 'station' is configured to be sent - msg = self.add_header(msg) + msg['header'] = self.standard_header() msg_id = self.sender.add_message(cmdr, msg) if not (config.get_int('output') & config.OUT_EDDN_DELAY): # No delay in sending configured, so attempt immediately self.sender.send_message_by_id(msg_id) - def add_header(self, msg: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + def standard_header( + self, game_version: Optional[str] = None, game_build: Optional[str] = None + ) -> MutableMapping[str, Any]: """ - Augment the given EDDN message with our header defaults. + Return the standard header for an EDDN message, given tracked state. NB: This should *only* be called for newly constructed messages, not for either a legacy message or an already queued one! - :param msg: Message to be augmented - :return: The augmented version + :return: The standard header """ - if 'header' in msg: - logger.error("Passed `msg` which already has a header") - return msg - - msg['header'] = { + return { 'softwareName': f'{applongname} [{system() if sys.platform != "darwin" else "Mac OS"}]', 'softwareVersion': str(appversion_nobuild()), 'uploaderID': this.cmdr_name, - 'gameversion': this.game_version, - 'gamebuild': this.game_build, + 'gameversion': game_version or this.game_version, + 'gamebuild': game_build or this.game_build, } - return msg - def export_journal_generic(self, cmdr: str, is_beta: bool, entry: Mapping[str, Any]) -> None: """ Send an EDDN event on the journal schema. @@ -1457,7 +1452,8 @@ class EDDN: msg = { '$schemaRef': f'https://eddn.edcd.io/schemas/fcmaterials_capi/1{"/test" if is_beta else ""}', - 'message': entry + 'message': entry, + 'header': self.standard_header(game_version='CAPI-commodity', game_build='CAPI-commodity'), } this.eddn.send_message(data['commander']['name'], msg) From f6e25042165789808fe76b7f89bddde7ef53f730 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 22 Nov 2022 17:37:15 +0000 Subject: [PATCH 35/46] eddn: Fix capi_fcmaterials gameversion/build & add to CAPI commodity * Erroneously used 'CAPI-commoodity' when it's 'CAPI-market' (name of the CAPI endpoint, not anything to do with EDDN schema names, and '-commodity' would also be wrong for that). * Set `header` for (CAPI) `export_commodities()`. --- plugins/eddn.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 4204b226..2a129462 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -635,6 +635,7 @@ class EDDN: self.send_message(data['commander']['name'], { '$schemaRef': f'https://eddn.edcd.io/schemas/commodity/3{"/test" if is_beta else ""}', 'message': message, + 'header': self.standard_header(game_version='CAPI-market', game_build='CAPI-market') }) this.commodities = commodities @@ -1453,7 +1454,7 @@ class EDDN: msg = { '$schemaRef': f'https://eddn.edcd.io/schemas/fcmaterials_capi/1{"/test" if is_beta else ""}', 'message': entry, - 'header': self.standard_header(game_version='CAPI-commodity', game_build='CAPI-commodity'), + 'header': self.standard_header(game_version='CAPI-market', game_build='CAPI-market'), } this.eddn.send_message(data['commander']['name'], msg) From f2dbfacf70071632013910519ff7374ea42f67d8 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Tue, 22 Nov 2022 17:40:33 +0000 Subject: [PATCH 36/46] eddn: Add header for CAPI outfitting and shipyard exports --- plugins/eddn.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 2a129462..1bd6e834 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -635,7 +635,7 @@ class EDDN: self.send_message(data['commander']['name'], { '$schemaRef': f'https://eddn.edcd.io/schemas/commodity/3{"/test" if is_beta else ""}', 'message': message, - 'header': self.standard_header(game_version='CAPI-market', game_build='CAPI-market') + 'header': self.standard_header(game_version='CAPI-market', game_build='CAPI-market'), }) this.commodities = commodities @@ -729,6 +729,7 @@ class EDDN: ('modules', outfitting), ('odyssey', this.odyssey), ]), + 'header': self.standard_header(game_version='CAPI-shipyard', game_build='CAPI-shipyard'), }) this.outfitting = (horizons, outfitting) @@ -773,6 +774,7 @@ class EDDN: ('ships', shipyard), ('odyssey', this.odyssey), ]), + 'header': self.standard_header(game_version='CAPI-shipyard', game_build='CAPI-shipyard'), }) this.shipyard = (horizons, shipyard) From fda91df04fa0c7d68a4a1c7f09ec5171acd49540 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 23 Nov 2022 13:29:47 +0000 Subject: [PATCH 37/46] eddn: Working with tk `after()`, on timer or when docked * An aborted attempt was made to use a thread worker, but: 1. sqlite3 doesn't allow cross-thread use of the same sqlite3 connection. 2. Having an on-going query on one cursor, e.g. gathering all the outstanding message `id`, whilst trying to DELETE a row hits a "database is locked" error. * So, back to tk `after()`. `send_message_by_id()` has been audited to ensure its boolean return is accurate. So there shouldn't be any way in which to get hung up on a single message *other than if the EDDN Gateway is having issues, and thus it should be retried anyway*. Any reason for a 'bad message' will cause `True` return and thus deletion of the message in *this* call to `queue_check_and_send()`. * There is a new `reschedule` parameter to `queue_check_and_send()`. If `True` then at the end it should re-schedule. There is a check in `journal_entry()` for the `Docked` event, and if this occurs it will schedule `queue_check_and_send()` with `reschedule` set to `False` so that we don't end up with multiple parallel schedulings. It's still possible for a docking to have coincided with a scheduled run and thus cause double-rate sending to EDDN, but we can live with that. * The same scheduling mechanism is used, with a much smaller delay, to process more than one queued message per run. Hence the `have_rescheduled` bool *in* the function to indicate if a 'fast' reschedule has already been set. This prevents the slow one *also* being set in this scenario. The latter will be scheduled when the fast one found no more rows to process. --- plugins/eddn.py | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 1bd6e834..2158c6de 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -176,7 +176,7 @@ class EDDNSender: self.queue_processing = Lock() # Initiate retry/send-now timer - self.eddn.parent.after(self.eddn.REPLAY_PERIOD, self.queue_check_and_send) + self.eddn.parent.after(self.eddn.REPLAY_PERIOD, self.queue_check_and_send, True) def sqlite_queue_v1(self) -> sqlite3.Connection: """ @@ -431,15 +431,24 @@ class EDDNSender: return False - def queue_check_and_send(self) -> None: - """Check if we should be sending queued messages, and send if we should.""" - # logger.debug("Called") + def queue_check_and_send(self, reschedule: bool = False) -> None: + """ + Check if we should be sending queued messages, and send if we should. + + :param reschedule: Boolean indicating if we should call `after()` again. + """ + logger.debug("Called") # Mutex in case we're already processing if not self.queue_processing.acquire(blocking=False): logger.debug("Couldn't obtain mutex") + if reschedule: + self.eddn.parent.after(self.eddn.REPLAY_PERIOD, self.queue_check_and_send, reschedule) + return # logger.debug("Obtained mutex") + # Used to indicate if we've rescheduled at the faster rate already. + have_rescheduled = False # We send either if docked or 'Delay sending until docked' not set if this.docked or not (config.get_int('output') & config.OUT_EDDN_DELAY): # logger.debug("Should send") @@ -461,6 +470,7 @@ class EDDNSender: """ SELECT id FROM messages ORDER BY created ASC + LIMIT 1 """ ) @@ -468,20 +478,24 @@ class EDDNSender: logger.exception("DB error querying queued messages") else: - while row := db_cursor.fetchone(): + row = db_cursor.fetchone() + if row: row = dict(zip([c[0] for c in db_cursor.description], row)) self.send_message_by_id(row['id']) - time.sleep(self.eddn.REPLAY_DELAY) + # Always re-schedule as this is only a "Don't hammer EDDN" delay + self.eddn.parent.after(self.eddn.REPLAY_DELAY, self.queue_check_and_send, reschedule) + have_rescheduled = True - db_cursor.close() + db_cursor.close() # else: # logger.debug("Should NOT send") - # Set us up to run again after a delay self.queue_processing.release() # logger.debug("Mutex released") - self.eddn.parent.after(self.eddn.REPLAY_PERIOD, self.queue_check_and_send) + if reschedule and not have_rescheduled: + # Set us up to run again per the configured period + self.eddn.parent.after(self.eddn.REPLAY_PERIOD, self.queue_check_and_send, reschedule) def _log_response( self, @@ -539,8 +553,8 @@ class EDDN: DEFAULT_URL = f'http://{edmc_data.DEBUG_WEBSERVER_HOST}:{edmc_data.DEBUG_WEBSERVER_PORT}/eddn' # FIXME: Change back to `300_000` - REPLAY_PERIOD = 1_000 # How often to try (re-)sending the queue, [milliseconds] - REPLAY_DELAY = 0.400 # Roughly two messages per second, accounting for send delays [seconds] + REPLAY_PERIOD = 300_000 # How often to try (re-)sending the queue, [milliseconds] + REPLAY_DELAY = 400 # Roughly two messages per second, accounting for send delays [milliseconds] REPLAYFLUSH = 20 # Update log on disk roughly every 10 seconds MODULE_RE = re.compile(r'^Hpt_|^Int_|Armour_', re.IGNORECASE) CANONICALISE_RE = re.compile(r'\$(.+)_name;') @@ -2081,6 +2095,9 @@ def journal_entry( # noqa: C901, CCR001 # Yes, explicitly state `None` here, so it's crystal clear. this.systemaddress = entry.get('SystemAddress', None) # type: ignore + if event_name == 'docked': + this.eddn.parent.after(this.eddn.REPLAY_DELAY, this.eddn.sender.queue_check_and_send, False) + elif event_name == 'approachbody': this.body_name = entry['Body'] this.body_id = entry.get('BodyID') From 524c0425fb784bf04bb5cad8a7d76cee2277a7a1 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 23 Nov 2022 13:46:49 +0000 Subject: [PATCH 38/46] eddn: EDDN.send_message(): Only set standard header if not already set. Else we risk overwriting e.g. `CAPI-shipyard`. --- plugins/eddn.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 2158c6de..d201e291 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -30,7 +30,6 @@ import pathlib import re import sqlite3 import sys -import time import tkinter as tk from collections import OrderedDict from platform import system @@ -929,14 +928,18 @@ class EDDN: # 'Station data' if config.get_int('output') & config.OUT_EDDN_SEND_STATION_DATA: # And user has 'station data' configured to be sent - msg['header'] = self.standard_header() + if 'header' not in msg: + msg['header'] = self.standard_header() + msg_id = self.sender.add_message(cmdr, msg) # 'Station data' is never delayed on construction of message self.sender.send_message_by_id(msg_id) elif config.get_int('output') & config.OUT_EDDN_SEND_NON_STATION: # Any data that isn't 'station' is configured to be sent - msg['header'] = self.standard_header() + if 'header' not in msg: + msg['header'] = self.standard_header() + msg_id = self.sender.add_message(cmdr, msg) if not (config.get_int('output') & config.OUT_EDDN_DELAY): # No delay in sending configured, so attempt immediately From b7769821bede7db3e6a79971917c9dc0baa54a96 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 23 Nov 2022 13:56:01 +0000 Subject: [PATCH 39/46] eddn: Send immediately if docked This has the "are we delaying sending?" check, but without the short-circuit for "we are actually docked right now". --- plugins/eddn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index d201e291..db6dbf96 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -941,7 +941,7 @@ class EDDN: msg['header'] = self.standard_header() msg_id = self.sender.add_message(cmdr, msg) - if not (config.get_int('output') & config.OUT_EDDN_DELAY): + if this.docked or not (config.get_int('output') & config.OUT_EDDN_DELAY): # No delay in sending configured, so attempt immediately self.sender.send_message_by_id(msg_id) From 3d9bb643300bf8d8684138414ddd548abdf0c950 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 23 Nov 2022 14:12:43 +0000 Subject: [PATCH 40/46] eddn: Remove outdated comment about replaylog rework And using tk `parent` *is* absolutely necessary. --- plugins/eddn.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index db6dbf96..da519cc4 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -1797,10 +1797,7 @@ def plugin_app(parent: tk.Tk) -> Optional[tk.Frame]: Set up any plugin-specific UI. In this case we need the tkinter parent in order to later call - `update_idletasks()` on it. - - TODO: Re-work the whole replaylog and general sending to EDDN so this isn't - necessary. + `update_idletasks()` on it, or schedule things with `after()`. :param parent: tkinter parent frame. :return: Optional tk.Frame, if the tracking UI is active. From 95fa9d577c31494bb4910504f62ddb50b616c45c Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 23 Nov 2022 14:20:24 +0000 Subject: [PATCH 41/46] eddn: Use a shorter, 10 second, delay at startup for first queue check --- plugins/eddn.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index da519cc4..cc94ff1a 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -175,7 +175,7 @@ class EDDNSender: self.queue_processing = Lock() # Initiate retry/send-now timer - self.eddn.parent.after(self.eddn.REPLAY_PERIOD, self.queue_check_and_send, True) + self.eddn.parent.after(self.eddn.REPLAY_STARTUP_DELAY, self.queue_check_and_send, True) def sqlite_queue_v1(self) -> sqlite3.Connection: """ @@ -552,6 +552,7 @@ class EDDN: DEFAULT_URL = f'http://{edmc_data.DEBUG_WEBSERVER_HOST}:{edmc_data.DEBUG_WEBSERVER_PORT}/eddn' # FIXME: Change back to `300_000` + REPLAY_STARTUP_DELAY = 10_000 # Delay during startup before checking queue [milliseconds] REPLAY_PERIOD = 300_000 # How often to try (re-)sending the queue, [milliseconds] REPLAY_DELAY = 400 # Roughly two messages per second, accounting for send delays [milliseconds] REPLAYFLUSH = 20 # Update log on disk roughly every 10 seconds From 0eb33e011b80997c1df6759b79f97e3fef3d27dd Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 23 Nov 2022 15:57:25 +0000 Subject: [PATCH 42/46] PLUGINS.md: Document new `IsDocked` state flag. --- PLUGINS.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/PLUGINS.md b/PLUGINS.md index 1649b47e..e8d47a3e 100644 --- a/PLUGINS.md +++ b/PLUGINS.md @@ -617,6 +617,7 @@ Content of `state` (updated to the current journal entry): | `Modules` | `dict` | Currently fitted modules | | `NavRoute` | `dict` | Last plotted multi-hop route | | `ModuleInfo` | `dict` | Last loaded ModulesInfo.json data | +| `IsDocked` | `bool` | Whether the Cmdr is currently docked *in their own ship*. | | `OnFoot` | `bool` | Whether the Cmdr is on foot | | `Component` | `dict` | 'Component' MicroResources in Odyssey, `int` count each. | | `Item` | `dict` | 'Item' MicroResources in Odyssey, `int` count each. | @@ -710,6 +711,17 @@ NB: It *is* possible, if a player is quick enough, to plot and clear a route before we load it, in which case we'd be retaining the *previous* plotted route. +New in version 5.6.0: + +`IsDocked` boolean added to `state`. This is set True for a `Location` event +having `"Docked":true"`, or the `Docked` event. It is set back to False (its +default value) for an `Undocked` event. Being on-foot in a station at login +time does *not* count as docked for this. + +In general on-foot, including being in a taxi, might not set this 100% +correctly. Its main use in core code is to detect being docked so as to send +any stored EDDN messages due to "Delay sending until docked" option. + ___ ##### Synthetic Events From 32229217b2c5676ae8c6530fcf1bc026131705dc Mon Sep 17 00:00:00 2001 From: Athanasius Date: Wed, 23 Nov 2022 16:00:28 +0000 Subject: [PATCH 43/46] eddn: Bail from sending queued messages if one failed NB: This is failed *not* due to the message being 'bad' in some manner. It will mean the Gateway timed out, refused connection etc. --- plugins/eddn.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index cc94ff1a..805f45cf 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -364,6 +364,8 @@ class EDDNSender: options to not send to EDDN, or to delay the sending until docked, are checked. + It *is* however the one 'sending' place that the EDDN killswitches are checked. + Should catch and handle all failure conditions. A `True` return might mean that the message was successfully sent, *or* that this message should not be retried after a failure, i.e. too large. @@ -480,10 +482,14 @@ class EDDNSender: row = db_cursor.fetchone() if row: row = dict(zip([c[0] for c in db_cursor.description], row)) - self.send_message_by_id(row['id']) - # Always re-schedule as this is only a "Don't hammer EDDN" delay - self.eddn.parent.after(self.eddn.REPLAY_DELAY, self.queue_check_and_send, reschedule) - have_rescheduled = True + if self.send_message_by_id(row['id']): + # If `True` was returned then we're done with this message. + # `False` means "failed to send, but not because the message + # is bad", i.e. an EDDN Gateway problem. Thus, in that case + # we do *NOT* schedule attempting the next message. + # Always re-schedule as this is only a "Don't hammer EDDN" delay + self.eddn.parent.after(self.eddn.REPLAY_DELAY, self.queue_check_and_send, reschedule) + have_rescheduled = True db_cursor.close() From 073afc842fb6a3d0d7e22b374d8396d4c921bbee Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 24 Nov 2022 14:09:57 +0000 Subject: [PATCH 44/46] eddn: New `--trace-on plugins.eddn.send` & other logging tweaks * In case of apparent issues, have a `--trace-on` to better see what's (not) happening. All the old DBEUG logging, even if commented out, is now under this. * Also added some INFO level logging for the legacy replay.jsonl conversion, as it should be one-time per user. * Some additional DEBUG logging for closing down. --- plugins/eddn.py | 42 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 805f45cf..366f29f3 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -175,6 +175,10 @@ class EDDNSender: self.queue_processing = Lock() # Initiate retry/send-now timer + logger.trace_if( + "plugin.eddn.send", + f"First queue run scheduled for {self.eddn.REPLAY_STARTUP_DELAY}ms from now" + ) self.eddn.parent.after(self.eddn.REPLAY_STARTUP_DELAY, self.queue_check_and_send, True) def sqlite_queue_v1(self) -> sqlite3.Connection: @@ -227,6 +231,9 @@ class EDDNSender: db_conn.close() raise e + else: + logger.info("New `eddn_queue-v1.db` created") + # We return only the connection, so tidy up db.close() @@ -237,6 +244,7 @@ class EDDNSender: filename = config.app_dir_path / 'replay.jsonl' try: with open(filename, 'r+', buffering=1) as replay_file: + logger.info("Converting legacy `replay.jsonl` to `eddn_queue-v1.db`") for line in replay_file: cmdr, msg = json.loads(line) self.add_message(cmdr, msg) @@ -247,15 +255,18 @@ class EDDNSender: finally: # Best effort at removing the file/contents # NB: The legacy code assumed it could write to the file. + logger.info("Converson` to `eddn_queue-v1.db` complete, removing `replay.jsonl`") replay_file = open(filename, 'w') # Will truncate replay_file.close() os.unlink(filename) def close(self) -> None: """Clean up any resources.""" + logger.debug('Closing db cursor.') if self.db: self.db.close() + logger.debug('Closing db connection.') if self.db_conn: self.db_conn.close() @@ -277,6 +288,7 @@ class EDDNSender: :param msg: The full, transmission-ready, EDDN message. :return: ID of the successfully inserted row. """ + logger.trace_if("plugin.eddn.send", f"Message for {msg['$schemaRef']=}") # Cater for legacy replay.json messages if 'header' not in msg: msg['header'] = { @@ -315,6 +327,7 @@ class EDDNSender: # Can't possibly be a valid row id return -1 + logger.trace_if("plugin.eddn.send", f"Message for {msg['$schemaRef']=} recorded, id={self.db.lastrowid}") return self.db.lastrowid or -1 def delete_message(self, row_id: int) -> None: @@ -323,6 +336,7 @@ class EDDNSender: :param row_id: id of message to be deleted. """ + logger.trace_if("plugin.eddn.send", f"Deleting message with {row_id=}") self.db.execute( """ DELETE FROM messages WHERE id = :row_id @@ -338,6 +352,7 @@ class EDDNSender: :param id: :return: """ + logger.trace_if("plugin.eddn.send", f"Sending message with {id=}") self.db.execute( """ SELECT * FROM messages WHERE id = :row_id @@ -373,6 +388,7 @@ class EDDNSender: :param msg: Fully formed, string, message. :return: `True` for "now remove this message from the queue" """ + logger.trace_if("plugin.eddn.send", "Sending message") should_return, new_data = killswitch.check_killswitch('plugins.eddn.send', json.loads(msg)) if should_return: logger.warning('eddn.send has been disabled via killswitch. Returning.') @@ -432,27 +448,31 @@ class EDDNSender: return False - def queue_check_and_send(self, reschedule: bool = False) -> None: + def queue_check_and_send(self, reschedule: bool = False) -> None: # noqa: CCR001 """ Check if we should be sending queued messages, and send if we should. :param reschedule: Boolean indicating if we should call `after()` again. """ - logger.debug("Called") + logger.trace_if("plugin.eddn.send", "Called") # Mutex in case we're already processing if not self.queue_processing.acquire(blocking=False): - logger.debug("Couldn't obtain mutex") + logger.trace_if("plugin.eddn.send", "Couldn't obtain mutex") if reschedule: + logger.trace_if("plugin.eddn.send", f"Next run scheduled for {self.eddn.REPLAY_PERIOD}ms from now") self.eddn.parent.after(self.eddn.REPLAY_PERIOD, self.queue_check_and_send, reschedule) + else: + logger.trace_if("plugin.eddn.send", "NO next run scheduled (there should be another one already set)") + return - # logger.debug("Obtained mutex") + logger.trace_if("plugin.eddn.send", "Obtained mutex") # Used to indicate if we've rescheduled at the faster rate already. have_rescheduled = False # We send either if docked or 'Delay sending until docked' not set if this.docked or not (config.get_int('output') & config.OUT_EDDN_DELAY): - # logger.debug("Should send") + logger.trace_if("plugin.eddn.send", "Should send") # We need our own cursor here, in case the semantics of # tk `after()` could allow this to run in the middle of other # database usage. @@ -488,18 +508,21 @@ class EDDNSender: # is bad", i.e. an EDDN Gateway problem. Thus, in that case # we do *NOT* schedule attempting the next message. # Always re-schedule as this is only a "Don't hammer EDDN" delay + logger.trace_if("plugin.eddn.send", f"Next run scheduled for {self.eddn.REPLAY_DELAY}ms from " + "now") self.eddn.parent.after(self.eddn.REPLAY_DELAY, self.queue_check_and_send, reschedule) have_rescheduled = True db_cursor.close() - # else: - # logger.debug("Should NOT send") + else: + logger.trace_if("plugin.eddn.send", "Should NOT send") self.queue_processing.release() - # logger.debug("Mutex released") + logger.trace_if("plugin.eddn.send", "Mutex released") if reschedule and not have_rescheduled: # Set us up to run again per the configured period + logger.trace_if("plugin.eddn.send", f"Next run scheduled for {self.eddn.REPLAY_PERIOD}ms from now") self.eddn.parent.after(self.eddn.REPLAY_PERIOD, self.queue_check_and_send, reschedule) def _log_response( @@ -935,6 +958,7 @@ class EDDN: # 'Station data' if config.get_int('output') & config.OUT_EDDN_SEND_STATION_DATA: # And user has 'station data' configured to be sent + logger.trace_if("plugin.eddn.send", "Recording/sending 'station' message") if 'header' not in msg: msg['header'] = self.standard_header() @@ -944,12 +968,14 @@ class EDDN: elif config.get_int('output') & config.OUT_EDDN_SEND_NON_STATION: # Any data that isn't 'station' is configured to be sent + logger.trace_if("plugin.eddn.send", "Recording 'non-station' message") if 'header' not in msg: msg['header'] = self.standard_header() msg_id = self.sender.add_message(cmdr, msg) if this.docked or not (config.get_int('output') & config.OUT_EDDN_DELAY): # No delay in sending configured, so attempt immediately + logger.trace_if("plugin.eddn.send", "Sending 'non-station' message") self.sender.send_message_by_id(msg_id) def standard_header( From 4d29eb605943080edcc4f89ebf79b9a8d150b975 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Thu, 24 Nov 2022 14:42:34 +0000 Subject: [PATCH 45/46] appversion: Change to 5.6.0-alpha0 for 'new EDDN code' testing * There's a Pre-Release 5.5.1-alpha0 out there, we want to look distinct from it. --- config/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/__init__.py b/config/__init__.py index 67eae43b..70d0603e 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -52,7 +52,7 @@ appcmdname = 'EDMC' # # Major.Minor.Patch(-prerelease)(+buildmetadata) # NB: Do *not* import this, use the functions appversion() and appversion_nobuild() -_static_appversion = '5.5.1-alpha0' +_static_appversion = '5.6.0-alpha0' _cached_version: Optional[semantic_version.Version] = None copyright = '© 2015-2019 Jonathan Harris, 2020-2022 EDCD' From 345938a81d8cddcd5d6e232bf316a65a1b685644 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Fri, 25 Nov 2022 15:45:07 +0000 Subject: [PATCH 46/46] eddn: Set `gamebuild=""` for CAPI-sourced data Whilst setting it to the same "CAPI-" string as `gameversion` in these cases would probably be OK, that's not the intent of the EDDN documentation, which has now been clarified. --- plugins/eddn.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/eddn.py b/plugins/eddn.py index 366f29f3..ad58d641 100644 --- a/plugins/eddn.py +++ b/plugins/eddn.py @@ -678,7 +678,7 @@ class EDDN: self.send_message(data['commander']['name'], { '$schemaRef': f'https://eddn.edcd.io/schemas/commodity/3{"/test" if is_beta else ""}', 'message': message, - 'header': self.standard_header(game_version='CAPI-market', game_build='CAPI-market'), + 'header': self.standard_header(game_version='CAPI-market', game_build=''), }) this.commodities = commodities @@ -772,7 +772,7 @@ class EDDN: ('modules', outfitting), ('odyssey', this.odyssey), ]), - 'header': self.standard_header(game_version='CAPI-shipyard', game_build='CAPI-shipyard'), + 'header': self.standard_header(game_version='CAPI-shipyard', game_build=''), }) this.outfitting = (horizons, outfitting) @@ -817,7 +817,7 @@ class EDDN: ('ships', shipyard), ('odyssey', this.odyssey), ]), - 'header': self.standard_header(game_version='CAPI-shipyard', game_build='CAPI-shipyard'), + 'header': self.standard_header(game_version='CAPI-shipyard', game_build=''), }) this.shipyard = (horizons, shipyard) @@ -1506,7 +1506,7 @@ class EDDN: msg = { '$schemaRef': f'https://eddn.edcd.io/schemas/fcmaterials_capi/1{"/test" if is_beta else ""}', 'message': entry, - 'header': self.standard_header(game_version='CAPI-market', game_build='CAPI-market'), + 'header': self.standard_header(game_version='CAPI-market', game_build=''), } this.eddn.send_message(data['commander']['name'], msg)