From 292b508985f49b12cc70a7f3d940481d3dc9b149 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 7 Aug 2021 14:20:05 +0100 Subject: [PATCH 1/3] monitor: Prepare for more-paranoid Shiplocker.json loading --- monitor.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/monitor.py b/monitor.py index f1390b14..73a99064 100644 --- a/monitor.py +++ b/monitor.py @@ -203,7 +203,7 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below key=lambda x: x.split('.')[1:] ) - self.logfile = join(self.currentdir, logfiles[-1]) if logfiles else None + self.logfile = join(self.currentdir, logfiles[-1]) if logfiles else None # type: ignore except Exception: logger.exception('Failed to find latest logfile') @@ -872,9 +872,10 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below # Always attempt loading of this. # Confirmed filename for 4.0.0.400 + currentdir_path = pathlib.Path(str(self.currentdir)) + shiplocker_filename = currentdir_path / 'ShipLocker.json' try: - currentdir_path = pathlib.Path(str(self.currentdir)) - with open(currentdir_path / 'ShipLocker.json', 'rb') as h: # type: ignore + with open(shiplocker_filename, 'rb') as h: # type: ignore entry = json.load(h, object_pairs_hook=OrderedDict) self.state['ShipLockerJSON'] = entry @@ -1601,7 +1602,7 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below self.state['GameVersion'] = entry['gameversion'] self.state['GameBuild'] = entry['build'] self.version = self.state['GameVersion'] - self.is_beta = any(v in self.version.lower() for v in ('alpha', 'beta')) + self.is_beta = any(v in self.version.lower() for v in ('alpha', 'beta')) # type: ignore except KeyError: if not suppress: raise From dc953b49a66c75bf4f230b504f0ce8fa78ff17e1 Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 7 Aug 2021 14:38:03 +0100 Subject: [PATCH 2/3] shiplocker: Implement ShipLocker.json load/decoding retries * Currently a maximum of 5 attempts, 10ms apart. * I was going to blank `entry = {}` for this, but there's a chance the file load would fail when at startup and Embark the in-Journal event actually has all the data. So we cross our fingers and hope that's the case if loading fails. * Changed the "not all the categories" logging to WARN to call out the failure. * Removed old comment about not touching Backpack here. --- monitor.py | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/monitor.py b/monitor.py index 73a99064..ed6e90d8 100644 --- a/monitor.py +++ b/monitor.py @@ -866,29 +866,38 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below elif event_type == 'shiplocker': # As of 4.0.0.400 (2021-06-10) - # "ShipLocker" will be a full list written to the journal at startup/boarding/disembarking, and also + # "ShipLocker" will be a full list written to the journal at startup/boarding, and also # written to a separate shiplocker.json file - other updates will just update that file and mention it # has changed with an empty shiplocker event in the main journal. - # Always attempt loading of this. - # Confirmed filename for 4.0.0.400 + # Always attempt loading of this, but if it fails we'll hope this was + # a startup/boarding version and thus `entry` contains + # the data anyway. currentdir_path = pathlib.Path(str(self.currentdir)) shiplocker_filename = currentdir_path / 'ShipLocker.json' - try: - with open(shiplocker_filename, 'rb') as h: # type: ignore - entry = json.load(h, object_pairs_hook=OrderedDict) - self.state['ShipLockerJSON'] = entry + shiplocker_max_attempts = 5 + shiplocker_fail_sleep = 0.01 + attempts = 0 + while attempts < shiplocker_max_attempts: + attempts += 1 + try: + with open(shiplocker_filename, 'rb') as h: # type: ignore + entry = json.load(h, object_pairs_hook=OrderedDict) + self.state['ShipLockerJSON'] = entry + break - except FileNotFoundError: - logger.warning('ShipLocker event but no ShipLocker.json file') - pass + except FileNotFoundError: + logger.warning('ShipLocker event but no ShipLocker.json file') + sleep(shiplocker_fail_sleep) + pass - except json.JSONDecodeError as e: - logger.warning(f'ShipLocker.json failed to decode:\n{e!r}\n') - pass + except json.JSONDecodeError as e: + logger.warning(f'ShipLocker.json failed to decode:\n{e!r}\n') + sleep(shiplocker_fail_sleep) + pass if not all(t in entry for t in ('Components', 'Consumables', 'Data', 'Items')): - logger.trace('ShipLocker event is an empty one (missing at least one data type)') + logger.warning('ShipLocker event is missing at least one category') # This event has the current totals, so drop any current data self.state['Component'] = defaultdict(int) @@ -896,10 +905,6 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below self.state['Item'] = defaultdict(int) self.state['Data'] = defaultdict(int) - # 4.0.0.400 - No longer zeroing out the BackPack in this event, - # as we should now always get either `Backpack` event/file or - # `BackpackChange` as needed. - clean_components = self.coalesce_cargo(entry['Components']) self.state['Component'].update( {self.canonicalise(x['Name']): x['Count'] for x in clean_components} From cc6b52a9ed74472939a8b1eba265eb63b5311bba Mon Sep 17 00:00:00 2001 From: Athanasius Date: Sat, 7 Aug 2021 14:45:46 +0100 Subject: [PATCH 3/3] Shiplocker: while/else a failure logging --- monitor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/monitor.py b/monitor.py index ed6e90d8..01a614e7 100644 --- a/monitor.py +++ b/monitor.py @@ -896,6 +896,10 @@ class EDLogs(FileSystemEventHandler): # type: ignore # See below sleep(shiplocker_fail_sleep) pass + else: + logger.warning(f'Failed to load & decode shiplocker after {shiplocker_max_attempts} tries. ' + 'Giving up.') + if not all(t in entry for t in ('Components', 'Consumables', 'Data', 'Items')): logger.warning('ShipLocker event is missing at least one category')